diff --git a/CLAUDE.md b/CLAUDE.md index 45f53733..3f590568 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -27,7 +27,7 @@ unless Jacob asks for that variant by name. | Remote | `git@github.com:dinglebear-ai/soma.git` | | Default branch | `main` | | Former names | `rmcp-template`, then `rtemplate-mcp` — both GitHub names now redirect here. Fix any local remote still pointing at `jmagar/*`. | -| Workspace | 41 cargo members (largest in the fleet) under `crates/soma/*`, `crates/shared/*`, `crates/integrations/*`, `apps/soma`, `packages/python`, plus `xtask`. Note `apps/web` and `apps/palette` are **not** cargo members — they are frontend assets. | +| Workspace | 44 cargo members (largest in the fleet) under `crates/soma/*`, `crates/shared/*`, `crates/integrations/*`, `apps/soma`, `packages/python`, plus `xtask`. Note `apps/web` and `apps/palette` are **not** cargo members — they are frontend assets. | | rmcp pin | `rmcp = { version = "=3.1.0", default-features = false }` — an **exact** pin in `[workspace.dependencies]`, deliberately duplicated on the `rmcp-client` alias entry (`Cargo.toml:81`) because TOML cannot cross-reference. Bump both together. | `crates/soma/*` is product code (this server). `crates/shared/*` is reusable diff --git a/Cargo.lock b/Cargo.lock index 2b1239c0..26d81efb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -777,14 +777,56 @@ dependencies = [ "libc", ] +[[package]] +name = "cortex-domain" +version = "0.1.0" +dependencies = [ + "chrono", + "serde", + "serde_json", + "thiserror 2.0.20", +] + [[package]] name = "cortex-ingest-core" version = "0.1.0" dependencies = [ + "serde", "serde_json", "sha2 0.10.9", ] +[[package]] +name = "cortex-inventory" +version = "0.1.0" +dependencies = [ + "serde", + "serde_json", +] + +[[package]] +name = "cortex-storage-sqlite" +version = "0.1.0" +dependencies = [ + "anyhow", + "chrono", + "cortex-domain", + "cortex-ingest-core", + "cortex-inventory", + "parking_lot", + "r2d2", + "r2d2_sqlite", + "regex", + "rusqlite", + "rustix 1.1.4", + "scheduled-thread-pool", + "serde", + "serde_json", + "sha2 0.10.9", + "tempfile", + "tracing", +] + [[package]] name = "cpp_demangle" version = "0.5.1" @@ -4491,6 +4533,28 @@ version = "6.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" +[[package]] +name = "r2d2" +version = "0.8.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51de85fb3fb6524929c8a2eb85e6b6d363de4e8c48f9e2c2eac4944abc181c93" +dependencies = [ + "log", + "parking_lot", + "scheduled-thread-pool", +] + +[[package]] +name = "r2d2_sqlite" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d362d41ad1f9a278dd971842c63c0fc7ae2e01bccbfbbc8c9a4689a46fa7051f" +dependencies = [ + "r2d2", + "rusqlite", + "uuid", +] + [[package]] name = "rand" version = "0.8.7" @@ -5157,6 +5221,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "scheduled-thread-pool" +version = "0.2.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3cbc66816425a074528352f5789333ecff06ca41b36b0b0efdfbb29edc391a19" +dependencies = [ + "parking_lot", +] + [[package]] name = "schemars" version = "0.8.22" @@ -7621,6 +7694,7 @@ checksum = "bf3923a6f5c4c6382e0b653c4117f48d631ea17f38ed86e2a828e6f7412f5239" dependencies = [ "getrandom 0.4.3", "js-sys", + "rand 0.10.2", "serde_core", "wasm-bindgen", ] diff --git a/Cargo.toml b/Cargo.toml index 56716a68..e6f3c9b7 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -14,7 +14,10 @@ members = [ "crates/shared/cli-core", "crates/shared/codemode", "crates/shared/codex-app-server-client", + "crates/shared/cortex/domain", "crates/shared/cortex/ingest-core", + "crates/shared/cortex/inventory", + "crates/shared/cortex/storage-sqlite", "crates/shared/http-api", "crates/shared/http-server", "crates/shared/incus-client", @@ -120,7 +123,10 @@ pyo3 = { version = "0.29", features = ["abi3-py311"] } zeroize = { version = "1", features = ["zeroize_derive"] } # Shared crates +cortex-domain = { path = "crates/shared/cortex/domain" } cortex-ingest-core = { path = "crates/shared/cortex/ingest-core" } +cortex-inventory = { path = "crates/shared/cortex/inventory" } +cortex-storage-sqlite = { path = "crates/shared/cortex/storage-sqlite" } rmcp-traces = { path = "crates/shared/traces" } soma-auth = { path = "crates/shared/auth" } soma-cli-core = { path = "crates/shared/cli-core" } diff --git a/README.md b/README.md index a9370ea2..ff8f10a6 100644 --- a/README.md +++ b/README.md @@ -666,12 +666,12 @@ just validate-plugin ### Workspace layout -41 cargo members: +44 cargo members: | Path | Contents | |---|---| | `crates/soma/*` | Product code for this server — domain, application, config, client, api, cli, mcp, runtime, integrations, palette, web, test-support | -| `crates/shared/*` | Reusable engine crates other servers consume — auth, mcp (client/server/proxy/gateway), provider-core, provider-adapters, http-api, http-server, observability, openapi, self-update, traces, codemode, cli-core, and namespaced reusable families such as `cortex/ingest-core` | +| `crates/shared/*` | Reusable engine crates other servers consume — auth, mcp (client/server/proxy/gateway), provider-core, provider-adapters, http-api, http-server, observability, openapi, self-update, traces, codemode, cli-core, and namespaced reusable families such as `cortex/domain`, `cortex/ingest-core`, `cortex/inventory`, and `cortex/storage-sqlite` | | `crates/integrations/*` | Upstream service bridges — `gotify`, `unifi` | | `apps/soma` | The `soma` binary and its integration tests. **The only cargo member under `apps/`** — `apps/web` (Next.js) and `apps/palette` (assets) are not Rust crates. | | `packages/python` | pyo3 Python provider platform (`abi3-py311`) | diff --git a/crates/shared/cortex/domain/Cargo.toml b/crates/shared/cortex/domain/Cargo.toml new file mode 100644 index 00000000..df7f8187 --- /dev/null +++ b/crates/shared/cortex/domain/Cargo.toml @@ -0,0 +1,32 @@ +[package] +name = "cortex-domain" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +description = "Storage- and transport-neutral domain contracts and deterministic reasoning primitives extracted from Cortex." +homepage.workspace = true +license = "AGPL-3.0-only" +repository.workspace = true +readme = "README.md" +keywords = ["observability", "logs", "incidents", "domain"] +categories = ["development-tools::debugging", "data-structures"] +publish = false + +[package.metadata.soma-architecture] +layer = "shared" + +[package.metadata.docs.rs] +all-features = true + +[features] +default = [] + +[dependencies] +chrono = "0.4" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +thiserror = "2" + +[lints] +workspace = true diff --git a/crates/shared/cortex/domain/README.md b/crates/shared/cortex/domain/README.md new file mode 100644 index 00000000..39649e40 --- /dev/null +++ b/crates/shared/cortex/domain/README.md @@ -0,0 +1,46 @@ +# cortex-domain + +`cortex-domain` is the storage- and transport-neutral semantic core extracted +from Cortex into Soma. It contains data and deterministic rules whose meaning +does not depend on SQLite, Axum, RMCP, CLI parsing, process globals, scanner +implementations, file-tail runtime state, or host configuration. + +## What belongs here + +- request identity used by application/domain policy; +- normalized log and incident entities; +- heartbeat state, pressure/status policy, and derived fleet/correlation summaries; +- graph entities, relationships, evidence, deterministic narratives, and confidence policy; +- AI incident/event entities used by deterministic finding engines; +- investigation claims and evidence summaries; +- topology findings and stable reason/category constants; +- deterministic hook/MCP/skill signal detectors; +- stable observatory identity-key construction; +- domain validation/not-found errors. + +The deterministic incident, hook, MCP, and skill finding engines were moved with +their donor parity tests because they are pure rule evaluation: no database +queries and no model calls. + +## What does not belong here + +Transport request/response envelopes, REST/MCP-specific limit policy, SQLite +rows and maintenance results, database statistics, persistence conversions, OS +journal responses, file-tail operations, scanner health implementation types, +receiver counters, inventory collection/runtime state, notification runtime config, and +process/runtime state are intentionally excluded. Pure inventory snapshot contracts live in +`cortex-inventory`, not in the domain crate. + +Database row conversions are owned by the planned `cortex-storage-sqlite` +adapter. Transport envelopes are owned by the planned `cortex-api` and +`cortex-mcp` crates. Runtime-only state belongs to the capability/runtime crate +that produces it. + +## Provenance and compatibility + +The extraction baseline is Cortex commit +`7edf23fadb94650c2d2a2f9c80111fb44319eea8`. Public semantic field names and +serde behavior are preserved for extracted types. See +`docs/cortex-extraction/MODEL-CLASSIFICATION.md` for the complete donor model +ownership inventory and `docs/cortex-extraction/VERIFICATION.md` for parity +gates. diff --git a/crates/shared/cortex/domain/src/actor.rs b/crates/shared/cortex/domain/src/actor.rs new file mode 100644 index 00000000..d02abfeb --- /dev/null +++ b/crates/shared/cortex/domain/src/actor.rs @@ -0,0 +1,74 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[serde(deny_unknown_fields)] +pub struct RequestActor { + pub surface: String, + pub display: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub subject: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub email: Option, +} + +impl RequestActor { + pub fn new(surface: impl Into, display: impl Into) -> Self { + Self { + surface: surface.into(), + display: display.into(), + subject: None, + email: None, + } + } + + pub fn api() -> Self { + Self::new("api", "api") + } + + pub fn cli() -> Self { + Self::new("cli", "cli") + } + + pub fn mcp_loopback() -> Self { + Self::new("mcp", "mcp:loopback") + } + + pub fn mcp_bearer() -> Self { + Self::new("mcp", "mcp:bearer") + } + + pub fn mcp_oauth() -> Self { + Self::new("mcp", "mcp:oauth") + } + + pub fn mcp_identity(subject: Option, email: Option) -> Self { + let display = email + .as_deref() + .filter(|value| !value.is_empty()) + .or_else(|| subject.as_deref().filter(|value| !value.is_empty())) + .unwrap_or("mcp:oauth") + .to_string(); + Self { + surface: "mcp".to_string(), + display, + subject, + email, + } + } +} + +impl From<&str> for RequestActor { + fn from(value: &str) -> Self { + Self::new("unknown", value) + } +} + +impl From for RequestActor { + fn from(value: String) -> Self { + Self::new("unknown", value) + } +} + +#[cfg(test)] +#[path = "actor_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/domain/src/actor_tests.rs b/crates/shared/cortex/domain/src/actor_tests.rs new file mode 100644 index 00000000..b1b94701 --- /dev/null +++ b/crates/shared/cortex/domain/src/actor_tests.rs @@ -0,0 +1,16 @@ +use super::*; + +#[test] +fn mcp_identity_prefers_verified_email_for_display() { + let actor = RequestActor::mcp_identity(Some("sub-123".into()), Some("me@example.com".into())); + assert_eq!(actor.surface, "mcp"); + assert_eq!(actor.display, "me@example.com"); + assert_eq!(actor.subject.as_deref(), Some("sub-123")); + assert_eq!(actor.email.as_deref(), Some("me@example.com")); +} + +#[test] +fn request_actor_wire_shape_matches_donor() { + let value = serde_json::to_value(RequestActor::api()).unwrap(); + assert_eq!(value, serde_json::json!({"surface":"api","display":"api"})); +} diff --git a/crates/shared/cortex/domain/src/ai.rs b/crates/shared/cortex/domain/src/ai.rs new file mode 100644 index 00000000..5915f2ed --- /dev/null +++ b/crates/shared/cortex/domain/src/ai.rs @@ -0,0 +1,410 @@ +use crate::{HeartbeatWindowSummary, LogEntry}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AbuseIncident { + pub incident_id: String, + pub project: String, + pub tool: String, + pub session_id: String, + pub hostname: String, + pub first_seen: String, + pub last_seen: String, + pub duration_secs: i64, + pub abuse_count: usize, + pub terms: Vec, + pub anchor_ids: Vec, + pub priority_score: f64, + pub priority_label: String, + pub window_minutes: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiCorrelationAnchor { + pub entry: LogEntry, + pub window_from: String, + pub window_to: String, + pub related: Vec, + pub related_truncated: bool, +} + +/// A graph entity a topic term resolved to. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ResolvedTopicEntity { + #[serde(rename = "type")] + pub entity_type: String, + pub key: String, + /// How it matched: `exact`, `prefix`, `label`, or `alias`. + pub match_kind: String, + /// Resolver outcome: `resolved` for exact canonical-key and alias + /// identity matches, `ambiguous` for weak label/prefix candidates that + /// never drive log fan-out. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resolver_status: Option, +} + +/// An entity reached by graph expansion from the resolved seeds. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TopicExpansionEntity { + #[serde(rename = "type")] + pub entity_type: String, + pub key: String, +} + +/// One unified-timeline row in a topic correlation. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TopicTimelineEntry { + pub timestamp: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_kind: Option, + /// Discovery lane: `agent_command`, `shell_history`, or `graph:host:`. + pub entity_path: String, + pub hostname: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub app_name: Option, + pub message: String, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub session_id: Option, + /// Why this row is in the timeline (`service_instance`, `graph_related`, + /// `host_context`). + #[serde(default, skip_serializing_if = "Option::is_none")] + pub inclusion_reason: Option, + /// Resolver outcome for this row's inclusion path. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub resolver_status: Option, + /// Set when the row was included by an explicit degraded fallback + /// (`explicit_degraded_host_context`), never silently. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub fallback_kind: Option, +} + +/// One log row in a graph-anchored session correlation, annotated with how it +/// was reached and which source lane it belongs to. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CorrelatedLogRow { + pub entry: LogEntry, + /// Source kind parsed from the row (`agent-command`, `shell-history`, + /// `syslog-udp`, `docker-stream`, ...); `None` if not recorded on the row. + #[serde(default, skip_serializing_if = "Option::is_none")] + pub source_kind: Option, + /// How the graph traversal reached this row: `agent_command`, + /// `shell_history`, or `graph:host:`. + pub discovery: String, +} + +/// Graph-anchored, session-scoped correlation result for `ai_correlate`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct GraphSessionCorrelation { + pub session_id: String, + pub session_start: String, + pub session_end: String, + /// `true` when the session's `ai_session` graph entity was found and used to + /// discover related hosts; `false` for the time-windowed fallback (session + /// not yet projected into the graph). + pub used_graph: bool, + pub discovered_hosts: Vec, + pub discovered_entities: Vec, + pub logs: Vec, + /// Count of agent-command rows (Claude's bash tool calls) in this session. + pub agent_command_count: usize, + /// Count of shell-history rows (the operator's own shell) in the window. + pub shell_history_count: usize, + /// Heartbeat pressure summaries for the discovered hosts over the window. + pub heartbeat_summaries: Vec, + pub truncated: bool, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct HookSignalCounts { + pub hook_failed: usize, + pub hook_timed_out: usize, + pub hook_output_parse_error: usize, + pub hook_invoked_too_often: usize, + pub user_correction_after_hook: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HookIncident { + pub incident_id: String, + pub hook_event: String, + pub hook_name: Option, + pub hook_source: Option, + pub tool: String, + pub project: String, + pub session_id: String, + pub hostname: String, + pub first_seen: String, + pub last_seen: String, + pub duration_secs: i64, + pub hook_event_count: usize, + pub hook_event_ids: Vec, + pub anchor_log_ids: Vec, + pub signal_counts: HookSignalCounts, + pub signals_present: Vec, + pub has_runtime_evidence: bool, + pub priority_score: f64, + pub priority_label: String, + pub window_minutes: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HookIncidentSummary { + pub incident_id: String, + pub first_seen: String, + pub last_seen: String, + pub priority_score: f64, + pub priority_label: String, + pub has_runtime_evidence: bool, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct McpSignalCounts { + pub repeated_call_failure: usize, + pub timeout_or_rate_limit: usize, + pub auth_or_permission_failure: usize, + pub schema_or_validation_error: usize, + pub unknown_tool_or_server: usize, + pub user_correction_after_tool_call: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpIncident { + pub incident_id: String, + pub mcp_server: String, + pub mcp_tool: Option, + pub tool: String, + pub project: String, + pub session_id: String, + pub hostname: String, + pub first_seen: String, + pub last_seen: String, + pub duration_secs: i64, + pub event_count: usize, + pub error_count: usize, + pub mcp_event_ids: Vec, + pub anchor_log_ids: Vec, + pub signal_counts: McpSignalCounts, + pub signals_present: Vec, + pub priority_score: f64, + pub priority_label: String, + pub window_minutes: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpIncidentSummary { + pub incident_id: String, + pub first_seen: String, + pub last_seen: String, + pub priority_score: f64, + pub priority_label: String, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct SkillSignalCounts { + pub user_correction_after_skill: usize, + pub tool_failure_after_skill: usize, + pub scope_or_source_confusion: usize, + pub ignored_skill_or_policy_instruction: usize, + pub overlong_loop_after_skill: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SkillIncident { + pub incident_id: String, + pub skill_name: String, + pub skill_plugin: Option, + pub tool: String, + pub project: String, + pub session_id: String, + pub hostname: String, + pub first_seen: String, + pub last_seen: String, + pub duration_secs: i64, + pub skill_event_count: usize, + pub skill_event_ids: Vec, + pub anchor_log_ids: Vec, + pub signal_counts: SkillSignalCounts, + pub signals_present: Vec, + pub priority_score: f64, + pub priority_label: String, + pub window_minutes: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SkillIncidentSummary { + pub incident_id: String, + pub first_seen: String, + pub last_seen: String, + pub priority_score: f64, + pub priority_label: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HookEventEntry { + pub id: i64, + pub log_id: Option, + pub ai_tool: String, + pub ai_project: Option, + pub ai_session_id: Option, + pub hostname: String, + pub timestamp: String, + pub hook_event: String, + pub hook_name: Option, + pub hook_source: Option, + pub hook_command: Option, + pub status: String, + pub exit_code: Option, + pub duration_ms: Option, + pub stdout_preview: Option, + pub stderr_preview: Option, + pub persisted_output_path: Option, + pub trusted_hash: Option, + pub evidence_kind: String, + pub metadata_json: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpEventEntry { + pub id: i64, + pub call_log_id: Option, + pub result_log_id: Option, + pub ai_tool: String, + pub ai_project: Option, + pub ai_session_id: Option, + pub hostname: String, + pub timestamp: String, + pub turn_id: Option, + pub call_id: String, + pub tool_name: String, + pub mcp_server: Option, + pub mcp_tool: Option, + pub event_kind: String, + pub status: Option, + pub duration_ms: Option, + pub is_error: Option, + pub arguments_json: Option, + pub output_preview: Option, + pub error_text: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SkillEventEntry { + pub id: i64, + pub log_id: i64, + pub ai_tool: String, + pub ai_project: Option, + pub ai_session_id: Option, + pub hostname: String, + pub timestamp: String, + pub skill_name: String, + pub skill_plugin: Option, + pub event_kind: String, + pub evidence_kind: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiSessionEntry { + /// Stable response-local key for this host/tool/project/session tuple. + pub session_key: String, + pub project: String, + pub tool: String, + pub session_id: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub transcript_path: Option, + pub hostname: String, + pub first_seen: String, + pub last_seen: String, + pub event_count: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SearchedSessionEntry { + /// Stable response-local key for this host/tool/project/session tuple. + pub session_key: String, + pub project: String, + pub tool: String, + pub session_id: String, + pub hostname: String, + pub first_seen: String, + pub last_seen: String, + pub event_count: i64, + pub match_count: i64, + #[serde(skip_serializing_if = "Option::is_none")] + pub best_snippet: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AbuseMatch { + pub term: String, + pub entry: LogEntry, + pub before: Vec, + pub after: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct UsageBlock { + pub bucket_start: String, + pub bucket_end: String, + pub project: String, + pub tool: String, + pub session_count: i64, + pub event_count: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiToolEntry { + pub tool: String, + pub event_count: i64, + pub session_count: i64, + pub first_seen: String, + pub last_seen: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiProjectEntry { + pub project: String, + pub tools: Vec, + pub event_count: i64, + pub session_count: i64, + pub first_seen: String, + pub last_seen: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CorrelatedSession { + pub session_id: String, + pub project: String, + pub tool: String, + pub match_count: i64, + #[serde(skip_serializing_if = "Option::is_none")] + pub best_snippet: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IncidentCluster { + pub hostname: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub app_name: Option, + pub window_start: String, + pub window_end: String, + pub log_count: i64, + pub severity_peak: String, + pub representative_messages: Vec, + pub correlated_sessions: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SeverityCount { + pub severity: String, + pub count: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AppLogCount { + pub app_name: Option, + pub count: i64, +} + +#[cfg(test)] +#[path = "ai_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/domain/src/ai_tests.rs b/crates/shared/cortex/domain/src/ai_tests.rs new file mode 100644 index 00000000..2e0c5c24 --- /dev/null +++ b/crates/shared/cortex/domain/src/ai_tests.rs @@ -0,0 +1,24 @@ +use super::*; + +#[test] +fn abuse_incident_wire_shape_matches_donor_fields() { + let incident = AbuseIncident { + incident_id: "inc-1".into(), + project: "/tmp/project".into(), + tool: "claude".into(), + session_id: "sess-1".into(), + hostname: "dookie".into(), + first_seen: "2026-01-01T00:00:00Z".into(), + last_seen: "2026-01-01T00:05:00Z".into(), + duration_secs: 300, + abuse_count: 2, + terms: vec!["term".into()], + anchor_ids: vec![1, 2], + priority_score: 0.8, + priority_label: "high".into(), + window_minutes: 10, + }; + let value = serde_json::to_value(incident).unwrap(); + assert_eq!(value["incident_id"], "inc-1"); + assert_eq!(value["anchor_ids"], serde_json::json!([1, 2])); +} diff --git a/crates/shared/cortex/domain/src/error.rs b/crates/shared/cortex/domain/src/error.rs new file mode 100644 index 00000000..70ae92d8 --- /dev/null +++ b/crates/shared/cortex/domain/src/error.rs @@ -0,0 +1,24 @@ +use thiserror::Error; + +/// Storage- and transport-neutral failures produced while evaluating Cortex +/// domain rules. +/// +/// Operational failures such as SQLite busy/timeout, constraint violations, +/// pool starvation, and opaque runtime errors intentionally remain outside the +/// domain crate. Adapters translate those failures at the application boundary. +#[derive(Debug, Clone, PartialEq, Eq, Error)] +pub enum DomainError { + /// Caller-supplied or derived input violates a domain invariant. + #[error("{0}")] + InvalidInput(String), + /// A requested semantic entity cannot be resolved. + #[error("{0}")] + NotFound(String), +} + +/// Result type for storage-neutral Cortex domain operations. +pub type DomainResult = Result; + +#[cfg(test)] +#[path = "error_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/domain/src/error_tests.rs b/crates/shared/cortex/domain/src/error_tests.rs new file mode 100644 index 00000000..08b9729d --- /dev/null +++ b/crates/shared/cortex/domain/src/error_tests.rs @@ -0,0 +1,13 @@ +use super::*; + +#[test] +fn domain_error_preserves_caller_facing_messages() { + assert_eq!( + DomainError::InvalidInput("bad timestamp".into()).to_string(), + "bad timestamp" + ); + assert_eq!( + DomainError::NotFound("missing host".into()).to_string(), + "missing host" + ); +} diff --git a/crates/shared/cortex/domain/src/evidence.rs b/crates/shared/cortex/domain/src/evidence.rs new file mode 100644 index 00000000..92a26bdf --- /dev/null +++ b/crates/shared/cortex/domain/src/evidence.rs @@ -0,0 +1,164 @@ +use crate::{ + AbuseIncident, HookEventEntry, HookIncident, LogEntry, McpEventEntry, McpIncident, + SkillEventEntry, SkillIncident, hook_incident_findings, incident_findings, + mcp_incident_findings, skill_incident_findings, +}; +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IncidentEvidence { + pub incident: AbuseIncident, + pub transcript_before: Vec, + pub transcript_before_truncated: bool, + pub transcript_after: Vec, + pub transcript_after_truncated: bool, + pub anchors: Vec, + pub nearby_logs: Vec, + pub nearby_logs_truncated: bool, + pub nearby_errors: Vec, + /// Deterministic, rule-based failure hypotheses and prevention hints + /// derived from this bundle (bead kmib.4). Never an LLM summary -- see + /// [`crate::incident_findings`]. + pub findings: incident_findings::IncidentFindings, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiAssessEvidenceSummary { + pub total_incidents: usize, + pub evidence_bundle_count: usize, + pub total_anchors: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HookIncidentEvidence { + pub incident: HookIncident, + pub hook_events: Vec, + pub hook_events_truncated: bool, + pub signal_anchors: Vec, + pub signal_anchors_truncated: bool, + pub transcript_before: Vec, + pub transcript_before_truncated: bool, + pub transcript_after: Vec, + pub transcript_after_truncated: bool, + pub nearby_tool_calls: Vec, + pub nearby_tool_calls_truncated: bool, + pub nearby_logs: Vec, + pub nearby_logs_truncated: bool, + pub nearby_errors: Vec, + pub nearby_errors_truncated: bool, + /// Deterministic, rule-based findings. Never an LLM summary -- see + /// [`crate::hook_incident_findings`]. + pub findings: hook_incident_findings::HookIncidentFindings, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpIncidentEvidence { + pub incident: McpIncident, + pub mcp_events: Vec, + pub mcp_events_truncated: bool, + pub signal_anchors: Vec, + pub signal_anchors_truncated: bool, + pub transcript_before: Vec, + pub transcript_before_truncated: bool, + pub transcript_after: Vec, + pub transcript_after_truncated: bool, + pub nearby_user_corrections: Vec, + pub nearby_user_corrections_truncated: bool, + pub nearby_logs: Vec, + pub nearby_logs_truncated: bool, + pub nearby_errors: Vec, + pub nearby_errors_truncated: bool, + /// Deterministic, rule-based findings. Never an LLM summary -- see + /// [`crate::mcp_incident_findings`]. + pub findings: mcp_incident_findings::McpIncidentFindings, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SkillIncidentEvidence { + pub incident: SkillIncident, + pub skill_events: Vec, + pub skill_events_truncated: bool, + pub signal_anchors: Vec, + pub signal_anchors_truncated: bool, + pub transcript_before: Vec, + pub transcript_before_truncated: bool, + pub transcript_after: Vec, + pub transcript_after_truncated: bool, + pub nearby_tool_failures: Vec, + pub nearby_tool_failures_truncated: bool, + pub nearby_user_corrections: Vec, + pub nearby_user_corrections_truncated: bool, + pub nearby_logs: Vec, + pub nearby_logs_truncated: bool, + pub nearby_errors: Vec, + pub nearby_errors_truncated: bool, + /// Deterministic, rule-based findings. Never an LLM summary -- see + /// [`crate::skill_incident_findings`]. + pub findings: skill_incident_findings::SkillIncidentFindings, +} + +/// One assessed incident's result (LLM assessment is `None` when the +/// caller requested deterministic-findings-only, e.g. `--no-llm` or an +/// MCP/REST caller). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HookAssessResult { + pub incident_id: String, + pub findings: hook_incident_findings::HookIncidentFindings, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub assessment: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prompt_preview: Option, +} + +/// One assessed incident's result (LLM assessment is `None` when the +/// caller requested deterministic-findings-only, e.g. `--no-llm` or an +/// MCP/REST caller). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpAssessResult { + pub incident_id: String, + pub findings: mcp_incident_findings::McpIncidentFindings, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub assessment: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prompt_preview: Option, +} + +/// One assessed incident's result (LLM assessment is `None` when the +/// caller requested deterministic-findings-only, e.g. `--no-llm` or an +/// MCP/REST caller -- see `src/cli/commands/assess.rs` and +/// `src/app/services/skill_assessment.rs`). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SkillAssessResult { + pub incident_id: String, + pub findings: skill_incident_findings::SkillIncidentFindings, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub assessment: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub prompt_preview: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CorrelatedHost { + pub hostname: String, + pub event_count: usize, + pub events: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ErrorSignatureEntry { + pub signature_hash: String, + pub template: String, + pub sample_message: String, + pub severity: String, + pub sample_hostname: String, + pub sample_app_name: Option, + pub first_seen_at: String, + pub last_seen_at: String, + pub total_count: i64, + pub count_last_1h: i64, + pub acknowledged_at: Option, +} + +#[cfg(test)] +#[path = "evidence_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/domain/src/evidence_tests.rs b/crates/shared/cortex/domain/src/evidence_tests.rs new file mode 100644 index 00000000..d1415388 --- /dev/null +++ b/crates/shared/cortex/domain/src/evidence_tests.rs @@ -0,0 +1,27 @@ +use super::*; + +#[test] +fn assessment_result_omits_optional_llm_fields() { + let result = HookAssessResult { + incident_id: "hook-1".into(), + findings: hook_incident_findings::HookIncidentFindings::default(), + assessment: None, + prompt_preview: None, + }; + let value = serde_json::to_value(result).unwrap(); + assert!(value.get("assessment").is_none()); + assert!(value.get("prompt_preview").is_none()); +} + +#[test] +fn error_signature_entry_preserves_operator_contract() { + let json = serde_json::json!({ + "signature_hash":"abc", "template":"failed ", "sample_message":"failed 42", + "severity":"err", "sample_hostname":"dookie", "sample_app_name":null, + "first_seen_at":"2026-01-01T00:00:00Z", "last_seen_at":"2026-01-01T00:01:00Z", + "total_count":3, "count_last_1h":2, "acknowledged_at":null + }); + let entry: ErrorSignatureEntry = serde_json::from_value(json.clone()).unwrap(); + assert_eq!(entry.total_count, 3); + assert_eq!(serde_json::to_value(entry).unwrap(), json); +} diff --git a/crates/shared/cortex/domain/src/graph.rs b/crates/shared/cortex/domain/src/graph.rs new file mode 100644 index 00000000..bb48c212 --- /dev/null +++ b/crates/shared/cortex/domain/src/graph.rs @@ -0,0 +1,122 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct GraphEntity { + pub id: i64, + pub entity_type: String, + pub canonical_key: String, + pub display_label: String, + pub source_kind: String, + pub source_id: String, + pub trust_level: String, + pub first_seen_at: Option, + pub last_seen_at: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct GraphEntityCandidate { + pub entity: GraphEntity, + pub match_reason: String, + pub alias_type: Option, + pub alias_key: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct GraphRelationship { + pub id: i64, + pub relationship_key: String, + pub src_entity_id: i64, + pub dst_entity_id: i64, + #[serde(skip_serializing_if = "Option::is_none")] + pub src_entity: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub dst_entity: Option, + pub relationship_type: String, + pub reason_code: String, + pub trust_level: String, + pub confidence: f64, + pub evidence_count: i64, + pub evidence_ids: Vec, + pub first_seen_at: Option, + pub last_seen_at: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct GraphEntitySummary { + pub id: i64, + pub entity_type: String, + pub canonical_key: String, + pub display_label: String, + pub trust_level: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct GraphEvidence { + pub id: i64, + pub relationship_id: i64, + pub source_kind: String, + pub source_id: String, + pub source_log_id: Option, + pub source_heartbeat_id: Option, + pub source_signature_hash: Option, + pub observed_at: String, + pub reason_code: String, + pub reason_text: Option, + pub confidence_delta: f64, + pub trust_level: String, + pub safe_excerpt: Option, + pub metadata_path: Option, + pub evidence_count: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct GraphSourceLogSummary { + pub id: i64, + pub timestamp: String, + pub received_at: String, + pub hostname: String, + pub severity: String, + pub app_name: Option, + pub process_id: Option, + pub source_ip: String, + pub message: String, + pub message_truncated: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct GraphIncidentNarrative { + pub title: String, + pub summary: String, + pub confidence: String, + pub relationship_ids: Vec, + pub evidence_ids: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct GraphNarrativeChain { + pub chain_id: String, + pub confidence: String, + pub score: f64, + pub summary: String, + pub entities: Vec, + pub relationships: Vec, + pub evidence_ids: Vec, + pub relationship_ids: Vec, + pub open_questions: Vec, +} + +impl From<&GraphEntity> for GraphEntitySummary { + fn from(value: &GraphEntity) -> Self { + Self { + id: value.id, + entity_type: value.entity_type.clone(), + canonical_key: value.canonical_key.clone(), + display_label: value.display_label.clone(), + trust_level: value.trust_level.clone(), + } + } +} + +#[cfg(test)] +#[path = "graph_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/domain/src/graph_confidence.rs b/crates/shared/cortex/domain/src/graph_confidence.rs new file mode 100644 index 00000000..8d5ae55c --- /dev/null +++ b/crates/shared/cortex/domain/src/graph_confidence.rs @@ -0,0 +1,140 @@ +//! Pure confidence math for the investigation graph. +//! +//! All functions here are pure (no DB, no clock): they take stored values and +//! return derived confidences. Temporal decay and evidence combination are +//! applied at *query time* on top of the stored peak confidence — nothing here +//! mutates persisted data, so there is no schema impact. +//! +//! Three ideas: +//! - **Noisy-OR** combines confidences from *independent* sources: +//! `1 - product(1 - c_i)`. Monotonic, bounded `[0,1]`, rewards corroboration. +//! - **BEWA diminishing returns** collapses *same-source* repetition: 1000 +//! syslog lines are one fact seen 1000 times, not 1000 independent facts. +//! Each doubling of `evidence_count` adds one effective observation. +//! - **CountTRuCoLa-style temporal decay** ages edges toward a floor with a +//! per-relationship half-life, so stale `runs_on` edges fade while structural +//! `worked_on` edges persist. + +const REASON_AGENT_COMMAND_CWD_INFER: &str = "agent_command_cwd_infer"; +const REASON_AGENT_COMMAND_GIT_COMMIT: &str = "agent_command_git_commit"; +const REASON_AGENT_COMMAND_SESSION: &str = "agent_command_session"; +const REASON_AI_SESSION_PROJECT: &str = "ai_session_project"; +const REASON_COMPOSE_CONFIG: &str = "compose_config"; +const REASON_DOCKER_CONTAINER_ID: &str = "docker_container_id"; +const REASON_DOCKER_NETWORK: &str = "docker_network"; +const REASON_DOCKER_SERVICE_LABEL: &str = "docker_service_label"; +const REASON_ERROR_SIGNATURE_MATCH: &str = "error_signature_match"; +const REASON_HEARTBEAT_HOST_STATE: &str = "heartbeat_host_state"; +const REASON_LOG_APP_NAME: &str = "log_app_name"; +const REASON_REVERSE_PROXY_CONFIG: &str = "reverse_proxy_config"; +const REASON_SHELL_HISTORY_GIT_COMMIT: &str = "shell_history_git_commit"; +const REASON_SYSLOG_CLAIMED_HOSTNAME: &str = "syslog_claimed_hostname"; +const TRUST_CORRELATED: &str = "correlated"; +const TRUST_REFUTED: &str = "refuted"; + +/// ln(2), the half-life constant for an exponential `exp(-lambda*t)` decay. +const LN2: f64 = std::f64::consts::LN_2; + +/// Effective-confidence ceiling for `correlated`-trust edges. `correlated` marks +/// a derivation *method* (temporal co-occurrence), not a verified fact, so its +/// confidence is capped well below structural edges. +pub const TRUST_CORRELATED_CEILING: f64 = 0.5; + +/// Cap a confidence by trust level: `refuted` edges contribute nothing, +/// `correlated` edges are capped at `TRUST_CORRELATED_CEILING`, everything else +/// passes through. Use after computing effective confidence. +pub fn apply_trust_ceiling(confidence: f64, trust_level: &str) -> f64 { + match trust_level { + TRUST_REFUTED => 0.0, + TRUST_CORRELATED => confidence.min(TRUST_CORRELATED_CEILING), + _ => confidence, + } +} + +/// Combine independent confidences via noisy-OR: `1 - product(1 - c_i)`. +/// +/// A single source is returned unchanged; corroborating sources push the result +/// up toward (never past) 1.0. Inputs are clamped to `[0, 1]`; an empty slice +/// yields 0.0. +pub fn noisy_or_combine(confidences: &[f64]) -> f64 { + let product = confidences + .iter() + .map(|c| 1.0 - c.clamp(0.0, 1.0)) + .product::(); + (1.0 - product).clamp(0.0, 1.0) +} + +/// BEWA diminishing returns: the effective independent-observation count implied +/// by a raw same-source `evidence_count`. `log2(1 + n)` — each doubling of +/// same-source evidence adds one effective unit (1→1, 1000→~10). +pub fn bewa_effective_count(evidence_count: i64) -> f64 { + if evidence_count <= 0 { + return 0.0; + } + (1.0 + evidence_count as f64).ln() / LN2 +} + +/// Confidence accumulated from `evidence_count` same-source observations, each +/// of `per_observation` confidence, with BEWA diminishing returns folded into a +/// noisy-OR: `1 - (1 - p)^effective_count`. +pub fn confidence_from_repeated(per_observation: f64, evidence_count: i64) -> f64 { + let p = per_observation.clamp(0.0, 1.0); + let n = bewa_effective_count(evidence_count); + (1.0 - (1.0 - p).powf(n)).clamp(0.0, 1.0) +} + +/// Per-hour decay rate `lambda = ln2 / half_life_hours` for a reason code. `0` means +/// the edge never decays (structural facts like session→project). +pub fn decay_lambda_per_hour(reason_code: &str) -> f64 { + let half_life_hours = match reason_code { + // Volatile runtime topology — a container's host can change on restart. + REASON_DOCKER_CONTAINER_ID | REASON_DOCKER_SERVICE_LABEL => 0.25, + // Recent observations that age over a day. + REASON_LOG_APP_NAME | REASON_SYSLOG_CLAIMED_HOSTNAME => 24.0, + // Point-in-time signals decay fast. + REASON_ERROR_SIGNATURE_MATCH | REASON_HEARTBEAT_HOST_STATE => 1.0, + // Config-derived structure is stable for weeks. + REASON_COMPOSE_CONFIG | REASON_REVERSE_PROXY_CONFIG | REASON_DOCKER_NETWORK => 720.0, + // Structural / FK-backed facts never decay. + REASON_AI_SESSION_PROJECT + | REASON_AGENT_COMMAND_SESSION + | REASON_AGENT_COMMAND_CWD_INFER + | REASON_AGENT_COMMAND_GIT_COMMIT + | REASON_SHELL_HISTORY_GIT_COMMIT => return 0.0, + // Default: slow weekly decay for anything unlisted. + _ => 168.0, + }; + LN2 / half_life_hours +} + +/// Asymptotic confidence floor `phi` for a reason code — the minimum the edge +/// decays toward as `delta_t -> infinity`. Point-in-time signals fall to 0; most edges keep +/// a small residual. +pub fn asymptotic_floor(reason_code: &str) -> f64 { + match reason_code { + REASON_ERROR_SIGNATURE_MATCH | REASON_HEARTBEAT_HOST_STATE => 0.0, + _ => 0.1, + } +} + +/// Recency factor in `[phi, 1]`: `phi + (1 - phi) * exp(-lambda * delta_t)`. +/// `lambda = 0` (never-decay edges) returns exactly 1.0; `delta_t <= 0` returns 1.0. +pub fn compute_recency(lambda_per_hour: f64, delta_hours: f64, phi: f64) -> f64 { + if lambda_per_hour <= 0.0 || delta_hours <= 0.0 { + return 1.0; + } + let phi = phi.clamp(0.0, 1.0); + phi + (1.0 - phi) * (-lambda_per_hour * delta_hours).exp() +} + +/// Query-time effective confidence: `stored * recency(reason_code, delta_t)`. +/// `delta_hours` is `(now - last_seen_at)` in hours, computed by the caller. +pub fn compute_effective_confidence(stored: f64, reason_code: &str, delta_hours: f64) -> f64 { + let lambda = decay_lambda_per_hour(reason_code); + let phi = asymptotic_floor(reason_code); + stored.clamp(0.0, 1.0) * compute_recency(lambda, delta_hours, phi) +} + +#[cfg(test)] +#[path = "graph_confidence_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/domain/src/graph_confidence_tests.rs b/crates/shared/cortex/domain/src/graph_confidence_tests.rs new file mode 100644 index 00000000..e02b7f37 --- /dev/null +++ b/crates/shared/cortex/domain/src/graph_confidence_tests.rs @@ -0,0 +1,101 @@ +//! Tests for the pure graph confidence math. + +use super::*; + +#[test] +fn noisy_or_single_source_is_unchanged() { + assert!((noisy_or_combine(&[0.5]) - 0.5).abs() < 1e-9); + assert!((noisy_or_combine(&[0.9]) - 0.9).abs() < 1e-9); + assert_eq!(noisy_or_combine(&[]), 0.0); +} + +#[test] +fn noisy_or_corroboration_increases_but_stays_bounded() { + let combined = noisy_or_combine(&[0.9, 0.9]); + assert!( + combined > 0.9, + "corroboration must raise confidence: {combined}" + ); + assert!(combined < 1.0, "noisy-OR is bounded below 1.0: {combined}"); + // 1 - (0.1 * 0.1) = 0.99 + assert!((combined - 0.99).abs() < 1e-9); +} + +#[test] +fn bewa_diminishes_same_source_repetition() { + // log2(1+1000) ≈ 9.97 — a thousand lines is ~10 effective observations. + let n = bewa_effective_count(1000); + assert!(n > 9.0 && n < 11.0, "effective count was {n}"); + assert!( + n < 1000.0 * 0.1, + "diminishing returns must be far below raw count" + ); + assert_eq!(bewa_effective_count(0), 0.0); + assert_eq!(bewa_effective_count(-5), 0.0); + + // Repeated weak evidence accumulates but never exceeds 1.0. + let c = confidence_from_repeated(0.3, 1000); + assert!(c > 0.3 && c < 1.0, "repeated confidence was {c}"); +} + +#[test] +fn recency_decays_monotonically_to_floor() { + let lambda = decay_lambda_per_hour(REASON_DOCKER_CONTAINER_ID); + let phi = asymptotic_floor(REASON_DOCKER_CONTAINER_ID); + let now = compute_recency(lambda, 0.0, phi); + let quarter = compute_recency(lambda, 0.25, phi); // one 15-min half-life + let far = compute_recency(lambda, 100.0, phi); + + assert!((now - 1.0).abs() < 1e-9, "delta_t=0 -> full recency"); + // At one half-life the (1-phi) component halves: phi + (1-phi)*0.5. + let expected = phi + (1.0 - phi) * 0.5; + assert!( + (quarter - expected).abs() < 1e-6, + "half-life recency {quarter}" + ); + assert!(quarter < now, "decays over time"); + assert!((far - phi).abs() < 1e-3, "approaches the floor {far}"); +} + +#[test] +fn never_decay_reason_keeps_full_confidence() { + // ai_session_project (lambda=0) does not decay even after a long gap. + let lambda = decay_lambda_per_hour(REASON_AI_SESSION_PROJECT); + assert_eq!(lambda, 0.0); + let eff = compute_effective_confidence(0.9, REASON_AI_SESSION_PROJECT, 10_000.0); + assert!( + (eff - 0.9).abs() < 1e-9, + "structural edge must not decay: {eff}" + ); +} + +#[test] +fn structural_old_edge_outranks_recent_volatile_edge() { + // A day-old, never-decaying structural edge vs a fresh-but-volatile edge + // that has aged a couple of hours. Effective confidence must keep the + // structural edge ahead, so beam truncation preserves it. + let structural = compute_effective_confidence(0.9, REASON_AI_SESSION_PROJECT, 24.0); + let volatile_old = compute_effective_confidence(0.95, REASON_DOCKER_CONTAINER_ID, 2.0); + assert!( + structural > volatile_old, + "structural {structural} must outrank decayed volatile {volatile_old}" + ); +} + +#[test] +fn point_in_time_signal_decays_to_zero_floor() { + let eff = compute_effective_confidence(0.8, REASON_ERROR_SIGNATURE_MATCH, 10_000.0); + assert!(eff < 0.01, "error signature decays toward 0: {eff}"); +} + +#[test] +fn trust_ceiling_caps_correlated_and_zeroes_refuted() { + // Verified passes through unchanged. + assert!((apply_trust_ceiling(0.9, "verified") - 0.9).abs() < 1e-9); + // Correlated is capped at the ceiling. + assert!((apply_trust_ceiling(0.9, TRUST_CORRELATED) - TRUST_CORRELATED_CEILING).abs() < 1e-9); + // A correlated edge already below the ceiling is unchanged. + assert!((apply_trust_ceiling(0.3, TRUST_CORRELATED) - 0.3).abs() < 1e-9); + // Refuted contributes nothing. + assert_eq!(apply_trust_ceiling(0.99, TRUST_REFUTED), 0.0); +} diff --git a/crates/shared/cortex/domain/src/graph_tests.rs b/crates/shared/cortex/domain/src/graph_tests.rs new file mode 100644 index 00000000..da42e313 --- /dev/null +++ b/crates/shared/cortex/domain/src/graph_tests.rs @@ -0,0 +1,20 @@ +use super::*; + +#[test] +fn entity_summary_preserves_domain_identity() { + let entity = GraphEntity { + id: 7, + entity_type: "host".into(), + canonical_key: "dookie".into(), + display_label: "DOOKIE".into(), + source_kind: "inventory".into(), + source_id: "host:dookie".into(), + trust_level: "observed".into(), + first_seen_at: None, + last_seen_at: None, + }; + let summary = GraphEntitySummary::from(&entity); + assert_eq!(summary.id, 7); + assert_eq!(summary.canonical_key, "dookie"); + assert_eq!(summary.trust_level, "observed"); +} diff --git a/crates/shared/cortex/domain/src/heartbeat.rs b/crates/shared/cortex/domain/src/heartbeat.rs new file mode 100644 index 00000000..bccf5a06 --- /dev/null +++ b/crates/shared/cortex/domain/src/heartbeat.rs @@ -0,0 +1,306 @@ +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HeartbeatHostState { + pub host_id: String, + pub hostname: String, + pub total_samples: usize, + pub truncated: bool, + pub flags: HeartbeatStateFlags, + pub latest: Option, + pub samples: Vec, +} + +/// Server-computed derived signals for a heartbeat sample. +/// These are the canonical source of truth for fleet views and correlation; +/// agent-supplied local flags are informational only. +/// +/// Flag computation is owned by [`heartbeat_flags_from_sample`] so every +/// transport and storage adapter shares identical thresholds and logic. +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct HeartbeatStateFlags { + // -- Availability --------------------------------------------------------- + pub collector_partial: bool, + pub heartbeat_late: bool, + pub clock_skew: bool, + // -- Resource pressure ---------------------------------------------------- + pub cpu_pressure: bool, + pub memory_pressure: bool, + pub swap_pressure: bool, + pub disk_capacity_pressure: bool, + pub network_error_pressure: bool, + pub container_unhealthy: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HeartbeatSampleState { + pub heartbeat_id: i64, + pub host_id: String, + pub hostname: String, + pub sampled_at: String, + pub received_at: String, + pub source_ip: String, + pub boot_id: String, + pub sequence: i64, + pub uptime_secs: i64, + pub collection_ms: i64, + pub partial: bool, + pub agent_version: String, + pub os: String, + pub kernel: Option, + pub architecture: String, + pub metadata: Option, + pub cpu: Option, + pub memory: Option, + pub disks: Vec, + pub network: Vec, + pub processes: Option, + pub containers: Vec, +} + +/// Return all heartbeat rows for `host_id` within `[from, to]` (inclusive), +/// with lightweight summaries for `correlate_state`. +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HeartbeatWindowSummary { + pub host_id: String, + pub hostname: String, + pub samples: usize, + pub partial_samples: usize, + pub max_cpu_usage_percent: Option, + pub min_mem_available_bytes: Option, + pub pressure_flags: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct FleetStateHostRow { + pub host_id: String, + pub hostname: String, + pub last_heartbeat_at: String, + pub status: String, + pub pressure: Vec, + pub partial: bool, + pub clock_skew: bool, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct FleetStateSummary { + pub total: usize, + pub ok: usize, + pub late: usize, + pub partial: usize, + pub pressure: usize, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CorrelateStateWindow { + pub from: String, + pub to: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CorrelateStateHostEntry { + pub host_id: String, + pub hostname: String, + pub heartbeat_summary: HeartbeatWindowSummary, + pub logs: Vec, +} + +const CPU_PRESSURE_THRESHOLD: f64 = 90.0; +const MEM_PRESSURE_THRESHOLD: f64 = 90.0; +const SWAP_PRESSURE_RATIO: f64 = 0.9; +const DISK_CAPACITY_THRESHOLD: f64 = 90.0; +const LATE_MULTIPLIER_MS: i64 = 2500; +const CLOCK_SKEW_THRESHOLD_SECS: i64 = 30; + +/// Derive canonical heartbeat state flags from a fully-loaded sample. +pub fn heartbeat_flags_from_sample(sample: &HeartbeatSampleState) -> HeartbeatStateFlags { + let interval_secs = sample + .metadata + .as_ref() + .and_then(|m| m.pointer("/agent/interval_secs")) + .and_then(Value::as_i64) + .unwrap_or(30) + .max(1); + + let max_disk = sample + .disks + .iter() + .filter_map(disk_pressure_used_percent) + .fold(None::, |acc, value| { + Some(acc.map_or(value, |current| current.max(value))) + }); + let network_errors: i64 = sample + .network + .iter() + .map(|network| { + network["rx_errors"].as_i64().unwrap_or(0) + network["tx_errors"].as_i64().unwrap_or(0) + }) + .sum(); + + HeartbeatStateFlags { + collector_partial: sample.partial, + heartbeat_late: compute_late(&sample.received_at, interval_secs), + clock_skew: compute_clock_skew(&sample.sampled_at, &sample.received_at), + cpu_pressure: sample + .cpu + .as_ref() + .and_then(|cpu| cpu["usage_percent"].as_f64()) + .is_some_and(|percent| percent > CPU_PRESSURE_THRESHOLD), + memory_pressure: sample + .memory + .as_ref() + .and_then(|memory| memory["used_percent"].as_f64()) + .is_some_and(|percent| percent > MEM_PRESSURE_THRESHOLD), + swap_pressure: swap_ratio( + sample + .memory + .as_ref() + .and_then(|memory| memory["swap_total_bytes"].as_i64()), + sample + .memory + .as_ref() + .and_then(|memory| memory["swap_used_bytes"].as_i64()), + ), + disk_capacity_pressure: max_disk.is_some_and(|percent| percent > DISK_CAPACITY_THRESHOLD), + network_error_pressure: network_errors > 0, + container_unhealthy: sample + .containers + .iter() + .any(|container| container["unhealthy"].as_i64().unwrap_or(0) > 0), + } +} + +/// Return active resource-pressure signal names in canonical order. +pub fn heartbeat_pressure_names(flags: &HeartbeatStateFlags) -> Vec { + let mut names = Vec::new(); + if flags.cpu_pressure { + names.push("cpu_pressure".to_owned()); + } + if flags.memory_pressure { + names.push("memory_pressure".to_owned()); + } + if flags.swap_pressure { + names.push("swap_pressure".to_owned()); + } + if flags.disk_capacity_pressure { + names.push("disk_capacity_pressure".to_owned()); + } + if flags.network_error_pressure { + names.push("network_error_pressure".to_owned()); + } + if flags.container_unhealthy { + names.push("container_unhealthy".to_owned()); + } + names +} + +/// Canonical fleet status label. Priority is late, partial, pressure, then ok. +pub fn heartbeat_host_status_label(flags: &HeartbeatStateFlags) -> &'static str { + let has_pressure = flags.cpu_pressure + || flags.memory_pressure + || flags.swap_pressure + || flags.disk_capacity_pressure + || flags.network_error_pressure + || flags.container_unhealthy; + if flags.heartbeat_late { + "late" + } else if flags.collector_partial { + "partial" + } else if has_pressure { + "pressure" + } else { + "ok" + } +} + +fn compute_late(received_at: &str, interval_secs: i64) -> bool { + chrono::DateTime::parse_from_rfc3339(received_at).is_ok_and(|dt| { + let elapsed = chrono::Utc::now().signed_duration_since(dt.with_timezone(&chrono::Utc)); + elapsed.num_milliseconds() > interval_secs.max(1) * LATE_MULTIPLIER_MS + }) +} + +fn compute_clock_skew(sampled_at: &str, received_at: &str) -> bool { + let sampled = chrono::DateTime::parse_from_rfc3339(sampled_at).ok(); + let received = chrono::DateTime::parse_from_rfc3339(received_at).ok(); + match (sampled, received) { + (Some(sampled), Some(received)) => { + let skew = sampled.with_timezone(&chrono::Utc) - received.with_timezone(&chrono::Utc); + skew.num_seconds().abs() > CLOCK_SKEW_THRESHOLD_SECS + } + _ => false, + } +} + +fn swap_ratio(swap_total: Option, swap_used: Option) -> bool { + match (swap_total, swap_used) { + (Some(total), Some(used)) if total > 0 => { + (used as f64 / total as f64) > SWAP_PRESSURE_RATIO + } + _ => false, + } +} + +fn disk_pressure_used_percent(disk: &Value) -> Option { + is_pressure_relevant_disk(disk) + .then(|| disk["used_percent"].as_f64()) + .flatten() +} + +fn is_pressure_relevant_disk(disk: &Value) -> bool { + let filesystem = disk["filesystem"] + .as_str() + .or_else(|| disk["fs_type"].as_str()) + .unwrap_or("") + .to_ascii_lowercase(); + let mount = disk["mountpoint"] + .as_str() + .or_else(|| disk["name"].as_str()) + .unwrap_or(""); + + if matches!( + filesystem.as_str(), + "autofs" + | "binfmt_misc" + | "bpf" + | "cgroup" + | "cgroup2" + | "configfs" + | "debugfs" + | "devpts" + | "devtmpfs" + | "efivarfs" + | "fuse.snapfuse" + | "fusectl" + | "hugetlbfs" + | "iso9660" + | "mqueue" + | "nsfs" + | "overlay" + | "proc" + | "pstore" + | "ramfs" + | "rootfs" + | "securityfs" + | "squashfs" + | "sysfs" + | "tmpfs" + | "tracefs" + ) { + return false; + } + + !matches!(mount, "" | "/init") + && !mount.starts_with("/snap/") + && !mount.starts_with("/mnt/wsl/docker-desktop/") + && !mount.starts_with("/mnt/wslg/") + && !mount.starts_with("/usr/lib/modules/") + && !mount.starts_with("/usr/lib/wsl/") + && !mount.starts_with("/run/") + && !mount.starts_with("/var/run/") +} + +#[cfg(test)] +#[path = "heartbeat_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/domain/src/heartbeat_tests.rs b/crates/shared/cortex/domain/src/heartbeat_tests.rs new file mode 100644 index 00000000..666c4b6f --- /dev/null +++ b/crates/shared/cortex/domain/src/heartbeat_tests.rs @@ -0,0 +1,79 @@ +use super::*; + +#[test] +fn heartbeat_flags_default_to_no_pressure() { + let flags = HeartbeatStateFlags::default(); + let value = serde_json::to_value(flags).unwrap(); + assert!( + value + .as_object() + .unwrap() + .values() + .all(|value| value == false) + ); +} + +#[test] +fn heartbeat_window_summary_round_trips_without_storage_types() { + let json = serde_json::json!({ + "host_id":"host-1", "hostname":"dookie", "samples":4, "partial_samples":1, + "max_cpu_usage_percent":72.5, "min_mem_available_bytes":1048576, + "pressure_flags":["cpu_pressure"] + }); + let summary: HeartbeatWindowSummary = serde_json::from_value(json.clone()).unwrap(); + assert_eq!(summary.samples, 4); + assert_eq!(serde_json::to_value(summary).unwrap(), json); +} + +#[test] +fn heartbeat_policy_matches_locked_pressure_semantics() { + let sample = HeartbeatSampleState { + heartbeat_id: 1, + host_id: "host-1".into(), + hostname: "dookie".into(), + sampled_at: "1970-01-01T00:00:00Z".into(), + received_at: "1970-01-01T00:00:00Z".into(), + source_ip: "127.0.0.1".into(), + boot_id: "boot-1".into(), + sequence: 1, + uptime_secs: 1, + collection_ms: 1, + partial: false, + agent_version: "test".into(), + os: "linux".into(), + kernel: None, + architecture: "x86_64".into(), + metadata: Some(serde_json::json!({"agent":{"interval_secs":30}})), + cpu: Some(serde_json::json!({"usage_percent":91.0})), + memory: Some(serde_json::json!({ + "used_percent": 89.0, "swap_total_bytes": 100, "swap_used_bytes": 91 + })), + disks: vec![ + serde_json::json!({"filesystem":"iso9660","mountpoint":"/snap/x","used_percent":100.0}), + serde_json::json!({"filesystem":"ext4","mountpoint":"/","used_percent":95.0}), + ], + network: vec![serde_json::json!({"rx_errors":1,"tx_errors":0})], + processes: None, + containers: vec![serde_json::json!({"unhealthy":1})], + }; + + let flags = heartbeat_flags_from_sample(&sample); + assert!(flags.heartbeat_late); + assert!(flags.cpu_pressure); + assert!(!flags.memory_pressure); + assert!(flags.swap_pressure); + assert!(flags.disk_capacity_pressure); + assert!(flags.network_error_pressure); + assert!(flags.container_unhealthy); + assert_eq!(heartbeat_host_status_label(&flags), "late"); + assert_eq!( + heartbeat_pressure_names(&flags), + vec![ + "cpu_pressure", + "swap_pressure", + "disk_capacity_pressure", + "network_error_pressure", + "container_unhealthy", + ] + ); +} diff --git a/crates/shared/cortex/domain/src/hook_incident_findings.rs b/crates/shared/cortex/domain/src/hook_incident_findings.rs new file mode 100644 index 00000000..7807204f --- /dev/null +++ b/crates/shared/cortex/domain/src/hook_incident_findings.rs @@ -0,0 +1,388 @@ +//! Deterministic failure-hypothesis and prevention-hint generation over +//! hook-usage incident evidence bundles. Pure rule evaluation -- never +//! queries the database and never calls an external LLM. Mirrors +//! `src/app/skill_incident_findings.rs` but targets hook-specific failure +//! categories from GH #105's "Suggested hook finding categories" list. +//! +//! CRITICAL: every finding function takes `has_runtime_evidence` from the +//! incident and must not claim a hook *executed* when the backing evidence +//! is config/trust-state only (`evidence_kind != "runtime_transcript"`). See +//! `evidence_kind_note` below -- every findings bundle carries an explicit +//! statement of which evidence class backs it. + +use serde::{Deserialize, Serialize}; + +use crate::{HookEventEntry, HookIncident, LogEntry}; + +// -- Stable failure-mode categories (GH #105) -------------------------------- +pub const HOOK_FAILED: &str = "hook_failed"; +pub const HOOK_TIMED_OUT: &str = "hook_timed_out"; +pub const HOOK_NOT_INVOKED: &str = "hook_not_invoked"; +pub const HOOK_INVOKED_TOO_OFTEN: &str = "hook_invoked_too_often"; +pub const HOOK_WRONG_SCOPE: &str = "hook_wrong_scope"; +pub const HOOK_OUTPUT_PARSE_ERROR: &str = "hook_output_parse_error"; +pub const HOOK_POLICY_DRIFT: &str = "hook_policy_drift"; +pub const HOOK_BLOCKED_AGENT_FLOW: &str = "hook_blocked_agent_flow"; +pub const HOOK_MUTATED_UNEXPECTED_STATE: &str = "hook_mutated_unexpected_state"; +pub const HOOK_CAUSED_TOOL_FAILURE: &str = "hook_caused_tool_failure"; +pub const UNKNOWN: &str = "unknown"; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct HookFailureMode { + pub category: String, + pub confidence: String, + pub evidence_ids: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct HookContributingFactor { + pub factor: String, + pub evidence_ids: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct HookPreventionHint { + pub category: String, + pub hint: String, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct HookIncidentFindings { + pub likely_failure_modes: Vec, + pub contributing_factors: Vec, + pub prevention_hints: Vec, + pub open_questions: Vec, + /// Explicit provenance statement -- GH #105's acceptance criterion that + /// `cortex assess hooks` "can explain whether it is using runtime hook + /// execution evidence or only config/trust-state evidence." + pub evidence_basis: String, +} + +const RUNTIME_EVIDENCE_BASIS: &str = "This incident is backed by at least one runtime_transcript hook event \ + (a Claude transcript hook-execution attachment) -- findings reflect \ + proven hook execution, not just configuration."; +const CONFIG_ONLY_EVIDENCE_BASIS: &str = "This incident is backed ONLY by config_inventory/trusted_hash_state \ + evidence (hook configuration/trust files, not a transcript-proven \ + execution). Do not treat these findings as proof the hook actually ran \ + -- they describe what is configured/trusted, not what executed."; + +pub fn evidence_basis_for(has_runtime_evidence: bool) -> String { + if has_runtime_evidence { + RUNTIME_EVIDENCE_BASIS.to_string() + } else { + CONFIG_ONLY_EVIDENCE_BASIS.to_string() + } +} + +fn confidence_for(count: usize) -> &'static str { + match count { + 0 | 1 => "low", + 2 => "medium", + _ => "high", + } +} + +const HOOK_EVENT_FACTOR_THRESHOLD: usize = 5; +const ERROR_BURST_THRESHOLD: usize = 3; + +fn scannable<'a>( + signal_anchors: &'a [LogEntry], + transcript_before: &'a [LogEntry], + transcript_after: &'a [LogEntry], + nearby_logs: &'a [LogEntry], + nearby_errors: &'a [LogEntry], +) -> impl Iterator { + signal_anchors + .iter() + .chain(transcript_before) + .chain(transcript_after) + .chain(nearby_logs) + .chain(nearby_errors) +} + +/// Derive deterministic findings from a hook-incident evidence bundle. Pure +/// and total: identical input always yields identical output; every +/// non-`unknown` failure mode / contributing factor cites at least one +/// evidence id; weak evidence yields `unknown` + `open_questions` rather than +/// an unsupported claim. +#[allow(clippy::too_many_arguments)] +pub fn derive_hook_incident_findings( + incident: &HookIncident, + hook_events: &[HookEventEntry], + signal_anchors: &[LogEntry], + transcript_before: &[LogEntry], + transcript_after: &[LogEntry], + nearby_tool_calls: &[LogEntry], + nearby_logs: &[LogEntry], + nearby_errors: &[LogEntry], +) -> HookIncidentFindings { + let mut findings = HookIncidentFindings { + evidence_basis: evidence_basis_for(incident.has_runtime_evidence), + ..Default::default() + }; + + // -- Direct signal-count-derived failure modes (from the incident's own + // deterministic signal counts, cited against the hook_events ids that + // produced them since those rows ARE the evidence for these categories) -- + let hook_event_ids: Vec = hook_events.iter().map(|e| e.id).collect(); + + if incident.signal_counts.hook_failed > 0 { + findings.likely_failure_modes.push(HookFailureMode { + category: HOOK_FAILED.to_owned(), + confidence: confidence_for(incident.signal_counts.hook_failed).to_owned(), + evidence_ids: hook_event_ids.clone(), + }); + findings.prevention_hints.push(HookPreventionHint { + category: HOOK_FAILED.to_owned(), + hint: "Review the hook command for the failure condition and add error handling or \ + a guard clause so it exits 0 on expected inputs." + .to_owned(), + }); + } + if incident.signal_counts.hook_timed_out > 0 { + findings.likely_failure_modes.push(HookFailureMode { + category: HOOK_TIMED_OUT.to_owned(), + confidence: confidence_for(incident.signal_counts.hook_timed_out).to_owned(), + evidence_ids: hook_event_ids.clone(), + }); + findings.prevention_hints.push(HookPreventionHint { + category: HOOK_TIMED_OUT.to_owned(), + hint: "Add an explicit timeout to the hook command and move slow work to a \ + background process so it does not block agent flow." + .to_owned(), + }); + } + if incident.signal_counts.hook_output_parse_error > 0 { + findings.likely_failure_modes.push(HookFailureMode { + category: HOOK_OUTPUT_PARSE_ERROR.to_owned(), + confidence: confidence_for(incident.signal_counts.hook_output_parse_error).to_owned(), + evidence_ids: hook_event_ids.clone(), + }); + findings.prevention_hints.push(HookPreventionHint { + category: HOOK_OUTPUT_PARSE_ERROR.to_owned(), + hint: "Validate the hook's stdout against its expected schema before returning it, \ + and emit structured JSON only (no mixed log lines) on stdout." + .to_owned(), + }); + } + if incident.signal_counts.hook_invoked_too_often > 0 { + findings.likely_failure_modes.push(HookFailureMode { + category: HOOK_INVOKED_TOO_OFTEN.to_owned(), + confidence: confidence_for(incident.signal_counts.hook_invoked_too_often).to_owned(), + evidence_ids: hook_event_ids.clone(), + }); + findings.prevention_hints.push(HookPreventionHint { + category: HOOK_INVOKED_TOO_OFTEN.to_owned(), + hint: "Narrow the hook's matcher/trigger event so it fires only for the intended \ + tool/event pattern instead of every turn." + .to_owned(), + }); + } + if incident.signal_counts.user_correction_after_hook > 0 { + findings.likely_failure_modes.push(HookFailureMode { + category: HOOK_BLOCKED_AGENT_FLOW.to_owned(), + confidence: confidence_for(incident.signal_counts.user_correction_after_hook) + .to_owned(), + evidence_ids: signal_anchors.iter().map(|a| a.id).collect(), + }); + findings.prevention_hints.push(HookPreventionHint { + category: HOOK_BLOCKED_AGENT_FLOW.to_owned(), + hint: "Review the hook's injected context/instructions for ambiguity or conflict \ + with the user's actual request; tighten the hook's output to be unambiguous." + .to_owned(), + }); + } + + // -- Phrase-scanned contributing factors over the transcript/log evidence -- + let mutation_hit_ids: Vec = scannable( + signal_anchors, + transcript_before, + transcript_after, + nearby_logs, + nearby_errors, + ) + .filter(|entry| { + let lower = entry.message.to_ascii_lowercase(); + [ + "unexpected config change", + "unexpected file change", + "mutated state", + ] + .iter() + .any(|kw| lower.contains(kw)) + }) + .map(|e| e.id) + .collect(); + if !mutation_hit_ids.is_empty() { + findings.likely_failure_modes.push(HookFailureMode { + category: HOOK_MUTATED_UNEXPECTED_STATE.to_owned(), + confidence: confidence_for(mutation_hit_ids.len()).to_owned(), + evidence_ids: mutation_hit_ids, + }); + findings.prevention_hints.push(HookPreventionHint { + category: HOOK_MUTATED_UNEXPECTED_STATE.to_owned(), + hint: "Scope the hook's file/config writes narrowly and document exactly what it is \ + allowed to mutate." + .to_owned(), + }); + } + + // -- Additional phrase-scanned failure modes ----------------------------- + let scope_hit_ids: Vec = scannable( + signal_anchors, + transcript_before, + transcript_after, + nearby_logs, + nearby_errors, + ) + .filter(|entry| { + let lower = entry.message.to_ascii_lowercase(); + [ + "hook fired on the wrong", + "wrong tool for this hook", + "hook matched too broadly", + "hook should not have run", + ] + .iter() + .any(|kw| lower.contains(kw)) + }) + .map(|e| e.id) + .collect(); + if !scope_hit_ids.is_empty() { + findings.likely_failure_modes.push(HookFailureMode { + category: HOOK_WRONG_SCOPE.to_owned(), + confidence: confidence_for(scope_hit_ids.len()).to_owned(), + evidence_ids: scope_hit_ids, + }); + findings.prevention_hints.push(HookPreventionHint { + category: HOOK_WRONG_SCOPE.to_owned(), + hint: "Tighten the hook's matcher so it fires only for the intended tool/event scope." + .to_owned(), + }); + } + + let drift_hit_ids: Vec = scannable( + signal_anchors, + transcript_before, + transcript_after, + nearby_logs, + nearby_errors, + ) + .filter(|entry| { + let lower = entry.message.to_ascii_lowercase(); + [ + "hook config drifted", + "hook policy changed", + "unexpected hook configuration", + "hook trust changed", + ] + .iter() + .any(|kw| lower.contains(kw)) + }) + .map(|e| e.id) + .collect(); + if !drift_hit_ids.is_empty() { + findings.likely_failure_modes.push(HookFailureMode { + category: HOOK_POLICY_DRIFT.to_owned(), + confidence: confidence_for(drift_hit_ids.len()).to_owned(), + evidence_ids: drift_hit_ids, + }); + findings.prevention_hints.push(HookPreventionHint { + category: HOOK_POLICY_DRIFT.to_owned(), + hint: "Pin the hook's config/trust state and review changes to the hook source before \ + re-trusting it." + .to_owned(), + }); + } + + // -- hook_not_invoked: a config/trust-only incident (a hook is configured + // and/or trusted but no runtime execution evidence exists in this + // incident) is the ONLY safe basis for a not-invoked signal, and ONLY as a + // low-confidence hypothesis -- per GH #105, config presence is never proof + // of non-execution across sessions, so this stays scoped to the incident's + // own evidence and is explicitly low confidence. + let has_config_evidence = hook_events + .iter() + .any(|e| e.evidence_kind == "config_inventory" || e.evidence_kind == "trusted_hash_state"); + if has_config_evidence && !incident.has_runtime_evidence { + findings.likely_failure_modes.push(HookFailureMode { + category: HOOK_NOT_INVOKED.to_owned(), + confidence: "low".to_owned(), + evidence_ids: hook_event_ids.clone(), + }); + findings.prevention_hints.push(HookPreventionHint { + category: HOOK_NOT_INVOKED.to_owned(), + hint: + "This hook is configured/trusted but shows no runtime execution evidence in this \ + window. Confirm it is wired to fire for the expected event, and compare against \ + runtime evidence for the same session before concluding it never ran." + .to_owned(), + }); + } + + if !nearby_tool_calls.is_empty() { + findings.contributing_factors.push(HookContributingFactor { + factor: format!( + "{} nearby tool-call failure(s) in the correlation window; hook output may have \ + contributed to the failing tool call.", + nearby_tool_calls.len() + ), + evidence_ids: nearby_tool_calls.iter().map(|e| e.id).collect(), + }); + findings.likely_failure_modes.push(HookFailureMode { + category: HOOK_CAUSED_TOOL_FAILURE.to_owned(), + confidence: "low".to_owned(), + evidence_ids: nearby_tool_calls.iter().map(|e| e.id).collect(), + }); + } + + if incident.hook_event_count >= HOOK_EVENT_FACTOR_THRESHOLD { + findings.contributing_factors.push(HookContributingFactor { + factor: format!( + "Repeated hook invocation: {} hook events within the incident window.", + incident.hook_event_count + ), + evidence_ids: hook_event_ids.clone(), + }); + } + if nearby_errors.len() >= ERROR_BURST_THRESHOLD { + findings.contributing_factors.push(HookContributingFactor { + factor: format!( + "Error burst: {} error-level logs in the correlation window.", + nearby_errors.len() + ), + evidence_ids: nearby_errors.iter().map(|e| e.id).collect(), + }); + } + + if findings.likely_failure_modes.is_empty() { + findings.likely_failure_modes.push(HookFailureMode { + category: UNKNOWN.to_owned(), + confidence: "low".to_owned(), + evidence_ids: Vec::new(), + }); + findings.open_questions.push( + "No deterministic failure signature matched the evidence window; manual transcript \ + review is recommended." + .to_owned(), + ); + } + if !incident.has_runtime_evidence { + findings.open_questions.push( + "No runtime_transcript evidence was found for this hook -- findings are based on \ + configuration/trust-state inventory only and do not prove the hook executed." + .to_owned(), + ); + } + if signal_anchors.is_empty() && hook_events.is_empty() { + findings + .open_questions + .push("No hook events or signal anchors were captured for this incident.".to_owned()); + } + + findings +} + +#[cfg(test)] +#[path = "hook_incident_findings_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/domain/src/hook_incident_findings_tests.rs b/crates/shared/cortex/domain/src/hook_incident_findings_tests.rs new file mode 100644 index 00000000..fbff0766 --- /dev/null +++ b/crates/shared/cortex/domain/src/hook_incident_findings_tests.rs @@ -0,0 +1,217 @@ +use super::*; +use crate::{HookEventEntry, HookIncident, HookSignalCounts, LogEntry}; + +fn log(id: i64, message: &str) -> LogEntry { + LogEntry { + id, + timestamp: "2026-01-01T00:00:00Z".to_string(), + hostname: "devhost".to_string(), + facility: None, + severity: "info".to_string(), + app_name: Some("ai-transcript".to_string()), + process_id: None, + message: message.to_string(), + received_at: "2026-01-01T00:00:00Z".to_string(), + source_ip: "127.0.0.1:0".to_string(), + ai_tool: Some("claude".to_string()), + ai_project: Some("/tmp/project".to_string()), + ai_session_id: Some("sess-1".to_string()), + ai_transcript_path: None, + metadata_json: None, + } +} + +fn hook_event_entry(id: i64) -> HookEventEntry { + HookEventEntry { + id, + log_id: Some(id), + ai_tool: "claude".to_string(), + ai_project: Some("/tmp/project".to_string()), + ai_session_id: Some("sess-1".to_string()), + hostname: "devhost".to_string(), + timestamp: "2026-01-01T00:00:00Z".to_string(), + hook_event: "PostToolUse".to_string(), + hook_name: Some("format-on-save".to_string()), + hook_source: None, + hook_command: None, + status: "failed".to_string(), + exit_code: Some(1), + duration_ms: None, + stdout_preview: None, + stderr_preview: None, + persisted_output_path: None, + trusted_hash: None, + evidence_kind: "runtime_transcript".to_string(), + metadata_json: None, + } +} + +fn incident(signal_counts: HookSignalCounts, has_runtime_evidence: bool) -> HookIncident { + let signals_present = { + let mut s = Vec::new(); + if signal_counts.hook_failed > 0 { + s.push("hook_failed".to_string()); + } + if signal_counts.hook_timed_out > 0 { + s.push("hook_timed_out".to_string()); + } + if signal_counts.hook_output_parse_error > 0 { + s.push("hook_output_parse_error".to_string()); + } + if signal_counts.hook_invoked_too_often > 0 { + s.push("hook_invoked_too_often".to_string()); + } + if signal_counts.user_correction_after_hook > 0 { + s.push("user_correction_after_hook".to_string()); + } + s + }; + HookIncident { + incident_id: "hook-inc-test".to_string(), + hook_event: "PostToolUse".to_string(), + hook_name: Some("format-on-save".to_string()), + hook_source: None, + tool: "claude".to_string(), + project: "/tmp/project".to_string(), + session_id: "sess-1".to_string(), + hostname: "devhost".to_string(), + first_seen: "2026-01-01T00:00:00Z".to_string(), + last_seen: "2026-01-01T00:05:00Z".to_string(), + duration_secs: 300, + hook_event_count: 1, + hook_event_ids: vec![1], + anchor_log_ids: vec![2], + signal_counts, + signals_present, + has_runtime_evidence, + priority_score: 22.0, + priority_label: "medium".to_string(), + window_minutes: 10, + } +} + +#[test] +fn detects_hook_failed_category_with_evidence_ids() { + let counts = HookSignalCounts { + hook_failed: 2, + ..Default::default() + }; + let inc = incident(counts, true); + let hook_events = vec![hook_event_entry(1)]; + let findings = derive_hook_incident_findings(&inc, &hook_events, &[], &[], &[], &[], &[], &[]); + let mode = findings + .likely_failure_modes + .iter() + .find(|f| f.category == HOOK_FAILED) + .expect("expected hook_failed finding"); + assert_eq!(mode.evidence_ids, vec![1]); + assert!( + findings + .prevention_hints + .iter() + .any(|h| h.category == HOOK_FAILED) + ); +} + +#[test] +fn detects_hook_timed_out_category() { + let counts = HookSignalCounts { + hook_timed_out: 1, + ..Default::default() + }; + let inc = incident(counts, true); + let hook_events = vec![hook_event_entry(1)]; + let findings = derive_hook_incident_findings(&inc, &hook_events, &[], &[], &[], &[], &[], &[]); + assert!( + findings + .likely_failure_modes + .iter() + .any(|f| f.category == HOOK_TIMED_OUT) + ); +} + +#[test] +fn detects_user_correction_as_blocked_agent_flow() { + let counts = HookSignalCounts { + user_correction_after_hook: 1, + ..Default::default() + }; + let inc = incident(counts, true); + let anchors = vec![log(2, "that's not what I asked for")]; + let findings = derive_hook_incident_findings(&inc, &[], &anchors, &[], &[], &[], &[], &[]); + let mode = findings + .likely_failure_modes + .iter() + .find(|f| f.category == HOOK_BLOCKED_AGENT_FLOW) + .expect("expected hook_blocked_agent_flow finding"); + assert_eq!(mode.evidence_ids, vec![2]); +} + +#[test] +fn runtime_evidence_basis_differs_from_config_only() { + let inc_runtime = incident(HookSignalCounts::default(), true); + let findings_runtime = + derive_hook_incident_findings(&inc_runtime, &[], &[], &[], &[], &[], &[], &[]); + assert!( + findings_runtime + .evidence_basis + .contains("runtime_transcript") + ); + assert!( + findings_runtime + .open_questions + .iter() + .all(|q| { !q.contains("configuration/trust-state inventory only") }) + ); + + let inc_config = incident(HookSignalCounts::default(), false); + let findings_config = + derive_hook_incident_findings(&inc_config, &[], &[], &[], &[], &[], &[], &[]); + assert!( + findings_config + .evidence_basis + .contains("config_inventory/trusted_hash_state") + ); + assert!( + findings_config + .open_questions + .iter() + .any(|q| q.contains("configuration/trust-state inventory only")), + "expected an explicit config-only caveat in open_questions, got {:?}", + findings_config.open_questions + ); +} + +#[test] +fn no_signals_yields_unknown_and_open_question() { + let inc = incident(HookSignalCounts::default(), true); + let findings = derive_hook_incident_findings(&inc, &[], &[], &[], &[], &[], &[], &[]); + assert!( + findings + .likely_failure_modes + .iter() + .any(|f| f.category == UNKNOWN) + ); + assert!(!findings.open_questions.is_empty()); +} + +#[test] +fn every_non_unknown_finding_cites_evidence() { + let counts = HookSignalCounts { + hook_failed: 1, + hook_output_parse_error: 1, + ..Default::default() + }; + let inc = incident(counts, true); + let hook_events = vec![hook_event_entry(1)]; + let findings = derive_hook_incident_findings(&inc, &hook_events, &[], &[], &[], &[], &[], &[]); + for mode in &findings.likely_failure_modes { + if mode.category != UNKNOWN { + assert!( + !mode.evidence_ids.is_empty(), + "category {} has no evidence ids", + mode.category + ); + } + } +} diff --git a/crates/shared/cortex/domain/src/hook_signal_detectors.rs b/crates/shared/cortex/domain/src/hook_signal_detectors.rs new file mode 100644 index 00000000..b50a9f40 --- /dev/null +++ b/crates/shared/cortex/domain/src/hook_signal_detectors.rs @@ -0,0 +1,108 @@ +//! Deterministic, phrase-boundary keyword detectors for hook-incident anchor +//! signals plus the two purely-numeric anchors (timeout/high duration and +//! invocation frequency). Pure functions/data — no DB, no LLM. Mirrors +//! `src/app/skill_signal_detectors.rs` but keyed on hook execution/config +//! evidence instead of skill-usage evidence. + +pub const SIGNAL_HOOK_FAILED: &str = "hook_failed"; +pub const SIGNAL_HOOK_TIMED_OUT: &str = "hook_timed_out"; +pub const SIGNAL_HOOK_NOT_INVOKED: &str = "hook_not_invoked"; +pub const SIGNAL_HOOK_INVOKED_TOO_OFTEN: &str = "hook_invoked_too_often"; +pub const SIGNAL_HOOK_OUTPUT_PARSE_ERROR: &str = "hook_output_parse_error"; +pub const SIGNAL_USER_CORRECTION_AFTER_HOOK: &str = "user_correction_after_hook"; + +/// All six locked anchor signal categories, in a stable order used for +/// `signals_present` sorting and CLI `--signals` validation. +pub const ALL_SIGNALS: &[&str] = &[ + SIGNAL_HOOK_FAILED, + SIGNAL_HOOK_TIMED_OUT, + SIGNAL_HOOK_NOT_INVOKED, + SIGNAL_HOOK_INVOKED_TOO_OFTEN, + SIGNAL_HOOK_OUTPUT_PARSE_ERROR, + SIGNAL_USER_CORRECTION_AFTER_HOOK, +]; + +/// Runtime status strings persisted in `ai_hook_events.status` that count as +/// a hook failure anchor. Storage and scanner adapters map their status types +/// into these stable domain values before evaluation. +const FAILURE_STATUSES: &[&str] = &["failed", "blocked", "error"]; + +pub fn is_hook_failure_status(status: &str) -> bool { + FAILURE_STATUSES.contains(&status) +} + +/// Above this `duration_ms`, a successful-or-unknown-status hook event still +/// counts as a `hook_timed_out` anchor (slow enough to plausibly block agent +/// flow even without an explicit timeout status). +pub const HOOK_TIMEOUT_DURATION_MS: i64 = 30_000; + +pub fn is_hook_timeout(_status: &str, duration_ms: Option) -> bool { + duration_ms.is_some_and(|ms| ms >= HOOK_TIMEOUT_DURATION_MS) +} + +/// Phrases in a hook's stdout/stderr preview indicating the hook's own +/// output could not be parsed/consumed by the caller (as opposed to the hook +/// process itself exiting nonzero, which is `is_hook_failure_status`). +const OUTPUT_PARSE_ERROR_PHRASES: &[&str] = &[ + "invalid json", + "json parse error", + "unexpected token", + "failed to parse hook output", + "malformed output", + "syntaxerror", +]; + +pub fn detect_hook_output_parse_error(preview: &str) -> bool { + let lower = preview.to_ascii_lowercase(); + OUTPUT_PARSE_ERROR_PHRASES.iter().any(|p| lower.contains(p)) +} + +/// Phrases indicating the user is correcting or pushing back on the +/// assistant immediately after hook-provided context/instructions. Reuses +/// the same conservative phrase-level style as +/// `skill_signal_detectors::USER_CORRECTION_PHRASES`. +const USER_CORRECTION_PHRASES: &[&str] = &[ + "that's not what i asked", + "that is not what i asked", + "you said you would", + "but you didn't", + "but you did not", + "is wrong", + "is just wrong", + "no, that's", + "no, that is", + "we wasted", + "stop, you", + "why did you", + "you shouldn't have", + "you should not have", +]; + +pub fn detect_user_correction(message: &str) -> bool { + let lower = message.to_ascii_lowercase(); + USER_CORRECTION_PHRASES.iter().any(|p| lower.contains(p)) +} + +/// Minimum invocation count for the same `(hook_event, hook_name)` pair +/// within one incident window (default 10 minutes, see +/// `search_ai_hook_incidents`'s `window_minutes`) that counts as "too +/// often". This is measured at session-window granularity, not per tool +/// call: a `PostToolUse` hook fires once per tool call by design, and a +/// single productive agentic-coding turn can easily do 10+ file +/// edits/reads within a 10-minute window (this repo's own CLAUDE.md +/// documents that pattern) without anything being wrong. Eng review fix: +/// the threshold was originally 10, chosen with a single-tool-call +/// justification that didn't match what the code actually counts +/// (invocations across the WHOLE window), causing false positives on +/// ordinary busy sessions. Raised well above a realistic productive +/// session's hook-fire count so this only fires for a genuinely +/// runaway/looping hook. +pub const HOOK_TOO_FREQUENT_THRESHOLD: usize = 30; + +pub fn detect_hook_invoked_too_often(invocation_count: usize) -> bool { + invocation_count >= HOOK_TOO_FREQUENT_THRESHOLD +} + +#[cfg(test)] +#[path = "hook_signal_detectors_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/domain/src/hook_signal_detectors_tests.rs b/crates/shared/cortex/domain/src/hook_signal_detectors_tests.rs new file mode 100644 index 00000000..e9a73651 --- /dev/null +++ b/crates/shared/cortex/domain/src/hook_signal_detectors_tests.rs @@ -0,0 +1,57 @@ +use super::*; + +#[test] +fn failure_statuses_detected() { + assert!(is_hook_failure_status("failed")); + assert!(is_hook_failure_status("blocked")); + assert!(is_hook_failure_status("error")); + assert!(!is_hook_failure_status("success")); + assert!(!is_hook_failure_status("unknown")); +} + +#[test] +fn timeout_detection_uses_duration_threshold() { + assert!(is_hook_timeout("success", Some(30_001))); + assert!(is_hook_timeout("error", Some(60_000))); + assert!(!is_hook_timeout("success", Some(1_000))); + assert!(!is_hook_timeout("success", None)); +} + +#[test] +fn output_parse_error_phrases_detected() { + assert!(detect_hook_output_parse_error( + "Error: invalid JSON in hook output" + )); + assert!(detect_hook_output_parse_error( + "SyntaxError: Unexpected token" + )); + assert!(!detect_hook_output_parse_error("hook ran fine")); +} + +#[test] +fn user_correction_phrases_detected() { + assert!(detect_user_correction("That's not what I asked for")); + assert!(detect_user_correction("Why did you run that hook again?")); + assert!(!detect_user_correction("no new errors were found")); +} + +#[test] +fn too_often_threshold() { + // Eng review fix: raised from 10 to 30 to avoid false-positiving on + // ordinary busy agentic-coding sessions (10+ tool calls in a 10-minute + // window is routine, not a runaway hook). + assert!(!detect_hook_invoked_too_often(29)); + assert!(detect_hook_invoked_too_often(30)); + assert!(detect_hook_invoked_too_often(100)); +} + +#[test] +fn all_signals_list_is_stable_and_complete() { + assert_eq!(ALL_SIGNALS.len(), 6); + assert!(ALL_SIGNALS.contains(&SIGNAL_HOOK_FAILED)); + assert!(ALL_SIGNALS.contains(&SIGNAL_HOOK_TIMED_OUT)); + assert!(ALL_SIGNALS.contains(&SIGNAL_HOOK_NOT_INVOKED)); + assert!(ALL_SIGNALS.contains(&SIGNAL_HOOK_INVOKED_TOO_OFTEN)); + assert!(ALL_SIGNALS.contains(&SIGNAL_HOOK_OUTPUT_PARSE_ERROR)); + assert!(ALL_SIGNALS.contains(&SIGNAL_USER_CORRECTION_AFTER_HOOK)); +} diff --git a/crates/shared/cortex/domain/src/incident_findings.rs b/crates/shared/cortex/domain/src/incident_findings.rs new file mode 100644 index 00000000..89d381a3 --- /dev/null +++ b/crates/shared/cortex/domain/src/incident_findings.rs @@ -0,0 +1,292 @@ +//! Deterministic failure-hypothesis and prevention-hint generation over abuse +//! incident evidence bundles (bead syslog-mcp-kmib.4). +//! +//! This is **pure rule evaluation** over an already-built evidence bundle -- it +//! never queries the database and never calls an external LLM. Every emitted +//! finding cites the log row ids that support it, confidence is conservative +//! (high only when multiple evidence items agree), and the result always +//! surfaces an `unknown` mode plus `open_questions` when the signal is weak, +//! so a downstream summariser is never tempted to overclaim a root cause. + +use serde::{Deserialize, Serialize}; + +use crate::{AbuseIncident, LogEntry}; + +// -- Stable failure-mode categories ------------------------------------------ +pub const COMMAND_FAILURE: &str = "command_failure"; +pub const TOOL_TIMEOUT: &str = "tool_timeout"; +pub const AUTH_OR_PERMISSION_FAILURE: &str = "auth_or_permission_failure"; +pub const STALE_BINARY_OR_VERSION_DRIFT: &str = "stale_binary_or_version_drift"; +pub const TEST_FAILURE: &str = "test_failure"; +pub const DOCKER_OR_SERVICE_RUNTIME_FAILURE: &str = "docker_or_service_runtime_failure"; +pub const DB_BUSY_OR_PERFORMANCE_BOTTLENECK: &str = "db_busy_or_performance_bottleneck"; +pub const UNCLEAR_INSTRUCTION_OR_SCOPE_DRIFT: &str = "unclear_instruction_or_scope_drift"; +pub const UNKNOWN: &str = "unknown"; + +/// One detected failure category with conservative confidence and the evidence +/// row ids that triggered it. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct FailureMode { + pub category: String, + /// `"low"`, `"medium"`, or `"high"`. `high` requires >=3 supporting rows; + /// `medium` requires >=2; a single hit is always `low`. + pub confidence: String, + pub evidence_ids: Vec, +} + +/// A contributing factor inferred from the evidence window. Always cites +/// evidence row ids. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ContributingFactor { + pub factor: String, + pub evidence_ids: Vec, +} + +/// A templated, category-tied prevention suggestion. +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct PreventionHint { + pub category: String, + pub hint: String, +} + +/// Deterministic findings for one incident evidence bundle. +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct IncidentFindings { + pub likely_failure_modes: Vec, + pub contributing_factors: Vec, + pub prevention_hints: Vec, + pub open_questions: Vec, +} + +/// `(category, keyword substrings, prevention hint)`. Keywords are matched +/// case-insensitively against log message text. Kept deliberately specific so +/// generic noise does not trip a category -- broad tokens like `error:` are +/// intentionally excluded. +type Rule = (&'static str, &'static [&'static str], &'static str); + +const RULES: &[Rule] = &[ + ( + TOOL_TIMEOUT, + &[ + "timeout", + "timed out", + "deadline exceeded", + "context deadline", + ], + "Add or raise an explicit timeout and retry the operation with backoff before escalating.", + ), + ( + AUTH_OR_PERMISSION_FAILURE, + &[ + "401", + "403", + "unauthorized", + "permission denied", + "forbidden", + "access denied", + "authentication failed", + ], + "Verify credentials/scopes and file or socket permissions before retrying the action.", + ), + ( + STALE_BINARY_OR_VERSION_DRIFT, + &[ + "version mismatch", + "version drift", + "stale binary", + "out of date", + "rebuild required", + "binary mismatch", + ], + "Rebuild and redeploy the affected binary/image so host and container versions match.", + ), + ( + TEST_FAILURE, + &[ + "test failed", + "tests failed", + "assertion failed", + "test result: failed", + "panicked at", + ], + "Reproduce the failing test in isolation and fix it before retrying the broader task.", + ), + ( + DOCKER_OR_SERVICE_RUNTIME_FAILURE, + &[ + "oomkilled", + "out of memory", + "container exited", + "crashloop", + "unhealthy", + "restarting", + "segmentation fault", + ], + "Inspect container/service logs and resource limits; raise limits or fix the crash loop.", + ), + ( + DB_BUSY_OR_PERFORMANCE_BOTTLENECK, + &[ + "database is locked", + "database busy", + "sqlite_busy", + "worker limit", + "too many connections", + "deadlock", + ], + "Reduce write concurrency, add retry-on-busy, or widen the DB worker/connection budget.", + ), + ( + COMMAND_FAILURE, + &[ + "command not found", + "no such file or directory", + "non-zero exit", + "exit code", + "exit status", + ], + "Confirm the command, its arguments, and working directory exist before re-running it.", + ), + ( + UNCLEAR_INSTRUCTION_OR_SCOPE_DRIFT, + &[ + "not what i asked", + "going in circles", + "wrong file", + "misunderstood", + "that is not what", + ], + "Restate the goal and acceptance criteria explicitly and confirm scope before continuing.", + ), +]; + +/// Threshold above which the raw abuse-anchor count is treated as a +/// frustration contributing factor. +const ABUSE_FACTOR_THRESHOLD: usize = 3; +/// Number of nearby error rows that constitutes an "error burst" factor. +const ERROR_BURST_THRESHOLD: usize = 3; + +fn confidence_for(count: usize) -> &'static str { + match count { + 0 | 1 => "low", + 2 => "medium", + _ => "high", + } +} + +/// Evidence rows scanned for category keywords: transcript context on both +/// sides, the abuse anchors themselves, and nearby non-AI logs/errors. +fn scannable<'a>( + anchors: &'a [LogEntry], + transcript_before: &'a [LogEntry], + transcript_after: &'a [LogEntry], + nearby_logs: &'a [LogEntry], + nearby_errors: &'a [LogEntry], +) -> impl Iterator { + anchors + .iter() + .chain(transcript_before) + .chain(transcript_after) + .chain(nearby_logs) + .chain(nearby_errors) +} + +/// Derive deterministic findings from an incident evidence bundle. +/// +/// The function is total and side-effect free: identical input always yields +/// identical output, every failure mode / contributing factor cites at least +/// one evidence id, and weak evidence yields an `unknown` mode plus +/// `open_questions` rather than an unsupported claim. +pub fn derive_incident_findings( + incident: &AbuseIncident, + anchors: &[LogEntry], + transcript_before: &[LogEntry], + transcript_after: &[LogEntry], + nearby_logs: &[LogEntry], + nearby_errors: &[LogEntry], +) -> IncidentFindings { + let mut findings = IncidentFindings::default(); + + // -- Rule evaluation: collect supporting evidence ids per category ------- + for (category, keywords, hint) in RULES { + let mut ids: Vec = Vec::new(); + for entry in scannable( + anchors, + transcript_before, + transcript_after, + nearby_logs, + nearby_errors, + ) { + let haystack = entry.message.to_ascii_lowercase(); + if keywords.iter().any(|kw| haystack.contains(kw)) { + ids.push(entry.id); + } + } + ids.sort_unstable(); + ids.dedup(); + if !ids.is_empty() { + let confidence = confidence_for(ids.len()).to_owned(); + findings.likely_failure_modes.push(FailureMode { + category: (*category).to_owned(), + confidence, + evidence_ids: ids, + }); + findings.prevention_hints.push(PreventionHint { + category: (*category).to_owned(), + hint: (*hint).to_owned(), + }); + } + } + + // -- Contributing factors (each cites evidence) -------------------------- + if incident.abuse_count >= ABUSE_FACTOR_THRESHOLD && !anchors.is_empty() { + findings.contributing_factors.push(ContributingFactor { + factor: format!( + "Elevated frustration signal: {} abuse anchors within the incident window.", + incident.abuse_count + ), + evidence_ids: anchors.iter().map(|a| a.id).collect(), + }); + } + if nearby_errors.len() >= ERROR_BURST_THRESHOLD { + findings.contributing_factors.push(ContributingFactor { + factor: format!( + "Error burst: {} error-level logs in the correlation window.", + nearby_errors.len() + ), + evidence_ids: nearby_errors.iter().map(|e| e.id).collect(), + }); + } + + // -- Open questions / unknown handling ----------------------------------- + if findings.likely_failure_modes.is_empty() { + // No deterministic signature matched -- never overclaim. + findings.likely_failure_modes.push(FailureMode { + category: UNKNOWN.to_owned(), + confidence: "low".to_owned(), + evidence_ids: Vec::new(), + }); + findings.open_questions.push( + "No deterministic failure signature matched the evidence window; manual transcript \ + review is recommended." + .to_owned(), + ); + } + if anchors.is_empty() { + findings + .open_questions + .push("No abuse anchors were captured for this incident.".to_owned()); + } + if nearby_logs.is_empty() && nearby_errors.is_empty() { + findings.open_questions.push( + "No surrounding non-AI logs were available to corroborate the transcript signal." + .to_owned(), + ); + } + + findings +} + +#[cfg(test)] +#[path = "incident_findings_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/domain/src/incident_findings_tests.rs b/crates/shared/cortex/domain/src/incident_findings_tests.rs new file mode 100644 index 00000000..268041b1 --- /dev/null +++ b/crates/shared/cortex/domain/src/incident_findings_tests.rs @@ -0,0 +1,185 @@ +use super::*; +use crate::{AbuseIncident, LogEntry}; + +fn log(id: i64, message: &str) -> LogEntry { + LogEntry { + id, + timestamp: "2026-05-25T00:00:00Z".into(), + hostname: "nashost".into(), + facility: None, + severity: "err".into(), + app_name: None, + process_id: None, + message: message.into(), + received_at: "2026-05-25T00:00:00Z".into(), + source_ip: "192.0.2.10:514".into(), + ai_tool: Some("claude".into()), + ai_project: Some("/home/jmagar/workspace/cortex".into()), + ai_session_id: Some("sess-1".into()), + ai_transcript_path: None, + metadata_json: None, + } +} + +fn incident(abuse_count: usize) -> AbuseIncident { + AbuseIncident { + incident_id: "inc-1".into(), + project: "/home/jmagar/workspace/cortex".into(), + tool: "claude".into(), + session_id: "sess-1".into(), + hostname: "nashost".into(), + first_seen: "2026-05-25T00:00:00Z".into(), + last_seen: "2026-05-25T00:05:00Z".into(), + duration_secs: 300, + abuse_count, + terms: vec!["dang".into()], + anchor_ids: vec![1], + priority_score: 1.0, + priority_label: "medium".into(), + window_minutes: 10, + } +} + +/// Convenience wrapper: most tests only vary the anchors/nearby logs. +fn derive( + anchors: &[LogEntry], + nearby_logs: &[LogEntry], + nearby_errors: &[LogEntry], +) -> IncidentFindings { + derive_incident_findings(&incident(1), anchors, &[], &[], nearby_logs, nearby_errors) +} + +fn mode<'a>(f: &'a IncidentFindings, category: &str) -> Option<&'a FailureMode> { + f.likely_failure_modes + .iter() + .find(|m| m.category == category) +} + +#[test] +fn timeout_evidence_produces_tool_timeout_with_matching_ids() { + let f = derive(&[log(10, "tool call timed out after 120s")], &[], &[]); + let m = mode(&f, TOOL_TIMEOUT).expect("tool_timeout mode"); + assert_eq!(m.evidence_ids, vec![10]); + assert_eq!(m.confidence, "low"); // single hit -> conservative + // Prevention hint is tied to the detected category. + assert!( + f.prevention_hints + .iter() + .any(|h| h.category == TOOL_TIMEOUT && !h.hint.is_empty()) + ); +} + +#[test] +fn auth_evidence_produces_auth_or_permission_failure() { + let f = derive( + &[log(11, "request returned 401 Unauthorized")], + &[log(12, "permission denied opening /var/run/docker.sock")], + &[], + ); + let m = mode(&f, AUTH_OR_PERMISSION_FAILURE).expect("auth mode"); + assert_eq!(m.evidence_ids, vec![11, 12]); + assert_eq!(m.confidence, "medium"); // two supporting rows +} + +#[test] +fn version_drift_evidence_produces_stale_binary_mode() { + let f = derive( + &[log(20, "agent version mismatch: host 1.0 container 0.9")], + &[], + &[], + ); + let m = mode(&f, STALE_BINARY_OR_VERSION_DRIFT).expect("version drift mode"); + assert_eq!(m.evidence_ids, vec![20]); +} + +#[test] +fn failing_test_output_produces_test_failure() { + let f = derive( + &[log(30, "test result: FAILED. 1 passed; 2 failed")], + &[log(31, "assertion failed: left == right")], + &[], + ); + let m = mode(&f, TEST_FAILURE).expect("test_failure mode"); + assert_eq!(m.evidence_ids, vec![30, 31]); + assert_eq!(m.confidence, "medium"); +} + +#[test] +fn high_confidence_requires_three_or_more_supporting_rows() { + let f = derive( + &[ + log(40, "database is locked"), + log(41, "database is locked"), + log(42, "database is locked"), + ], + &[], + &[], + ); + let m = mode(&f, DB_BUSY_OR_PERFORMANCE_BOTTLENECK).expect("db mode"); + assert_eq!(m.evidence_ids, vec![40, 41, 42]); + assert_eq!(m.confidence, "high"); +} + +#[test] +fn weak_noisy_evidence_produces_unknown_and_open_questions() { + // Generic chatter with no category keyword must NOT trip a category. + let f = derive(&[log(50, "thinking about the next step here")], &[], &[]); + assert_eq!(f.likely_failure_modes.len(), 1); + let m = &f.likely_failure_modes[0]; + assert_eq!(m.category, UNKNOWN); + assert!(m.evidence_ids.is_empty()); + assert!( + !f.open_questions.is_empty(), + "expected open questions for weak evidence" + ); + // No category prevention hints emitted when nothing matched. + assert!(f.prevention_hints.is_empty()); +} + +#[test] +fn contributing_factors_cite_evidence_ids() { + let anchors = vec![log(60, "ugh dang it"), log(61, "still broken")]; + let nearby_errors = vec![ + log(70, "connection refused"), + log(71, "connection refused"), + log(72, "connection refused"), + ]; + let f = derive_incident_findings(&incident(4), &anchors, &[], &[], &[], &nearby_errors); + // Abuse-count factor cites the anchor ids. + let abuse_factor = f + .contributing_factors + .iter() + .find(|c| c.factor.contains("frustration")) + .expect("frustration factor"); + assert_eq!(abuse_factor.evidence_ids, vec![60, 61]); + // Error-burst factor cites the error ids. + let burst = f + .contributing_factors + .iter() + .find(|c| c.factor.contains("Error burst")) + .expect("error burst factor"); + assert_eq!(burst.evidence_ids, vec![70, 71, 72]); +} + +#[test] +fn findings_are_deterministic_for_fixed_input() { + let anchors = vec![log(80, "operation timed out"), log(81, "401 unauthorized")]; + let a = derive(&anchors, &[], &[]); + let b = derive(&anchors, &[], &[]); + assert_eq!(a, b); +} + +#[test] +fn every_failure_mode_cites_evidence_unless_unknown() { + let f = derive(&[log(90, "command not found: just")], &[], &[]); + for m in &f.likely_failure_modes { + if m.category == UNKNOWN { + continue; + } + assert!( + !m.evidence_ids.is_empty(), + "non-unknown mode {} must cite evidence", + m.category + ); + } +} diff --git a/crates/shared/cortex/domain/src/investigation.rs b/crates/shared/cortex/domain/src/investigation.rs new file mode 100644 index 00000000..f053f9af --- /dev/null +++ b/crates/shared/cortex/domain/src/investigation.rs @@ -0,0 +1,92 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct InvestigationBudget { + pub max_graph_calls: u32, + pub max_log_rows: u32, + pub max_evidence_rows: u32, + pub max_candidate_explanations: u32, + pub max_wall_time_ms: u32, + pub max_payload_bytes: u32, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct InvestigationBudgetUsed { + pub graph_calls: u32, + pub log_rows: u32, + pub evidence_rows: u32, + pub candidate_explanations: u32, + pub wall_time_ms: u32, + pub payload_bytes: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum InvestigationClaimType { + Verified, + SupportedCorrelation, + WeakCorrelation, + OpenQuestion, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct InvestigationClaim { + pub claim_type: InvestigationClaimType, + pub title: String, + pub summary: String, + pub confidence: String, + pub relationship_ids: Vec, + pub evidence_ids: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct AppEntitySummary { + pub id: i64, + pub entity_type: String, + pub key: String, + pub label: String, + pub trust_level: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct AppRelationshipSummary { + pub id: i64, + pub source_entity_id: i64, + pub target_entity_id: i64, + pub relationship_type: String, + pub reason_code: String, + pub trust_level: String, + pub confidence: f64, + pub evidence_count: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct AppEvidenceSummary { + pub id: i64, + pub relationship_id: i64, + pub source_kind: String, + pub source_log_id: Option, + pub observed_at: String, + pub reason_code: String, + pub reason_text: Option, + pub confidence_delta: f64, + pub trust_level: String, + pub excerpt: Option, + pub missing_source_reason: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct AppLogSummary { + pub id: i64, + pub timestamp: String, + pub received_at: String, + pub hostname: String, + pub severity: String, + pub app_name: Option, + pub message: String, + pub message_truncated: bool, +} + +#[cfg(test)] +#[path = "investigation_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/domain/src/investigation_tests.rs b/crates/shared/cortex/domain/src/investigation_tests.rs new file mode 100644 index 00000000..9951ac0c --- /dev/null +++ b/crates/shared/cortex/domain/src/investigation_tests.rs @@ -0,0 +1,14 @@ +use super::*; + +#[test] +fn claim_type_uses_stable_snake_case_wire_values() { + assert_eq!( + serde_json::to_value(InvestigationClaimType::SupportedCorrelation).unwrap(), + "supported_correlation" + ); + assert_eq!( + serde_json::from_value::(serde_json::json!("open_question")) + .unwrap(), + InvestigationClaimType::OpenQuestion + ); +} diff --git a/crates/shared/cortex/domain/src/lib.rs b/crates/shared/cortex/domain/src/lib.rs new file mode 100644 index 00000000..b1708b9e --- /dev/null +++ b/crates/shared/cortex/domain/src/lib.rs @@ -0,0 +1,40 @@ +//! Storage- and transport-neutral Cortex domain contracts. +//! +//! This crate owns product meaning that remains useful when SQLite, HTTP, MCP, +//! CLI, process supervision, and host-specific collectors are replaced. It +//! intentionally does not expose database row types, filesystem paths, scanner +//! implementations, receiver counters, runtime configuration, or transport +//! request/response envelopes. +//! +//! The source was extracted from Cortex donor commit +//! `7edf23fadb94650c2d2a2f9c80111fb44319eea8`. Database-to-domain mapping +//! remains adapter work and belongs to `cortex-storage-sqlite`. + +pub mod actor; +pub mod ai; +pub mod error; +pub mod evidence; +pub mod graph; +pub mod graph_confidence; +pub mod heartbeat; +pub mod hook_incident_findings; +pub mod hook_signal_detectors; +pub mod incident_findings; +pub mod investigation; +pub mod logs; +pub mod mcp_incident_findings; +pub mod mcp_signal_detectors; +pub mod observatory_identity; +pub mod skill_incident_findings; +pub mod skill_signal_detectors; +pub mod topology; + +pub use actor::RequestActor; +pub use ai::*; +pub use error::{DomainError, DomainResult}; +pub use evidence::*; +pub use graph::*; +pub use heartbeat::*; +pub use investigation::*; +pub use logs::*; +pub use topology::*; diff --git a/crates/shared/cortex/domain/src/logs.rs b/crates/shared/cortex/domain/src/logs.rs new file mode 100644 index 00000000..9d1e904a --- /dev/null +++ b/crates/shared/cortex/domain/src/logs.rs @@ -0,0 +1,35 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IncidentEvent { + pub timestamp: String, + pub source: String, + pub host: Option, + pub severity: Option, + pub app: Option, + pub message: String, + pub log_id: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct LogEntry { + pub id: i64, + pub timestamp: String, + pub hostname: String, + pub facility: Option, + pub severity: String, + pub app_name: Option, + pub process_id: Option, + pub message: String, + pub received_at: String, + pub source_ip: String, + pub ai_tool: Option, + pub ai_project: Option, + pub ai_session_id: Option, + pub ai_transcript_path: Option, + pub metadata_json: Option, +} + +#[cfg(test)] +#[path = "logs_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/domain/src/logs_tests.rs b/crates/shared/cortex/domain/src/logs_tests.rs new file mode 100644 index 00000000..7d770444 --- /dev/null +++ b/crates/shared/cortex/domain/src/logs_tests.rs @@ -0,0 +1,29 @@ +use super::*; + +fn sample() -> LogEntry { + LogEntry { + id: 42, + timestamp: "2026-01-01T00:00:00Z".into(), + hostname: "claimed-host".into(), + facility: Some("local0".into()), + severity: "warning".into(), + app_name: Some("rsyslogd".into()), + process_id: Some("123".into()), + message: "message".into(), + received_at: "2026-01-01T00:00:01Z".into(), + source_ip: "192.0.2.10:514".into(), + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: None, + } +} + +#[test] +fn log_entry_wire_shape_preserves_network_sender_identity() { + let value = serde_json::to_value(sample()).unwrap(); + assert_eq!(value["hostname"], "claimed-host"); + assert_eq!(value["source_ip"], "192.0.2.10:514"); + assert_eq!(value["app_name"], "rsyslogd"); +} diff --git a/crates/shared/cortex/domain/src/mcp_incident_findings.rs b/crates/shared/cortex/domain/src/mcp_incident_findings.rs new file mode 100644 index 00000000..cbe13e69 --- /dev/null +++ b/crates/shared/cortex/domain/src/mcp_incident_findings.rs @@ -0,0 +1,271 @@ +//! Deterministic failure-hypothesis and prevention-hint generation over +//! MCP-incident evidence bundles. Pure rule evaluation -- never queries the +//! database and never calls an external LLM. Mirrors +//! `src/app/skill_incident_findings.rs` but targets the MCP finding +//! categories from GH #94's "Suggested MCP finding categories" section. + +use serde::{Deserialize, Serialize}; + +use crate::{LogEntry, McpIncident}; + +// -- Stable failure-mode categories (GH #94 "Suggested MCP finding +// categories") -------------------------------------------------------------- +pub const WRONG_MCP_TOOL_SELECTED: &str = "wrong_mcp_tool_selected"; +pub const MCP_SERVER_UNAVAILABLE: &str = "mcp_server_unavailable"; +pub const MCP_AUTH_OR_PERMISSION_FAILURE: &str = "mcp_auth_or_permission_failure"; +pub const MCP_SCHEMA_MISMATCH: &str = "mcp_schema_mismatch"; +pub const MCP_TIMEOUT_OR_RATE_LIMIT: &str = "mcp_timeout_or_rate_limit"; +pub const MCP_RESULT_MISINTERPRETED: &str = "mcp_result_misinterpreted"; +pub const MISSING_MCP_DISCOVERY_STEP: &str = "missing_mcp_discovery_step"; +pub const TOOL_SURFACE_CONFUSION: &str = "tool_surface_confusion"; +pub const UNKNOWN: &str = "unknown"; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct McpFailureMode { + pub category: String, + pub confidence: String, + pub evidence_ids: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct McpContributingFactor { + pub factor: String, + pub evidence_ids: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct McpPreventionHint { + pub category: String, + pub hint: String, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct McpIncidentFindings { + pub likely_failure_modes: Vec, + pub contributing_factors: Vec, + pub prevention_hints: Vec, + pub open_questions: Vec, +} + +/// `(category, keyword substrings, prevention hint)`. Kept specific -- no +/// broad single tokens. +type Rule = (&'static str, &'static [&'static str], &'static str); + +const RULES: &[Rule] = &[ + ( + MCP_SERVER_UNAVAILABLE, + &[ + "server unavailable", + "server not found", + "not connected", + "disconnected", + "mcp server error", + ], + "Check the MCP server's connection/health before retrying, and surface a clear \ + reconnect step in the calling skill/doc instead of retrying blind.", + ), + ( + MCP_AUTH_OR_PERMISSION_FAILURE, + &[ + "permission denied", + "unauthorized", + "forbidden", + "authentication failed", + "auth failed", + "invalid token", + "invalid credentials", + ], + "Add an auth-check step before the tool call (verify token/credential presence) and \ + document the expected auth setup for this MCP server.", + ), + ( + MCP_SCHEMA_MISMATCH, + &[ + "schema validation", + "invalid parameters", + "invalid arguments", + "missing required", + "does not match schema", + "validation error", + "invalidparams", + ], + "Review the tool's parameter schema against how the agent is calling it; update the \ + skill/tool doc with a concrete example matching the current schema.", + ), + ( + MCP_TIMEOUT_OR_RATE_LIMIT, + &[ + "timed out", + "timeout", + "rate limit", + "rate-limited", + "too many requests", + ], + "Add a backoff/retry policy note for this tool, and document a lower-cost alternative \ + call pattern if repeated calls are triggering rate limits.", + ), + ( + WRONG_MCP_TOOL_SELECTED, + &["wrong tool", "not the right tool", "you used the wrong"], + "Narrow the tool's description/trigger phrases so it's less likely to be selected for \ + out-of-scope requests, and cross-reference the correct tool in its docstring.", + ), + ( + TOOL_SURFACE_CONFUSION, + &[ + "unknown tool", + "tool not found", + "no such tool", + "wrong server", + ], + "Add a discovery step (list available tools/servers) before assuming a specific tool \ + name, and document the exact tool surface this skill depends on.", + ), + ( + MCP_RESULT_MISINTERPRETED, + &[ + "that's not what i asked", + "that is not what i asked", + "misread the result", + "misinterpreted", + ], + "Add a verification step requiring the agent to restate what the tool result actually \ + showed before acting on it.", + ), + ( + MISSING_MCP_DISCOVERY_STEP, + &[ + "should have checked", + "should have searched", + "didn't check available tools", + "did not check available tools", + ], + "Add an explicit discovery/search step to the skill doc before assuming a tool is \ + unavailable or using a guessed tool name.", + ), +]; + +fn confidence_for(count: usize) -> &'static str { + match count { + 0 | 1 => "low", + 2 => "medium", + _ => "high", + } +} + +const EVENT_COUNT_FACTOR_THRESHOLD: usize = 3; +const ERROR_BURST_THRESHOLD: usize = 2; + +fn scannable<'a>( + signal_anchors: &'a [LogEntry], + transcript_before: &'a [LogEntry], + transcript_after: &'a [LogEntry], + nearby_logs: &'a [LogEntry], + nearby_errors: &'a [LogEntry], +) -> impl Iterator { + signal_anchors + .iter() + .chain(transcript_before) + .chain(transcript_after) + .chain(nearby_logs) + .chain(nearby_errors) +} + +/// Derive deterministic findings from an MCP-incident evidence bundle. Pure +/// and total: identical input always yields identical output; every +/// non-`unknown` failure mode / contributing factor cites at least one +/// evidence id; weak evidence yields `unknown` + `open_questions` rather +/// than an unsupported claim. +#[allow(clippy::too_many_arguments)] +pub fn derive_mcp_incident_findings( + incident: &McpIncident, + _mcp_events: &[crate::McpEventEntry], + signal_anchors: &[LogEntry], + transcript_before: &[LogEntry], + transcript_after: &[LogEntry], + nearby_logs: &[LogEntry], + nearby_errors: &[LogEntry], +) -> McpIncidentFindings { + let mut findings = McpIncidentFindings::default(); + + for (category, keywords, hint) in RULES { + let mut ids: Vec = Vec::new(); + for entry in scannable( + signal_anchors, + transcript_before, + transcript_after, + nearby_logs, + nearby_errors, + ) { + let haystack = entry.message.to_ascii_lowercase(); + if keywords.iter().any(|kw| haystack.contains(kw)) { + ids.push(entry.id); + } + } + ids.sort_unstable(); + ids.dedup(); + if !ids.is_empty() { + let confidence = confidence_for(ids.len()).to_owned(); + findings.likely_failure_modes.push(McpFailureMode { + category: (*category).to_owned(), + confidence, + evidence_ids: ids, + }); + findings.prevention_hints.push(McpPreventionHint { + category: (*category).to_owned(), + hint: (*hint).to_owned(), + }); + } + } + + if incident.event_count >= EVENT_COUNT_FACTOR_THRESHOLD && incident.error_count > 0 { + findings.contributing_factors.push(McpContributingFactor { + factor: format!( + "Repeated tool calls with errors: {} events ({} errors) within the incident window.", + incident.event_count, incident.error_count + ), + evidence_ids: signal_anchors.iter().map(|a| a.id).collect(), + }); + } + if incident.error_count >= ERROR_BURST_THRESHOLD { + findings.contributing_factors.push(McpContributingFactor { + factor: format!( + "Error burst: {} error-flagged MCP events for {}/{}.", + incident.error_count, + incident.mcp_server, + incident.mcp_tool.as_deref().unwrap_or("*") + ), + evidence_ids: nearby_errors.iter().map(|e| e.id).collect(), + }); + } + + if findings.likely_failure_modes.is_empty() { + findings.likely_failure_modes.push(McpFailureMode { + category: UNKNOWN.to_owned(), + confidence: "low".to_owned(), + evidence_ids: Vec::new(), + }); + findings.open_questions.push( + "No deterministic failure signature matched the evidence window; manual transcript \ + review is recommended." + .to_owned(), + ); + } + if signal_anchors.is_empty() { + findings + .open_questions + .push("No negative signal anchors were captured for this incident.".to_owned()); + } + if nearby_logs.is_empty() && nearby_errors.is_empty() { + findings.open_questions.push( + "No surrounding non-AI logs were available to corroborate the transcript signal." + .to_owned(), + ); + } + + findings +} + +#[cfg(test)] +#[path = "mcp_incident_findings_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/domain/src/mcp_incident_findings_tests.rs b/crates/shared/cortex/domain/src/mcp_incident_findings_tests.rs new file mode 100644 index 00000000..a8919d52 --- /dev/null +++ b/crates/shared/cortex/domain/src/mcp_incident_findings_tests.rs @@ -0,0 +1,152 @@ +use super::*; +use crate::{LogEntry, McpIncident, McpSignalCounts}; + +fn log(id: i64, message: &str) -> LogEntry { + LogEntry { + id, + timestamp: "2026-01-01T00:00:00Z".to_string(), + hostname: "devhost".to_string(), + facility: None, + severity: "info".to_string(), + app_name: Some("ai-transcript".to_string()), + process_id: None, + message: message.to_string(), + received_at: "2026-01-01T00:00:00Z".to_string(), + source_ip: "127.0.0.1:0".to_string(), + ai_tool: Some("codex".to_string()), + ai_project: Some("/tmp/project".to_string()), + ai_session_id: Some("sess-1".to_string()), + ai_transcript_path: None, + metadata_json: None, + } +} + +fn incident(signals_present: Vec<&str>) -> McpIncident { + McpIncident { + incident_id: "mcp-inc-test".to_string(), + mcp_server: "labby".to_string(), + mcp_tool: Some("search".to_string()), + tool: "codex".to_string(), + project: "/tmp/project".to_string(), + session_id: "sess-1".to_string(), + hostname: "devhost".to_string(), + first_seen: "2026-01-01T00:00:00Z".to_string(), + last_seen: "2026-01-01T00:05:00Z".to_string(), + duration_secs: 300, + event_count: 3, + error_count: 2, + mcp_event_ids: vec![1], + anchor_log_ids: vec![2], + signal_counts: McpSignalCounts::default(), + signals_present: signals_present.into_iter().map(String::from).collect(), + priority_score: 22.0, + priority_label: "medium".to_string(), + window_minutes: 10, + } +} + +#[test] +fn detects_mcp_server_unavailable_category() { + let inc = incident(vec!["unknown_tool_or_server"]); + let anchors = vec![log(2, "mcp server error: server unavailable right now")]; + let findings = derive_mcp_incident_findings(&inc, &[], &anchors, &[], &[], &[], &[]); + assert!( + findings + .likely_failure_modes + .iter() + .any(|f| f.category == MCP_SERVER_UNAVAILABLE), + "expected mcp_server_unavailable category, got {:?}", + findings.likely_failure_modes + ); +} + +#[test] +fn detects_auth_or_permission_failure_category() { + let inc = incident(vec!["auth_or_permission_failure"]); + let anchors = vec![log(2, "permission denied calling the tool")]; + let findings = derive_mcp_incident_findings(&inc, &[], &anchors, &[], &[], &[], &[]); + assert!( + findings + .likely_failure_modes + .iter() + .any(|f| f.category == MCP_AUTH_OR_PERMISSION_FAILURE) + ); +} + +#[test] +fn detects_schema_mismatch_category() { + let inc = incident(vec!["schema_or_validation_error"]); + let anchors = vec![log(2, "schema validation failed for the request")]; + let findings = derive_mcp_incident_findings(&inc, &[], &anchors, &[], &[], &[], &[]); + assert!( + findings + .likely_failure_modes + .iter() + .any(|f| f.category == MCP_SCHEMA_MISMATCH) + ); +} + +#[test] +fn detects_timeout_or_rate_limit_category() { + let inc = incident(vec!["timeout_or_rate_limit"]); + let anchors = vec![log(2, "the call timed out after 30 seconds")]; + let findings = derive_mcp_incident_findings(&inc, &[], &anchors, &[], &[], &[], &[]); + assert!( + findings + .likely_failure_modes + .iter() + .any(|f| f.category == MCP_TIMEOUT_OR_RATE_LIMIT) + ); +} + +#[test] +fn every_non_unknown_failure_mode_cites_evidence() { + let inc = incident(vec!["auth_or_permission_failure"]); + let anchors = vec![log(2, "permission denied calling the tool")]; + let findings = derive_mcp_incident_findings(&inc, &[], &anchors, &[], &[], &[], &[]); + for mode in &findings.likely_failure_modes { + if mode.category != UNKNOWN { + assert!( + !mode.evidence_ids.is_empty(), + "non-unknown category {} must cite evidence", + mode.category + ); + } + } +} + +#[test] +fn no_matching_evidence_yields_unknown_and_open_question() { + let inc = incident(vec![]); + let findings = derive_mcp_incident_findings(&inc, &[], &[], &[], &[], &[], &[]); + assert_eq!(findings.likely_failure_modes.len(), 1); + assert_eq!(findings.likely_failure_modes[0].category, UNKNOWN); + assert!(!findings.open_questions.is_empty()); +} + +#[test] +fn repeated_errors_contributing_factor_present_when_threshold_met() { + let inc = incident(vec![]); + let findings = derive_mcp_incident_findings(&inc, &[], &[], &[], &[], &[], &[]); + assert!( + findings + .contributing_factors + .iter() + .any(|f| f.factor.contains("Repeated tool calls")) + ); + assert!( + findings + .contributing_factors + .iter() + .any(|f| f.factor.contains("Error burst")) + ); +} + +#[test] +fn deterministic_output_for_identical_input() { + let inc = incident(vec!["auth_or_permission_failure"]); + let anchors = vec![log(2, "permission denied calling the tool")]; + let f1 = derive_mcp_incident_findings(&inc, &[], &anchors, &[], &[], &[], &[]); + let f2 = derive_mcp_incident_findings(&inc, &[], &anchors, &[], &[], &[], &[]); + assert_eq!(f1, f2); +} diff --git a/crates/shared/cortex/domain/src/mcp_signal_detectors.rs b/crates/shared/cortex/domain/src/mcp_signal_detectors.rs new file mode 100644 index 00000000..5a208f88 --- /dev/null +++ b/crates/shared/cortex/domain/src/mcp_signal_detectors.rs @@ -0,0 +1,119 @@ +//! Deterministic, phrase-boundary keyword detectors for MCP-incident anchor +//! signals. Pure functions over log message text and `ai_mcp_events` row +//! fields — no DB, no LLM. Mirrors `src/app/skill_signal_detectors.rs`'s +//! phrase-boundary matching style, retargeted at the MCP incident anchor +//! list from GH #94's "MCP assessment design" section: repeated call +//! failures, `is_error`, timeout/rate-limit, schema errors, unknown +//! tool/server, and user correction after tool misuse. + +pub const SIGNAL_REPEATED_CALL_FAILURE: &str = "repeated_call_failure"; +pub const SIGNAL_TIMEOUT_OR_RATE_LIMIT: &str = "timeout_or_rate_limit"; +pub const SIGNAL_AUTH_OR_PERMISSION_FAILURE: &str = "auth_or_permission_failure"; +pub const SIGNAL_SCHEMA_OR_VALIDATION_ERROR: &str = "schema_or_validation_error"; +pub const SIGNAL_UNKNOWN_TOOL_OR_SERVER: &str = "unknown_tool_or_server"; +pub const SIGNAL_USER_CORRECTION_AFTER_TOOL_CALL: &str = "user_correction_after_tool_call"; + +/// All six locked anchor signal categories, in a stable order used for +/// `signals_present` sorting and CLI `--signals` validation. +pub const ALL_SIGNALS: &[&str] = &[ + SIGNAL_REPEATED_CALL_FAILURE, + SIGNAL_TIMEOUT_OR_RATE_LIMIT, + SIGNAL_AUTH_OR_PERMISSION_FAILURE, + SIGNAL_SCHEMA_OR_VALIDATION_ERROR, + SIGNAL_UNKNOWN_TOOL_OR_SERVER, + SIGNAL_USER_CORRECTION_AFTER_TOOL_CALL, +]; + +const TIMEOUT_OR_RATE_LIMIT_PHRASES: &[&str] = &[ + "timed out", + "timeout", + "rate limit", + "rate-limited", + "too many requests", +]; + +const AUTH_OR_PERMISSION_PHRASES: &[&str] = &[ + "permission denied", + "unauthorized", + "forbidden", + "authentication failed", + "auth failed", + "invalid token", + "invalid credentials", +]; + +const SCHEMA_OR_VALIDATION_PHRASES: &[&str] = &[ + "schema validation", + "invalid parameters", + "invalid arguments", + "missing required", + "does not match schema", + "validation error", + "invalidparams", +]; + +const UNKNOWN_TOOL_OR_SERVER_PHRASES: &[&str] = &[ + "unknown tool", + "tool not found", + "server unavailable", + "server not found", + "no such tool", + "not connected", + "disconnected", + "mcp server error", +]; + +/// Reused from the skill-incident correction phrase list style (same +/// conservative multi-word-phrase design) but scoped to tool-call +/// misuse/misinterpretation follow-up. +const USER_CORRECTION_PHRASES: &[&str] = &[ + "that's not what i asked", + "that is not what i asked", + "wrong tool", + "not the right tool", + "you used the wrong", + "that's the wrong", + "that is the wrong", + "no, that's wrong", + "no, that is wrong", +]; + +fn contains_any_phrase(haystack_lower: &str, phrases: &[&str]) -> bool { + phrases.iter().any(|p| haystack_lower.contains(p)) +} + +pub fn detect_timeout_or_rate_limit(message: &str) -> bool { + contains_any_phrase(&message.to_ascii_lowercase(), TIMEOUT_OR_RATE_LIMIT_PHRASES) +} + +pub fn detect_auth_or_permission_failure(message: &str) -> bool { + contains_any_phrase(&message.to_ascii_lowercase(), AUTH_OR_PERMISSION_PHRASES) +} + +pub fn detect_schema_or_validation_error(message: &str) -> bool { + contains_any_phrase(&message.to_ascii_lowercase(), SCHEMA_OR_VALIDATION_PHRASES) +} + +pub fn detect_unknown_tool_or_server(message: &str) -> bool { + contains_any_phrase( + &message.to_ascii_lowercase(), + UNKNOWN_TOOL_OR_SERVER_PHRASES, + ) +} + +pub fn detect_user_correction_after_tool_call(message: &str) -> bool { + contains_any_phrase(&message.to_ascii_lowercase(), USER_CORRECTION_PHRASES) +} + +/// Minimum error-event count for the same `(mcp_server, mcp_tool)` pair +/// within a group that counts as a "repeated call failure" anchor, as +/// opposed to a single one-off error. +pub const REPEATED_FAILURE_THRESHOLD: usize = 2; + +pub fn detect_repeated_call_failure(error_event_count: usize) -> bool { + error_event_count >= REPEATED_FAILURE_THRESHOLD +} + +#[cfg(test)] +#[path = "mcp_signal_detectors_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/domain/src/mcp_signal_detectors_tests.rs b/crates/shared/cortex/domain/src/mcp_signal_detectors_tests.rs new file mode 100644 index 00000000..fb22d0e1 --- /dev/null +++ b/crates/shared/cortex/domain/src/mcp_signal_detectors_tests.rs @@ -0,0 +1,90 @@ +use super::*; + +#[test] +fn detects_timeout_and_rate_limit_phrases() { + assert!(detect_timeout_or_rate_limit("request timed out after 30s")); + assert!(detect_timeout_or_rate_limit( + "hit a rate limit, please retry" + )); + assert!(!detect_timeout_or_rate_limit("everything worked fine")); +} + +#[test] +fn detects_auth_or_permission_failures() { + assert!(detect_auth_or_permission_failure( + "permission denied for tool" + )); + assert!(detect_auth_or_permission_failure( + "Unauthorized: invalid token" + )); + assert!(!detect_auth_or_permission_failure("all good here")); +} + +#[test] +fn detects_schema_or_validation_errors() { + assert!(detect_schema_or_validation_error( + "schema validation failed" + )); + assert!(detect_schema_or_validation_error( + "missing required field 'query'" + )); + assert!(!detect_schema_or_validation_error("success")); +} + +#[test] +fn detects_unknown_tool_or_server() { + assert!(detect_unknown_tool_or_server("unknown tool: mcp__foo__bar")); + assert!(detect_unknown_tool_or_server("server not found")); + assert!(!detect_unknown_tool_or_server("tool executed successfully")); +} + +#[test] +fn detects_user_correction_after_tool_call() { + assert!(detect_user_correction_after_tool_call( + "no, that's the wrong tool" + )); + assert!(detect_user_correction_after_tool_call( + "that's not what I asked for" + )); + assert!(!detect_user_correction_after_tool_call( + "thanks, that's correct" + )); +} + +#[test] +fn user_correction_ignores_generic_wrong_and_bare_no_that_phrases() { + // Eng review fix: the phrase list previously included bare "is wrong" + // and "no, that's"/"no, that is" prefixes, which false-positive on + // ordinary debugging dialogue unrelated to a tool-call correction. + assert!(!detect_user_correction_after_tool_call( + "my assumption is wrong here" + )); + assert!(!detect_user_correction_after_tool_call( + "no, that's actually fine" + )); + assert!(!detect_user_correction_after_tool_call( + "no, that is expected behavior" + )); + assert!(detect_user_correction_after_tool_call( + "no, that's wrong, try again" + )); +} + +#[test] +fn repeated_call_failure_requires_threshold() { + assert!(!detect_repeated_call_failure(0)); + assert!(!detect_repeated_call_failure(1)); + assert!(detect_repeated_call_failure(2)); + assert!(detect_repeated_call_failure(5)); +} + +#[test] +fn all_signals_list_is_stable_and_complete() { + assert_eq!(ALL_SIGNALS.len(), 6); + assert!(ALL_SIGNALS.contains(&SIGNAL_REPEATED_CALL_FAILURE)); + assert!(ALL_SIGNALS.contains(&SIGNAL_TIMEOUT_OR_RATE_LIMIT)); + assert!(ALL_SIGNALS.contains(&SIGNAL_AUTH_OR_PERMISSION_FAILURE)); + assert!(ALL_SIGNALS.contains(&SIGNAL_SCHEMA_OR_VALIDATION_ERROR)); + assert!(ALL_SIGNALS.contains(&SIGNAL_UNKNOWN_TOOL_OR_SERVER)); + assert!(ALL_SIGNALS.contains(&SIGNAL_USER_CORRECTION_AFTER_TOOL_CALL)); +} diff --git a/crates/shared/cortex/domain/src/observatory_identity.rs b/crates/shared/cortex/domain/src/observatory_identity.rs new file mode 100644 index 00000000..b39b3a1f --- /dev/null +++ b/crates/shared/cortex/domain/src/observatory_identity.rs @@ -0,0 +1,162 @@ +//! Stable, versioned Agent Observatory identities. + +use std::fmt; + +/// Maximum encoded projected-event key size from the Agent Observatory contract. +pub const MAX_EVENT_KEY_BYTES: usize = 1024; + +/// Validation failures produced while constructing durable identities. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum IdentityError { + /// A required component was empty after trimming. + Empty(&'static str), + /// A source kind or projection variant was not strict ASCII lower snake case. + InvalidLowerSnake(&'static str), + /// A projected-event key exceeded the durable contract limit. + EventKeyTooLong { + /// Encoded UTF-8 byte length. + actual: usize, + /// Maximum permitted UTF-8 byte length. + max: usize, + }, +} + +impl fmt::Display for IdentityError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + Self::Empty(component) => write!(formatter, "{component} must be non-empty"), + Self::InvalidLowerSnake(component) => { + write!(formatter, "{component} must be ASCII lower snake case") + } + Self::EventKeyTooLong { actual, max } => { + write!(formatter, "event key is {actual} bytes; maximum is {max}") + } + } + } +} + +impl std::error::Error for IdentityError {} + +fn required<'a>(value: &'a str, component: &'static str) -> Result<&'a str, IdentityError> { + let value = value.trim(); + if value.is_empty() { + Err(IdentityError::Empty(component)) + } else { + Ok(value) + } +} + +fn length_prefixed(parts: &[&str]) -> String { + parts + .iter() + .map(|part| format!("{}:{part}", part.len())) + .collect::>() + .join("|") +} + +/// Convert a provider tool label into its stable contract value. +/// +/// Known tools become lowercase names. Unknown labels remain Unicode-preserving +/// apart from Unicode lowercase conversion and surrounding-whitespace trimming. +pub fn canonical_tool(tool: &str) -> Result { + let tool = required(tool, "tool")?; + let normalized = tool.to_lowercase(); + + match normalized.as_str() { + "claude" | "codex" | "gemini" => Ok(normalized), + _ => { + let source = normalized + .strip_prefix("unknown:") + .map(str::trim) + .unwrap_or(normalized.as_str()); + if source.is_empty() { + return Err(IdentityError::Empty("tool")); + } + Ok(format!("unknown:{source}")) + } + } +} + +/// Build the version-one run identity from host, canonical tool, and native session. +pub fn run_key(host: &str, tool: &str, session: &str) -> Result { + let host = required(host, "host")?; + let tool = canonical_tool(tool)?; + let session = required(session, "session")?; + Ok(format!( + "v1|{}", + length_prefixed(&[host, tool.as_str(), session]) + )) +} + +/// Build the version-one repository identity from host and canonical common Git directory. +pub fn repository_key(host: &str, common_git_dir: &str) -> Result { + let host = required(host, "host")?; + let common_git_dir = required(common_git_dir, "common_git_dir")?; + Ok(format!("v1|{}", length_prefixed(&[host, common_git_dir]))) +} + +/// Build the version-one worktree identity from host and canonical worktree path. +pub fn worktree_key(host: &str, worktree_path: &str) -> Result { + let host = required(host, "host")?; + let worktree_path = required(worktree_path, "worktree_path")?; + Ok(format!("v1|{}", length_prefixed(&[host, worktree_path]))) +} + +/// Build the version-one actor identity nested under a complete run key. +pub fn actor_key(run_key: &str, actor_id: &str) -> Result { + let run_key = required(run_key, "run_key")?; + let actor_id = required(actor_id, "actor_id")?; + Ok(format!("v1|{}", length_prefixed(&[run_key, actor_id]))) +} + +fn is_ascii_lower_snake(value: &str) -> bool { + let bytes = value.as_bytes(); + if bytes.is_empty() || !bytes[0].is_ascii_lowercase() || bytes.last() == Some(&b'_') { + return false; + } + + let mut previous_was_underscore = false; + for byte in bytes { + match byte { + b'a'..=b'z' | b'0'..=b'9' => previous_was_underscore = false, + b'_' if !previous_was_underscore => previous_was_underscore = true, + _ => return false, + } + } + true +} + +/// Build a deterministic projected-event key. +/// +/// Source kinds and projection variants are strict ASCII lower snake case. The +/// source primary key is trimmed but otherwise preserved, including delimiters. +pub fn event_key( + source_kind: &str, + source_primary_key: &str, + projection_variant: &str, +) -> Result { + let source_kind = required(source_kind, "source_kind")?; + let source_primary_key = required(source_primary_key, "source_primary_key")?; + let projection_variant = required(projection_variant, "projection_variant")?; + + if !is_ascii_lower_snake(source_kind) { + return Err(IdentityError::InvalidLowerSnake("source_kind")); + } + if !is_ascii_lower_snake(projection_variant) { + return Err(IdentityError::InvalidLowerSnake("projection_variant")); + } + + let key = format!("v1:{source_kind}:{source_primary_key}:{projection_variant}"); + let actual = key.len(); + if actual > MAX_EVENT_KEY_BYTES { + return Err(IdentityError::EventKeyTooLong { + actual, + max: MAX_EVENT_KEY_BYTES, + }); + } + Ok(key) +} + +#[cfg(test)] +#[path = "observatory_identity_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/domain/src/observatory_identity_tests.rs b/crates/shared/cortex/domain/src/observatory_identity_tests.rs new file mode 100644 index 00000000..89ac85f4 --- /dev/null +++ b/crates/shared/cortex/domain/src/observatory_identity_tests.rs @@ -0,0 +1,164 @@ +use super::{ + IdentityError, MAX_EVENT_KEY_BYTES, actor_key, canonical_tool, event_key, repository_key, + run_key, worktree_key, +}; + +#[test] +fn canonical_tool_normalizes_known_unknown_and_explicit_unknown_values() { + assert_eq!(canonical_tool(" Claude ").unwrap(), "claude"); + assert_eq!(canonical_tool("CoDeX").unwrap(), "codex"); + assert_eq!(canonical_tool("GEMINI").unwrap(), "gemini"); + assert_eq!( + canonical_tool(" OpenAI DevTools ").unwrap(), + "unknown:openai devtools" + ); + assert_eq!( + canonical_tool(" UNKNOWN:\u{0394}\u{0395}\u{039b}\u{03a4}\u{0391} ").unwrap(), + "unknown:\u{03b4}\u{03b5}\u{03bb}\u{03c4}\u{03b1}" + ); +} + +#[test] +fn run_key_matches_contract_and_trims_identity_components() { + assert_eq!( + run_key( + " devhost ", + " Claude ", + " 00112233-4455-6677-8899-aabbccddeeff " + ) + .unwrap(), + "v1|7:devhost|6:claude|36:00112233-4455-6677-8899-aabbccddeeff" + ); +} + +#[test] +fn unicode_lengths_are_counted_in_utf8_bytes() { + assert_eq!( + run_key("\u{732b}", "Claude", "\u{4f1a}\u{8a71}").unwrap(), + "v1|3:\u{732b}|6:claude|6:\u{4f1a}\u{8a71}" + ); + assert_eq!( + repository_key("\u{732b}", "/srv/\u{4f1a}\u{8a71}/.git").unwrap(), + "v1|3:\u{732b}|16:/srv/\u{4f1a}\u{8a71}/.git" + ); +} + +#[test] +fn length_prefixes_make_delimiter_characters_unambiguous() { + assert_eq!( + repository_key("do|ok:ie", "/srv/repo|main:.git").unwrap(), + "v1|8:do|ok:ie|19:/srv/repo|main:.git" + ); + assert_eq!( + worktree_key("do|ok:ie", "/srv/repo|main:wt").unwrap(), + "v1|8:do|ok:ie|17:/srv/repo|main:wt" + ); +} + +#[test] +fn actor_key_nests_the_complete_run_key_without_ambiguity() { + let run = run_key("devhost", "Claude", "session-1").unwrap(); + assert_eq!( + actor_key(&run, "agent:1").unwrap(), + format!("v1|{}:{run}|7:agent:1", run.len()) + ); +} + +#[test] +fn event_key_is_deterministic_and_allows_delimiters_in_source_identity() { + assert_eq!( + event_key("otel_spans", "trace:span|1", "primary").unwrap(), + "v1:otel_spans:trace:span|1:primary" + ); + assert_eq!( + event_key("repository_observations", "42", "git_head").unwrap(), + "v1:repository_observations:42:git_head" + ); +} + +#[test] +fn event_key_rejects_non_lower_snake_components() { + for invalid in ["OTEL_SPANS", "otel-spans", "_otel", "otel_", "otel__spans"] { + assert_eq!( + event_key(invalid, "1", "primary"), + Err(IdentityError::InvalidLowerSnake("source_kind")) + ); + } + assert_eq!( + event_key("logs", "1", "Primary"), + Err(IdentityError::InvalidLowerSnake("projection_variant")) + ); +} + +#[test] +fn event_key_enforces_1024_byte_contract_limit() { + let exact = "x".repeat(MAX_EVENT_KEY_BYTES - "v1:logs::primary".len()); + let key = event_key("logs", &exact, "primary").unwrap(); + assert_eq!(key.len(), MAX_EVENT_KEY_BYTES); + + let oversized = format!("{exact}x"); + assert_eq!( + event_key("logs", &oversized, "primary"), + Err(IdentityError::EventKeyTooLong { + actual: MAX_EVENT_KEY_BYTES + 1, + max: MAX_EVENT_KEY_BYTES, + }) + ); +} + +#[test] +fn every_identity_rejects_empty_trimmed_components() { + assert_eq!(canonical_tool(" "), Err(IdentityError::Empty("tool"))); + assert_eq!( + run_key("", "claude", "session"), + Err(IdentityError::Empty("host")) + ); + assert_eq!( + run_key("host", "", "session"), + Err(IdentityError::Empty("tool")) + ); + assert_eq!( + run_key("host", "claude", ""), + Err(IdentityError::Empty("session")) + ); + assert_eq!( + repository_key("host", " "), + Err(IdentityError::Empty("common_git_dir")) + ); + assert_eq!( + worktree_key("host", " "), + Err(IdentityError::Empty("worktree_path")) + ); + assert_eq!( + actor_key(" ", "actor"), + Err(IdentityError::Empty("run_key")) + ); + assert_eq!( + actor_key("run", " "), + Err(IdentityError::Empty("actor_id")) + ); + assert_eq!( + event_key("logs", " ", "primary"), + Err(IdentityError::Empty("source_primary_key")) + ); +} + +#[test] +fn property_style_vectors_are_stable_across_repeated_calls() { + let vectors = [ + ("devhost", "claude", "abc"), + (" host:one ", "Custom Tool", "session|two"), + ( + "\u{03b4}\u{03bf}\u{03ba}\u{03b9}\u{03bc}\u{03ae}", + "Gemini", + "\u{4f1a}\u{8a71}:\u{4e09}", + ), + ]; + + for (host, tool, session) in vectors { + let first = run_key(host, tool, session).unwrap(); + for _ in 0..32 { + assert_eq!(run_key(host, tool, session).unwrap(), first); + } + } +} diff --git a/crates/shared/cortex/domain/src/skill_incident_findings.rs b/crates/shared/cortex/domain/src/skill_incident_findings.rs new file mode 100644 index 00000000..fa0a3c31 --- /dev/null +++ b/crates/shared/cortex/domain/src/skill_incident_findings.rs @@ -0,0 +1,267 @@ +//! Deterministic failure-hypothesis and prevention-hint generation over +//! skill-usage incident evidence bundles. Pure rule evaluation -- never +//! queries the database and never calls an external LLM. Mirrors +//! `src/app/incident_findings.rs` (the abuse-incident findings module) but +//! targets skill-specific failure categories. + +use serde::{Deserialize, Serialize}; + +use crate::{LogEntry, SkillIncident}; + +// -- Stable failure-mode categories ------------------------------------------ +pub const SKILL_SCOPE_MISMATCH: &str = "skill_scope_mismatch"; +pub const MISSING_PREREQUISITE_CHECK: &str = "missing_prerequisite_check"; +pub const WRONG_SOURCE_OF_TRUTH: &str = "wrong_source_of_truth"; +pub const OVERLY_BROAD_RESEARCH_LOOP: &str = "overly_broad_research_loop"; +pub const TOOL_POLICY_MISMATCH: &str = "tool_policy_mismatch"; +pub const MISSING_VERIFICATION_STEP: &str = "missing_verification_step"; +pub const AMBIGUOUS_SKILL_TRIGGER: &str = "ambiguous_skill_trigger"; +pub const STALE_OR_CONFLICTING_SKILL_INSTRUCTION: &str = "stale_or_conflicting_skill_instruction"; +pub const ASSISTANT_OVEREXPLAINED_SIMPLE_ANSWER: &str = "assistant_overexplained_simple_answer"; +pub const UNKNOWN: &str = "unknown"; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SkillFailureMode { + pub category: String, + pub confidence: String, + pub evidence_ids: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SkillContributingFactor { + pub factor: String, + pub evidence_ids: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct SkillPreventionHint { + pub category: String, + pub hint: String, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize, PartialEq, Eq)] +pub struct SkillIncidentFindings { + pub likely_failure_modes: Vec, + pub contributing_factors: Vec, + pub prevention_hints: Vec, + pub open_questions: Vec, +} + +/// `(category, keyword substrings, prevention hint)`. Kept specific -- no +/// broad single tokens. +type Rule = (&'static str, &'static [&'static str], &'static str); + +const RULES: &[Rule] = &[ + ( + WRONG_SOURCE_OF_TRUTH, + &[ + "wrong source of truth", + "wrong source", + "stale data", + "memory-vs-live", + "memory vs live", + "not the live", + ], + "Add a note to the skill doc naming the canonical source of truth for this data \ + (live system vs. memory/cache) and require the agent to confirm which one it used.", + ), + ( + WRONG_SOURCE_OF_TRUTH, + &["wrong repo"], + "Add a trigger-boundary note clarifying which repo this skill applies to, and require \ + the agent to confirm the working directory before acting.", + ), + ( + MISSING_VERIFICATION_STEP, + &[ + "without any verification", + "without verification", + "claimed success without", + ], + "Add a verification checklist item requiring live repo/runtime evidence before claiming \ + success.", + ), + ( + MISSING_PREREQUISITE_CHECK, + &["should have created a bead", "should have created an issue"], + "Add a prerequisite-check step to the skill doc: confirm an issue/bead exists (or create \ + one) before starting non-trivial work.", + ), + ( + TOOL_POLICY_MISMATCH, + &[ + "wrong transport", + "wrong source for this call", + "raw web instead of using axon", + "instead of using axon", + ], + "Add an explicit tool-policy line to the skill doc naming the required transport/source \ + (e.g. Axon before raw web search) and why.", + ), + ( + OVERLY_BROAD_RESEARCH_LOOP, + &["going in circles", "we wasted", "all you had to say"], + "Add an anti-loop rule: after two failed searches, summarize current evidence and switch \ + strategy instead of repeating the same approach.", + ), + ( + AMBIGUOUS_SKILL_TRIGGER, + &[ + "wrong skill", + "not the right skill", + "shouldn't have triggered", + "should not have triggered", + ], + "Add a trigger-boundary note that this skill is for implementation planning only (or \ + narrow its stated trigger phrases) so it stops firing on out-of-scope requests.", + ), + ( + STALE_OR_CONFLICTING_SKILL_INSTRUCTION, + &[ + "stale instruction", + "conflicting instruction", + "outdated skill", + "skill doc is wrong", + "skill doc is out of date", + ], + "Review and update the skill doc section that conflicts with current project conventions.", + ), + ( + ASSISTANT_OVEREXPLAINED_SIMPLE_ANSWER, + &[ + "all you had to say was", + "didn't need to touch", + "did not need to touch", + "you didn't need to", + ], + "Add a conciseness note to the skill doc: for simple factual questions, answer directly \ + before taking any action.", + ), + ( + SKILL_SCOPE_MISMATCH, + &["out of scope", "not what this skill is for"], + "Narrow the skill's stated scope in its description/trigger phrases to exclude this case.", + ), +]; + +fn confidence_for(count: usize) -> &'static str { + match count { + 0 | 1 => "low", + 2 => "medium", + _ => "high", + } +} + +const SKILL_EVENT_FACTOR_THRESHOLD: usize = 3; +const ERROR_BURST_THRESHOLD: usize = 3; + +fn scannable<'a>( + signal_anchors: &'a [LogEntry], + transcript_before: &'a [LogEntry], + transcript_after: &'a [LogEntry], + nearby_logs: &'a [LogEntry], + nearby_errors: &'a [LogEntry], +) -> impl Iterator { + signal_anchors + .iter() + .chain(transcript_before) + .chain(transcript_after) + .chain(nearby_logs) + .chain(nearby_errors) +} + +/// Derive deterministic findings from a skill-incident evidence bundle. Pure +/// and total: identical input always yields identical output; every +/// non-`unknown` failure mode / contributing factor cites at least one +/// evidence id; weak evidence yields `unknown` + `open_questions` rather than +/// an unsupported claim. +pub fn derive_skill_incident_findings( + incident: &SkillIncident, + _skill_events: &[crate::SkillEventEntry], + signal_anchors: &[LogEntry], + transcript_before: &[LogEntry], + transcript_after: &[LogEntry], + nearby_logs: &[LogEntry], + nearby_errors: &[LogEntry], +) -> SkillIncidentFindings { + let mut findings = SkillIncidentFindings::default(); + + for (category, keywords, hint) in RULES { + let mut ids: Vec = Vec::new(); + for entry in scannable( + signal_anchors, + transcript_before, + transcript_after, + nearby_logs, + nearby_errors, + ) { + let haystack = entry.message.to_ascii_lowercase(); + if keywords.iter().any(|kw| haystack.contains(kw)) { + ids.push(entry.id); + } + } + ids.sort_unstable(); + ids.dedup(); + if !ids.is_empty() { + let confidence = confidence_for(ids.len()).to_owned(); + findings.likely_failure_modes.push(SkillFailureMode { + category: (*category).to_owned(), + confidence, + evidence_ids: ids, + }); + findings.prevention_hints.push(SkillPreventionHint { + category: (*category).to_owned(), + hint: (*hint).to_owned(), + }); + } + } + + if incident.skill_event_count >= SKILL_EVENT_FACTOR_THRESHOLD && !signal_anchors.is_empty() { + findings.contributing_factors.push(SkillContributingFactor { + factor: format!( + "Repeated skill invocation: {} skill events within the incident window.", + incident.skill_event_count + ), + evidence_ids: signal_anchors.iter().map(|a| a.id).collect(), + }); + } + if nearby_errors.len() >= ERROR_BURST_THRESHOLD { + findings.contributing_factors.push(SkillContributingFactor { + factor: format!( + "Error burst: {} error-level logs in the correlation window.", + nearby_errors.len() + ), + evidence_ids: nearby_errors.iter().map(|e| e.id).collect(), + }); + } + + if findings.likely_failure_modes.is_empty() { + findings.likely_failure_modes.push(SkillFailureMode { + category: UNKNOWN.to_owned(), + confidence: "low".to_owned(), + evidence_ids: Vec::new(), + }); + findings.open_questions.push( + "No deterministic failure signature matched the evidence window; manual transcript \ + review is recommended." + .to_owned(), + ); + } + if signal_anchors.is_empty() { + findings + .open_questions + .push("No negative signal anchors were captured for this incident.".to_owned()); + } + if nearby_logs.is_empty() && nearby_errors.is_empty() { + findings.open_questions.push( + "No surrounding non-AI logs were available to corroborate the transcript signal." + .to_owned(), + ); + } + + findings +} + +#[cfg(test)] +#[path = "skill_incident_findings_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/domain/src/skill_incident_findings_tests.rs b/crates/shared/cortex/domain/src/skill_incident_findings_tests.rs new file mode 100644 index 00000000..32957b2d --- /dev/null +++ b/crates/shared/cortex/domain/src/skill_incident_findings_tests.rs @@ -0,0 +1,181 @@ +use super::*; +use crate::{LogEntry, SkillIncident, SkillSignalCounts}; + +fn log(id: i64, message: &str) -> LogEntry { + LogEntry { + id, + timestamp: "2026-01-01T00:00:00Z".to_string(), + hostname: "devhost".to_string(), + facility: None, + severity: "info".to_string(), + app_name: Some("ai-transcript".to_string()), + process_id: None, + message: message.to_string(), + received_at: "2026-01-01T00:00:00Z".to_string(), + source_ip: "127.0.0.1:0".to_string(), + ai_tool: Some("codex".to_string()), + ai_project: Some("/tmp/project".to_string()), + ai_session_id: Some("sess-1".to_string()), + ai_transcript_path: None, + metadata_json: None, + } +} + +fn incident(signals_present: Vec<&str>) -> SkillIncident { + SkillIncident { + incident_id: "skill-inc-test".to_string(), + skill_name: "lavra:lavra-plan".to_string(), + skill_plugin: Some("lavra".to_string()), + tool: "codex".to_string(), + project: "/tmp/project".to_string(), + session_id: "sess-1".to_string(), + hostname: "devhost".to_string(), + first_seen: "2026-01-01T00:00:00Z".to_string(), + last_seen: "2026-01-01T00:05:00Z".to_string(), + duration_secs: 300, + skill_event_count: 1, + skill_event_ids: vec![1], + anchor_log_ids: vec![2], + signal_counts: SkillSignalCounts::default(), + signals_present: signals_present.into_iter().map(String::from).collect(), + priority_score: 22.0, + priority_label: "medium".to_string(), + window_minutes: 10, + } +} + +#[test] +fn detects_wrong_source_of_truth_category() { + let inc = incident(vec!["scope_or_source_confusion"]); + let anchors = vec![log( + 2, + "you're using the wrong source of truth here, check the live container", + )]; + let findings = derive_skill_incident_findings(&inc, &[], &anchors, &[], &[], &[], &[]); + assert!( + findings + .likely_failure_modes + .iter() + .any(|f| f.category == WRONG_SOURCE_OF_TRUTH), + "expected wrong_source_of_truth category, got {:?}", + findings.likely_failure_modes + ); + let mode = findings + .likely_failure_modes + .iter() + .find(|f| f.category == WRONG_SOURCE_OF_TRUTH) + .unwrap(); + assert_eq!(mode.evidence_ids, vec![2]); + assert!( + findings + .prevention_hints + .iter() + .any(|h| h.category == WRONG_SOURCE_OF_TRUTH + && h.hint.to_ascii_lowercase().contains("source of truth")) + ); +} + +#[test] +fn detects_missing_verification_step_category() { + let inc = incident(vec!["ignored_skill_or_policy_instruction"]); + let anchors = vec![log( + 2, + "you claimed success without any verification of the running app", + )]; + let findings = derive_skill_incident_findings(&inc, &[], &anchors, &[], &[], &[], &[]); + assert!( + findings + .likely_failure_modes + .iter() + .any(|f| f.category == MISSING_VERIFICATION_STEP) + ); + let hint = findings + .prevention_hints + .iter() + .find(|h| h.category == MISSING_VERIFICATION_STEP) + .unwrap(); + assert!(hint.hint.to_ascii_lowercase().contains("verification")); +} + +#[test] +fn detects_overly_broad_research_loop_category() { + let inc = incident(vec!["overlong_loop_after_skill"]); + let counts = SkillSignalCounts { + overlong_loop_after_skill: 1, + ..Default::default() + }; + let mut inc2 = inc; + inc2.signal_counts = counts; + let anchors = vec![log( + 2, + "that's not what I asked, we wasted twenty minutes going in circles", + )]; + let findings = derive_skill_incident_findings(&inc2, &[], &anchors, &[], &[], &[], &[]); + assert!( + findings + .likely_failure_modes + .iter() + .any(|f| f.category == OVERLY_BROAD_RESEARCH_LOOP) + ); +} + +#[test] +fn detects_ambiguous_skill_trigger_category() { + let inc = incident(vec!["skill_scope_mismatch"]); + let anchors = vec![log( + 2, + "wrong skill triggered, this wasn't the right one for the task", + )]; + let findings = derive_skill_incident_findings(&inc, &[], &anchors, &[], &[], &[], &[]); + assert!( + findings + .likely_failure_modes + .iter() + .any(|f| f.category == AMBIGUOUS_SKILL_TRIGGER || f.category == SKILL_SCOPE_MISMATCH), + "expected ambiguous_skill_trigger or skill_scope_mismatch, got {:?}", + findings.likely_failure_modes + ); +} + +#[test] +fn weak_evidence_falls_back_to_unknown_with_open_questions() { + let inc = incident(vec![]); + let findings = derive_skill_incident_findings(&inc, &[], &[], &[], &[], &[], &[]); + assert!( + findings + .likely_failure_modes + .iter() + .any(|f| f.category == UNKNOWN) + ); + assert!(!findings.open_questions.is_empty()); +} + +#[test] +fn every_finding_cites_evidence_ids_when_not_unknown() { + let inc = incident(vec!["tool_failure_after_skill"]); + let anchors = vec![log(2, "command exited with exit code 1")]; + let findings = derive_skill_incident_findings(&inc, &[], &anchors, &[], &[], &[], &[]); + for mode in &findings.likely_failure_modes { + if mode.category != UNKNOWN { + assert!( + !mode.evidence_ids.is_empty(), + "category {} has no evidence ids", + mode.category + ); + } + } +} + +#[test] +fn prevention_hints_are_skill_doc_actionable() { + let inc = incident(vec!["scope_or_source_confusion"]); + let anchors = vec![log(2, "wrong repo, this is stale data not the live system")]; + let findings = derive_skill_incident_findings(&inc, &[], &anchors, &[], &[], &[], &[]); + for hint in &findings.prevention_hints { + assert!( + hint.hint.len() > 20, + "hint should be a concrete actionable sentence: {}", + hint.hint + ); + } +} diff --git a/crates/shared/cortex/domain/src/skill_signal_detectors.rs b/crates/shared/cortex/domain/src/skill_signal_detectors.rs new file mode 100644 index 00000000..55cd2b05 --- /dev/null +++ b/crates/shared/cortex/domain/src/skill_signal_detectors.rs @@ -0,0 +1,139 @@ +//! Deterministic, phrase-boundary keyword detectors for skill-incident anchor +//! signals. Pure functions over log message text — no DB, no LLM. Mirrors the +//! word/phrase-boundary matching style used by Cortex incident analysis. The +//! detectors are domain policy shared by persistence grouping and evidence +//! classification, with no database or transport dependency. + +pub const SIGNAL_USER_CORRECTION_AFTER_SKILL: &str = "user_correction_after_skill"; +pub const SIGNAL_TOOL_FAILURE_AFTER_SKILL: &str = "tool_failure_after_skill"; +pub const SIGNAL_SCOPE_OR_SOURCE_CONFUSION: &str = "scope_or_source_confusion"; +pub const SIGNAL_IGNORED_SKILL_OR_POLICY_INSTRUCTION: &str = "ignored_skill_or_policy_instruction"; +pub const SIGNAL_OVERLONG_LOOP_AFTER_SKILL: &str = "overlong_loop_after_skill"; + +/// All five locked anchor signal categories, in a stable order used for +/// `signals_present` sorting and CLI `--signals` validation. +pub const ALL_SIGNALS: &[&str] = &[ + SIGNAL_USER_CORRECTION_AFTER_SKILL, + SIGNAL_TOOL_FAILURE_AFTER_SKILL, + SIGNAL_SCOPE_OR_SOURCE_CONFUSION, + SIGNAL_IGNORED_SKILL_OR_POLICY_INSTRUCTION, + SIGNAL_OVERLONG_LOOP_AFTER_SKILL, +]; + +/// Phrases indicating the user is correcting or pushing back on the assistant +/// immediately after a skill loaded. Deliberately phrase-level (not single +/// words like "no" or "wrong" in isolation) to avoid false positives on +/// unrelated negatives ("no new errors were found"). +const USER_CORRECTION_PHRASES: &[&str] = &[ + "that's not what i asked", + "that is not what i asked", + "you said you would", + "but you didn't", + "but you did not", + "is wrong", + "is just wrong", + "no, that's", + "no, that is", + "we wasted", + "all you had to say", + "stop, you", + "you didn't need to", + "you did not need to", +]; + +/// Phrases indicating a tool/command failure surfaced in nearby transcript or +/// tool-output text. +const TOOL_FAILURE_PHRASES: &[&str] = &[ + "exit code", + "permission denied", + "not found", + "timed out", + "failed to", + "database is locked", + "rate limit", +]; + +/// Case-insensitive substring match on whole phrases (already multi-word, so +/// word-boundary checks are unnecessary for most entries — a phrase like +/// "you said" cannot ride inside an unrelated longer word the way a bare +/// single-word term like "hell" could). +fn contains_any_phrase(haystack_lower: &str, phrases: &[&str]) -> bool { + phrases.iter().any(|p| haystack_lower.contains(p)) +} + +pub fn detect_user_correction(message: &str) -> bool { + let lower = message.to_ascii_lowercase(); + contains_any_phrase(&lower, USER_CORRECTION_PHRASES) +} + +pub fn detect_tool_failure(message: &str) -> bool { + let lower = message.to_ascii_lowercase(); + contains_any_phrase(&lower, TOOL_FAILURE_PHRASES) +} + +/// Conservative phrases indicating the assistant is operating on the wrong +/// repo, stale data, wrong source of truth, or confusing an in-memory/cached +/// view with the live system. Starts from the same conservative style as +/// `UNCLEAR_INSTRUCTION_OR_SCOPE_DRIFT` in `src/app/incident_findings.rs` +/// plus skill-specific additions for source-of-truth confusion. +const SCOPE_OR_SOURCE_CONFUSION_PHRASES: &[&str] = &[ + "wrong repo", + "wrong file", + "stale data", + "wrong source", + "source of truth", + "memory-vs-live", + "memory vs live", + "not the live", + "going in circles", +]; + +/// Explicit-violation phrases for a fixed set of known instruction +/// categories: no verification after implementation, no issue/bead when +/// required, wrong transport/source, raw web when Axon/Labby required, +/// stopped at plan when asked to implement. Deliberately requires explicit +/// phrase evidence — no broad single-word matches. +const IGNORED_INSTRUCTION_PHRASES: &[&str] = &[ + "without any verification", + "without verification", + "claimed success without", + "should have created a bead", + "should have created an issue", + "wrong transport", + "wrong source for this call", + "raw web instead of using axon", + "instead of using axon", + "stopped at the plan", + "stopped at plan", +]; + +pub fn detect_scope_or_source_confusion(message: &str) -> bool { + let lower = message.to_ascii_lowercase(); + contains_any_phrase(&lower, SCOPE_OR_SOURCE_CONFUSION_PHRASES) +} + +pub fn detect_ignored_instruction(message: &str) -> bool { + let lower = message.to_ascii_lowercase(); + contains_any_phrase(&lower, IGNORED_INSTRUCTION_PHRASES) +} + +/// Minimum tool-call volume (rows between the skill event and resolution) +/// that counts as "many" for the overlong-loop signal. +const OVERLONG_LOOP_TOOL_CALL_THRESHOLD: usize = 15; + +/// `overlong_loop_after_skill` requires BOTH a high tool-call volume after +/// the skill event AND a co-occurring negative signal (user correction or +/// frustration). Long-but-successful work alone must never trigger this — +/// callers pass `has_correction_or_frustration_signal` computed from the +/// other four detectors / the abuse-term matcher over the same window. +pub fn detect_overlong_loop( + _skill_event_count: usize, + tool_call_count: usize, + has_correction_or_frustration_signal: bool, +) -> bool { + tool_call_count >= OVERLONG_LOOP_TOOL_CALL_THRESHOLD && has_correction_or_frustration_signal +} + +#[cfg(test)] +#[path = "skill_signal_detectors_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/domain/src/skill_signal_detectors_tests.rs b/crates/shared/cortex/domain/src/skill_signal_detectors_tests.rs new file mode 100644 index 00000000..a9ce1d4f --- /dev/null +++ b/crates/shared/cortex/domain/src/skill_signal_detectors_tests.rs @@ -0,0 +1,144 @@ +use super::*; + +#[test] +fn detects_direct_correction_phrases() { + let positives = [ + "That's not what I asked for, please redo it.", + "You said you would run the tests but you didn't.", + "This is just wrong, revert it.", + "No, that's the wrong file entirely.", + "We wasted twenty minutes on this dead end.", + "All you had to say was 'I don't know'.", + "Stop, you're going in circles.", + "You didn't need to touch that config at all.", + ]; + for msg in positives { + assert!( + detect_user_correction(msg), + "expected correction hit for: {msg}" + ); + } +} + +#[test] +fn does_not_flag_unrelated_negatives() { + let negatives = [ + "No new errors were found in the log scan.", + "The stop hook fired successfully.", + "You said the deploy finished — confirming that now.", + "wrongdoing was not detected in the audit", + ]; + for msg in negatives { + assert!( + !detect_user_correction(msg), + "unexpected correction hit for: {msg}" + ); + } +} + +#[test] +fn detects_tool_failure_phrases() { + let positives = [ + "Command exited with exit code 1", + "bash: permission denied", + "error: file not found", + "operation timed out after 30s", + "failed to connect to database", + "database is locked", + "429 rate limit exceeded", + ]; + for msg in positives { + assert!( + detect_tool_failure(msg), + "expected tool-failure hit for: {msg}" + ); + } +} + +#[test] +fn does_not_flag_successful_tool_output_as_failure() { + let negatives = [ + "build completed successfully in 4.2s", + "all 42 tests passed", + "pushed to origin/main", + ]; + for msg in negatives { + assert!( + !detect_tool_failure(msg), + "unexpected tool-failure hit for: {msg}" + ); + } +} + +#[test] +fn detects_scope_or_source_confusion_phrases() { + let positives = [ + "wait, this is the wrong repo entirely", + "that data is stale, we're looking at memory not the live system", + "you're using the wrong source of truth here", + "this is memory-vs-live confusion, check the running container", + ]; + for msg in positives { + assert!( + detect_scope_or_source_confusion(msg), + "expected scope/source confusion hit for: {msg}" + ); + } +} + +#[test] +fn does_not_flag_unrelated_text_as_scope_confusion() { + let negatives = [ + "the repo was cloned successfully", + "source code review complete", + "memory usage is within limits", + ]; + for msg in negatives { + assert!( + !detect_scope_or_source_confusion(msg), + "unexpected scope/source confusion hit for: {msg}" + ); + } +} + +#[test] +fn detects_ignored_instruction_phrases() { + let positives = [ + "you claimed success without any verification", + "you should have created a bead for this but didn't", + "you used the wrong transport for this call", + "you searched the raw web instead of using axon", + "you stopped at the plan instead of implementing it", + ]; + for msg in positives { + assert!( + detect_ignored_instruction(msg), + "expected ignored-instruction hit for: {msg}" + ); + } +} + +#[test] +fn does_not_flag_compliant_text_as_ignored_instruction() { + let negatives = [ + "verification passed, all tests green", + "created bead cortex-142 for the follow-up", + "used axon to research this before answering", + ]; + for msg in negatives { + assert!( + !detect_ignored_instruction(msg), + "unexpected ignored-instruction hit for: {msg}" + ); + } +} + +#[test] +fn overlong_loop_requires_both_volume_and_negative_signal() { + // Long-but-successful: many tool calls, no correction/frustration — must NOT trigger. + assert!(!detect_overlong_loop(3, 40, false)); + // Short loop with a correction — not "overlong", must NOT trigger. + assert!(!detect_overlong_loop(1, 3, true)); + // Long loop WITH a correction/frustration signal — must trigger. + assert!(detect_overlong_loop(2, 25, true)); +} diff --git a/crates/shared/cortex/domain/src/topology.rs b/crates/shared/cortex/domain/src/topology.rs new file mode 100644 index 00000000..fbcea6a2 --- /dev/null +++ b/crates/shared/cortex/domain/src/topology.rs @@ -0,0 +1,80 @@ +use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +pub mod topology_findings { + pub const TYPE_POTENTIAL_PUBLIC_ROUTE: &str = "potential_public_route"; + pub const TYPE_RISKY_MOUNTS: &str = "risky_mounts"; + pub const TYPE_COLLECTOR_HEALTH: &str = "collector_health"; + pub const TYPES: [&str; 3] = [ + TYPE_POTENTIAL_PUBLIC_ROUTE, + TYPE_RISKY_MOUNTS, + TYPE_COLLECTOR_HEALTH, + ]; + + pub const SEVERITY_CRITICAL: &str = "critical"; + pub const SEVERITY_HIGH: &str = "high"; + pub const SEVERITY_MEDIUM: &str = "medium"; + pub const SEVERITY_LOW: &str = "low"; + pub const SEVERITY_INFO: &str = "info"; + + pub mod reason { + pub const REVERSE_PROXY_ROUTE_CONFIGURED: &str = "reverse_proxy_route_configured"; + pub const REVERSE_PROXY_DOMAIN_WITHOUT_TARGET_PROOF: &str = + "reverse_proxy_domain_without_target_proof"; + pub const DOCKER_SOCKET_MOUNT: &str = "docker_socket_mount"; + pub const HOST_ROOT_MOUNT: &str = "host_root_mount"; + pub const APPDATA_ROOT_MOUNT: &str = "appdata_root_mount"; + pub const MOUNT_MISSING_SOURCE_DETAIL: &str = "mount_missing_source_detail"; + pub const GRAPH_PROJECTION_NOT_READY: &str = "graph_projection_not_ready"; + pub const INVENTORY_CACHE_MISSING: &str = "inventory_cache_missing"; + pub const INVENTORY_CACHE_STALE: &str = "inventory_cache_stale"; + pub const INVENTORY_CACHE_UNREADABLE: &str = "inventory_cache_unreadable"; + pub const COLLECTION_STATE_UNAVAILABLE: &str = "collection_state_unavailable"; + pub const COLLECTOR_DEGRADED: &str = "collector_degraded"; + pub const COLLECTOR_PARTIAL: &str = "collector_partial"; + } +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TopologyFinding { + pub finding_type: String, + pub severity: String, + pub confidence: f64, + pub reason_code: String, + pub affected_entities: Vec, + pub evidence: Vec, + /// Total safe evidence items available before per-finding and payload + /// budget limits were applied. + pub evidence_total: usize, + /// True when safe evidence was omitted from this finding. + pub evidence_truncated: bool, + /// Number of safe evidence items omitted from this finding. + pub evidence_omitted: usize, + pub remediation: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub degraded_reason: Option, + #[serde(skip_serializing_if = "Option::is_none")] + pub confidence_context: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TopologyFindingEntity { + pub entity_type: String, + pub key: String, + pub label: String, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub details: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct TopologyFindingEvidence { + #[serde(skip_serializing_if = "Option::is_none")] + pub evidence_id: Option, + pub source_kind: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub safe_excerpt: Option, +} + +#[cfg(test)] +#[path = "topology_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/domain/src/topology_tests.rs b/crates/shared/cortex/domain/src/topology_tests.rs new file mode 100644 index 00000000..c9b1e6c7 --- /dev/null +++ b/crates/shared/cortex/domain/src/topology_tests.rs @@ -0,0 +1,14 @@ +use super::*; + +#[test] +fn topology_finding_wire_shape_omits_empty_details() { + let entity = TopologyFindingEntity { + entity_type: "host".into(), + key: "dookie".into(), + label: "DOOKIE".into(), + details: Default::default(), + }; + let value = serde_json::to_value(entity).unwrap(); + assert!(value.get("details").is_none()); + assert!(topology_findings::TYPES.contains(&topology_findings::TYPE_COLLECTOR_HEALTH)); +} diff --git a/crates/shared/cortex/domain/tests/public_api.rs b/crates/shared/cortex/domain/tests/public_api.rs new file mode 100644 index 00000000..864288a2 --- /dev/null +++ b/crates/shared/cortex/domain/tests/public_api.rs @@ -0,0 +1,54 @@ +use cortex_domain::{ + DomainError, GraphEntity, GraphEntitySummary, HeartbeatStateFlags, RequestActor, + graph_confidence, hook_signal_detectors, mcp_signal_detectors, observatory_identity, + skill_signal_detectors, topology_findings, +}; + +#[test] +fn independent_consumer_can_use_domain_without_cortex_runtime() { + let actor = RequestActor::mcp_identity(Some("sub".into()), Some("user@example.com".into())); + assert_eq!(actor.display, "user@example.com"); + + let entity = GraphEntity { + id: 1, + entity_type: "host".into(), + canonical_key: "dookie".into(), + display_label: "DOOKIE".into(), + source_kind: "inventory".into(), + source_id: "host:dookie".into(), + trust_level: "observed".into(), + first_seen_at: None, + last_seen_at: None, + }; + assert_eq!(GraphEntitySummary::from(&entity).canonical_key, "dookie"); + assert!(!HeartbeatStateFlags::default().cpu_pressure); + assert!(topology_findings::TYPES.contains(&topology_findings::TYPE_RISKY_MOUNTS)); + assert!(hook_signal_detectors::is_hook_failure_status("failed")); + assert!((graph_confidence::noisy_or_combine(&[0.5, 0.5]) - 0.75).abs() < 1e-9); + assert!(mcp_signal_detectors::detect_timeout_or_rate_limit( + "tool timed out" + )); + assert!(skill_signal_detectors::detect_tool_failure( + "command failed to start" + )); + assert_eq!( + observatory_identity::run_key("dookie", "Codex", "session-1").unwrap(), + "v1|6:dookie|5:codex|9:session-1" + ); + assert_eq!( + DomainError::NotFound("missing".into()).to_string(), + "missing" + ); +} + +#[test] +fn public_api_manifest_has_no_product_specific_dependencies() { + let manifest = + std::fs::read_to_string(format!("{}/Cargo.toml", env!("CARGO_MANIFEST_DIR"))).unwrap(); + for forbidden in ["rusqlite", "r2d2", "axum", "rmcp", "lab-auth", "soma-auth"] { + assert!( + !manifest.contains(forbidden), + "unexpected dependency: {forbidden}" + ); + } +} diff --git a/crates/shared/cortex/ingest-core/Cargo.toml b/crates/shared/cortex/ingest-core/Cargo.toml index 16bef0a5..2ab54750 100644 --- a/crates/shared/cortex/ingest-core/Cargo.toml +++ b/crates/shared/cortex/ingest-core/Cargo.toml @@ -23,6 +23,7 @@ all-features = true default = [] [dependencies] +serde = { version = "1", features = ["derive"] } serde_json = "1" sha2 = "0.10" diff --git a/crates/shared/cortex/ingest-core/README.md b/crates/shared/cortex/ingest-core/README.md index 7c0017c2..ce2a79db 100644 --- a/crates/shared/cortex/ingest-core/README.md +++ b/crates/shared/cortex/ingest-core/README.md @@ -9,7 +9,8 @@ runtime, transport, authentication, CLI, or deployment code. - deterministic log-message normalization for error signature grouping; - stable SHA-256 signature hashing; - bounded JSON metadata encoding; -- recursive sensitive-key redaction and string/key/object limits. +- recursive sensitive-key redaction and string/key/object limits; +- canonical ingest `SourceKind` wire vocabulary and the agent-Docker source marker. It does not parse syslog or OTLP, open SQLite, start background tasks, or know about Cortex product configuration. Those concerns belong in higher extraction diff --git a/crates/shared/cortex/ingest-core/src/lib.rs b/crates/shared/cortex/ingest-core/src/lib.rs index c625709d..4760bb04 100644 --- a/crates/shared/cortex/ingest-core/src/lib.rs +++ b/crates/shared/cortex/ingest-core/src/lib.rs @@ -11,3 +11,7 @@ pub mod metadata; /// Log-message normalization and stable signature hashing. pub mod normalize; +/// Canonical ingest source-kind vocabulary. +pub mod source_kind; + +pub use source_kind::{AGENT_DOCKER_SOURCE_KIND, SourceKind}; diff --git a/crates/shared/cortex/ingest-core/src/source_kind.rs b/crates/shared/cortex/ingest-core/src/source_kind.rs new file mode 100644 index 00000000..60f1efee --- /dev/null +++ b/crates/shared/cortex/ingest-core/src/source_kind.rs @@ -0,0 +1,91 @@ +//! Canonical ingest source-kind contract shared across parsing and persistence. + +use serde::{Deserialize, Serialize}; + +/// Metadata source-kind value for agent-attested Docker identity records. +pub const AGENT_DOCKER_SOURCE_KIND: &str = "agent-docker"; + +/// Transport or collector that produced an ingest record. +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash, Serialize, Deserialize)] +#[serde(rename_all = "kebab-case")] +pub enum SourceKind { + /// UDP syslog listener. + SyslogUdp, + /// TCP syslog listener. + SyslogTcp, + /// Docker log stream. + DockerStream, + /// Docker event stream. + DockerEvent, + /// OpenTelemetry ingest. + Otlp, + /// AdGuard API collector. + AdguardApi, + /// UniFi API collector. + UnifiApi, + /// Generic agent ingest. + Agent, + /// Local shell-history backfill. + ShellHistory, + /// AI agent-launched command spool. + AgentCommand, + /// Cortex-managed local file tail. + FileTail, +} + +impl SourceKind { + /// Every source kind in canonical wire order. + pub const ALL: [Self; 11] = [ + Self::SyslogUdp, + Self::SyslogTcp, + Self::DockerStream, + Self::DockerEvent, + Self::Otlp, + Self::AdguardApi, + Self::UnifiApi, + Self::Agent, + Self::ShellHistory, + Self::AgentCommand, + Self::FileTail, + ]; + + /// Canonical kebab-case wire names in stable order. + pub fn all_wire_names() -> Vec<&'static str> { + Self::ALL.iter().map(|kind| kind.as_str()).collect() + } + + /// Stable kebab-case representation stored in metadata and transport contracts. + pub const fn as_str(self) -> &'static str { + match self { + Self::SyslogUdp => "syslog-udp", + Self::SyslogTcp => "syslog-tcp", + Self::DockerStream => "docker-stream", + Self::DockerEvent => "docker-event", + Self::Otlp => "otlp", + Self::AdguardApi => "adguard-api", + Self::UnifiApi => "unifi-api", + Self::Agent => "agent", + Self::ShellHistory => "shell-history", + Self::AgentCommand => "agent-command", + Self::FileTail => "file-tail", + } + } + + /// Parse a canonical kebab-case wire value. + pub fn from_wire(value: &str) -> Option { + let trimmed = value.trim(); + Self::ALL + .iter() + .copied() + .find(|kind| kind.as_str() == trimmed) + } + + /// Whether this source is one of the two syslog listener transports. + pub const fn is_syslog(self) -> bool { + matches!(self, Self::SyslogUdp | Self::SyslogTcp) + } +} + +#[cfg(test)] +#[path = "source_kind_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/ingest-core/src/source_kind_tests.rs b/crates/shared/cortex/ingest-core/src/source_kind_tests.rs new file mode 100644 index 00000000..9ccb43ee --- /dev/null +++ b/crates/shared/cortex/ingest-core/src/source_kind_tests.rs @@ -0,0 +1,37 @@ +use super::*; + +#[test] +fn wire_values_match_donor_contract() { + assert_eq!( + SourceKind::all_wire_names(), + vec![ + "syslog-udp", + "syslog-tcp", + "docker-stream", + "docker-event", + "otlp", + "adguard-api", + "unifi-api", + "agent", + "shell-history", + "agent-command", + "file-tail", + ] + ); +} + +#[test] +fn wire_round_trip_and_syslog_classification_are_stable() { + for kind in SourceKind::ALL { + assert_eq!(SourceKind::from_wire(kind.as_str()), Some(kind)); + } + assert_eq!( + SourceKind::from_wire(" syslog-udp "), + Some(SourceKind::SyslogUdp) + ); + assert_eq!(SourceKind::from_wire("syslog_udp"), None); + assert!(SourceKind::SyslogUdp.is_syslog()); + assert!(SourceKind::SyslogTcp.is_syslog()); + assert!(!SourceKind::DockerStream.is_syslog()); + assert_eq!(AGENT_DOCKER_SOURCE_KIND, "agent-docker"); +} diff --git a/crates/shared/cortex/inventory/Cargo.toml b/crates/shared/cortex/inventory/Cargo.toml new file mode 100644 index 00000000..d2796aa5 --- /dev/null +++ b/crates/shared/cortex/inventory/Cargo.toml @@ -0,0 +1,30 @@ +[package] +name = "cortex-inventory" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +description = "Pure Cortex inventory snapshot contracts and bounded inventory helpers." +homepage.workspace = true +license = "AGPL-3.0-only" +repository.workspace = true +readme = "README.md" +keywords = ["inventory", "observability", "homelab", "topology"] +categories = ["data-structures", "development-tools::debugging"] +publish = false + +[package.metadata.soma-architecture] +layer = "shared" + +[package.metadata.docs.rs] +all-features = true + +[features] +default = [] + +[dependencies] +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[lints] +workspace = true diff --git a/crates/shared/cortex/inventory/README.md b/crates/shared/cortex/inventory/README.md new file mode 100644 index 00000000..ea22c9c1 --- /dev/null +++ b/crates/shared/cortex/inventory/README.md @@ -0,0 +1,5 @@ +# cortex-inventory + +Pure inventory snapshot contracts extracted from Cortex donor commit `7edf23fadb94650c2d2a2f9c80111fb44319eea8`. + +This crate owns the serializable homelab inventory vocabulary and bounded inventory helpers. Collection, transport, persistence, graph application, SSH, Docker, and orchestration remain outside this crate. diff --git a/crates/shared/cortex/inventory/src/lib.rs b/crates/shared/cortex/inventory/src/lib.rs new file mode 100644 index 00000000..989e7662 --- /dev/null +++ b/crates/shared/cortex/inventory/src/lib.rs @@ -0,0 +1,12 @@ +//! Pure Cortex inventory snapshot contracts. +//! +//! This crate owns the serializable inventory vocabulary shared by collectors, +//! graph projection, storage, and transports. It deliberately contains no SSH, +//! Docker, HTTP, persistence, scheduling, or runtime orchestration code. + +/// Bounded inventory constants and collection-safe utility helpers. +pub mod limits; +/// Serializable homelab inventory snapshot schema. +pub mod schema; + +pub use schema::*; diff --git a/crates/shared/cortex/inventory/src/limits.rs b/crates/shared/cortex/inventory/src/limits.rs new file mode 100644 index 00000000..a5e573cd --- /dev/null +++ b/crates/shared/cortex/inventory/src/limits.rs @@ -0,0 +1,35 @@ +pub const INVENTORY_SCHEMA: &str = "cortex.homelab_inventory.v1"; +pub const MAP_SCHEMA: &str = "cortex.homelab_map.v2"; +pub const MAX_RAW_ARTIFACT_BYTES: usize = 512 * 1024; +pub const MAX_HTTP_BODY_BYTES: usize = 512 * 1024; +pub const MAX_COMMAND_OUTPUT_BYTES: usize = 256 * 1024; +pub const MAX_RAW_BATCH_OUTPUT_BYTES: usize = 8 * 1024 * 1024; +pub const MAX_JSON_DEPTH: usize = 12; +pub const MAX_ARRAY_ENTRIES: usize = 200; +pub const MAX_SECTION_ITEMS: usize = 250; +pub const DEFAULT_COLLECTION_DEADLINE_SECS: u64 = 45; +pub const DEFAULT_COLLECTOR_DEADLINE_SECS: u64 = 12; +pub const DEFAULT_PROBE_DEADLINE_SECS: u64 = 5; + +pub fn cap_vec(items: &mut Vec, limit: usize) -> bool { + if items.len() <= limit { + return false; + } + items.truncate(limit); + true +} + +pub fn truncate_text(input: &str, max_bytes: usize) -> (String, bool) { + if input.len() <= max_bytes { + return (input.to_string(), false); + } + let mut end = max_bytes; + while !input.is_char_boundary(end) { + end -= 1; + } + (input[..end].to_string(), true) +} + +#[cfg(test)] +#[path = "limits_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/inventory/src/limits_tests.rs b/crates/shared/cortex/inventory/src/limits_tests.rs new file mode 100644 index 00000000..35e29e6d --- /dev/null +++ b/crates/shared/cortex/inventory/src/limits_tests.rs @@ -0,0 +1,18 @@ +use super::*; + +#[test] +fn truncate_text_respects_utf8_boundary() { + let (out, truncated) = truncate_text("hello\u{1f600}world", 6); + + assert!(truncated); + assert_eq!(out, "hello"); + assert!(std::str::from_utf8(out.as_bytes()).is_ok()); +} + +#[test] +fn truncate_text_reports_untruncated_input() { + let (out, truncated) = truncate_text("hello", 64); + + assert!(!truncated); + assert_eq!(out, "hello"); +} diff --git a/crates/shared/cortex/inventory/src/schema.rs b/crates/shared/cortex/inventory/src/schema.rs new file mode 100644 index 00000000..5760592c --- /dev/null +++ b/crates/shared/cortex/inventory/src/schema.rs @@ -0,0 +1,329 @@ +use serde::{Deserialize, Serialize}; +use serde_json::Value; +use std::collections::BTreeMap; + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct HomelabInventory { + pub schema: String, + pub generated_at: String, + pub run_id: String, + pub freshness: InventoryFreshness, + pub summary: InventorySummary, + pub nodes: Vec, + pub services: Vec, + pub compose_projects: Vec, + pub reverse_proxies: Vec, + pub networks: Vec, + pub storage: Vec, + pub media_services: Vec, + pub projects: Vec, + pub artifact_refs: Vec, + pub collection_errors: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub graph_projection: Option, +} + +impl HomelabInventory { + pub fn empty(run_id: String, generated_at: String) -> Self { + Self { + schema: crate::limits::INVENTORY_SCHEMA.to_string(), + generated_at: generated_at.clone(), + run_id, + freshness: InventoryFreshness { + generated_at, + stale_after_secs: 86_400, + is_stale: false, + cache_status: "generated".to_string(), + }, + summary: InventorySummary::default(), + nodes: Vec::new(), + services: Vec::new(), + compose_projects: Vec::new(), + reverse_proxies: Vec::new(), + networks: Vec::new(), + storage: Vec::new(), + media_services: Vec::new(), + projects: Vec::new(), + artifact_refs: Vec::new(), + collection_errors: Vec::new(), + graph_projection: None, + } + } + + pub fn recompute_summary(&mut self) { + self.summary = InventorySummary { + nodes: self.nodes.len(), + services: self.services.len(), + compose_projects: self.compose_projects.len(), + reverse_proxies: self.reverse_proxies.len(), + networks: self.networks.len(), + storage: self.storage.len(), + media_services: self.media_services.len(), + projects: self.projects.len(), + artifacts: self.artifact_refs.len(), + errors: self.collection_errors.len(), + truncated: self.artifact_refs.iter().any(|artifact| artifact.truncated) + || self.collection_errors.iter().any(|error| error.truncated), + }; + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] +pub struct InventorySummary { + pub nodes: usize, + pub services: usize, + pub compose_projects: usize, + pub reverse_proxies: usize, + pub networks: usize, + pub storage: usize, + pub media_services: usize, + pub projects: usize, + pub artifacts: usize, + pub errors: usize, + pub truncated: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct InventoryFreshness { + pub generated_at: String, + pub stale_after_secs: u64, + pub is_stale: bool, + pub cache_status: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct InventoryNode { + pub id: String, + pub hostname: String, + pub trust_level: TrustLevel, + pub provenance: Provenance, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub roles: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub ips: Vec, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub os: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub cpu: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub memory: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub listeners: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub storage: Vec, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub extras: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct InventoryService { + pub id: String, + pub name: String, + pub kind: String, + pub trust_level: TrustLevel, + pub provenance: Provenance, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub host: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub image: Option, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub status: Option, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub domains: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub ports: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub mounts: Vec, + #[serde(default, skip_serializing_if = "Vec::is_empty")] + pub env_keys: Vec, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub labels: BTreeMap, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub details: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ComposeProject { + pub name: String, + pub provenance: Provenance, + pub services: Vec, + pub compose_files: Vec, + pub domains: Vec, + pub ports: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ReverseProxyRoute { + pub id: String, + pub server_names: Vec, + pub upstreams: Vec, + pub provenance: Provenance, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct NetworkSegment { + pub name: String, + pub kind: String, + pub members: Vec, + pub provenance: Provenance, + #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] + pub details: BTreeMap, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct StorageSummary { + pub id: String, + pub mount: String, + pub fs_type: Option, + pub total_bytes: Option, + pub available_bytes: Option, + pub provenance: Provenance, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct MediaService { + pub service: String, + pub base_url: String, + pub status: String, + pub version: Option, + pub topology: BTreeMap, + pub provenance: Provenance, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ProjectRepo { + pub path: String, + pub branch: Option, + pub head: Option, + pub dirty: bool, + pub ahead: Option, + pub behind: Option, + pub worktrees: Vec, + pub provenance: Provenance, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ListenerFact { + pub protocol: String, + pub bind: String, + pub port: Option, + pub process: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct PortMapping { + pub host_ip: Option, + pub host_port: Option, + pub container_port: Option, + pub protocol: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct MountRef { + pub source: Option, + pub target: String, + pub read_only: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct ArtifactRef { + pub id: String, + pub kind: String, + pub collector: String, + pub source_host: Option, + pub source_path: Option, + pub cache_path: String, + pub redaction: RedactionStatus, + pub byte_len: usize, + pub truncated: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct CollectionError { + pub collector: String, + pub phase: String, + pub severity: String, + pub message: String, + pub elapsed_ms: u128, + pub truncated: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct CollectionState { + pub schema: String, + pub run_id: String, + pub started_at: String, + pub finished_at: String, + pub status: String, + pub collectors: Vec, + pub artifact_refs: Vec, + pub errors: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct CollectorState { + pub name: String, + pub status: String, + pub started_at: String, + pub finished_at: String, + pub elapsed_ms: u128, + pub warnings: Vec, + pub artifacts: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, Default, PartialEq, Eq)] +pub struct GraphProjectionSummary { + pub status: String, + pub source_kinds_reserved: Vec, + pub next_queries: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct Provenance { + pub source: String, + pub source_kind: String, + pub collected_at: String, + pub evidence: Vec, +} + +impl Provenance { + pub fn new( + source: impl Into, + source_kind: impl Into, + collected_at: String, + ) -> Self { + Self { + source: source.into(), + source_kind: source_kind.into(), + collected_at, + evidence: Vec::new(), + } + } +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct EvidenceRef { + pub artifact_id: Option, + pub safe_excerpt: Option, + pub trust_level: TrustLevel, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum TrustLevel { + Verified, + Observed, + Claimed, + Inferred, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +#[serde(rename_all = "snake_case")] +pub enum RedactionStatus { + Redacted, + NoSecretsDetected, +} + +#[cfg(test)] +#[path = "schema_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/inventory/src/schema_tests.rs b/crates/shared/cortex/inventory/src/schema_tests.rs new file mode 100644 index 00000000..dde1afdf --- /dev/null +++ b/crates/shared/cortex/inventory/src/schema_tests.rs @@ -0,0 +1,32 @@ +use super::*; + +#[test] +fn empty_inventory_uses_locked_schema_and_defaults() { + let inventory = + HomelabInventory::empty("run-1".to_string(), "2026-08-18T12:00:00Z".to_string()); + assert_eq!(inventory.schema, crate::limits::INVENTORY_SCHEMA); + assert_eq!(inventory.run_id, "run-1"); + assert!(inventory.nodes.is_empty()); + assert!(inventory.services.is_empty()); + assert_eq!(inventory.summary, InventorySummary::default()); +} + +#[test] +fn trust_level_wire_values_match_donor_contract() { + assert_eq!( + serde_json::to_string(&TrustLevel::Verified).unwrap(), + "\"verified\"" + ); + assert_eq!( + serde_json::to_string(&TrustLevel::Observed).unwrap(), + "\"observed\"" + ); + assert_eq!( + serde_json::to_string(&TrustLevel::Claimed).unwrap(), + "\"claimed\"" + ); + assert_eq!( + serde_json::to_string(&TrustLevel::Inferred).unwrap(), + "\"inferred\"" + ); +} diff --git a/crates/shared/cortex/storage-sqlite/Cargo.toml b/crates/shared/cortex/storage-sqlite/Cargo.toml new file mode 100644 index 00000000..5fa40249 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/Cargo.toml @@ -0,0 +1,47 @@ +[package] +name = "cortex-storage-sqlite" +version = "0.1.0" +edition.workspace = true +rust-version.workspace = true +authors.workspace = true +description = "SQLite persistence adapter extracted from Cortex." +homepage.workspace = true +license = "AGPL-3.0-only" +repository.workspace = true +readme = "README.md" +keywords = ["sqlite", "observability", "logs", "storage"] +categories = ["database", "development-tools::debugging"] +publish = false + +[package.metadata.soma-architecture] +layer = "shared" + +[package.metadata.docs.rs] +all-features = true + +[features] +default = [] + +[dependencies] +anyhow = "1" +chrono = { version = "0.4", features = ["serde"] } +cortex-domain = { workspace = true } +cortex-ingest-core = { workspace = true } +cortex-inventory = { workspace = true } +parking_lot = "0.12" +r2d2 = "0.8" +r2d2_sqlite = "0.35" +rusqlite = { version = "0.40", features = ["bundled", "vtab", "backup", "hooks"] } +rustix = { version = "1", features = ["fs"] } +scheduled-thread-pool = "0.2" +serde = { version = "1", features = ["derive"] } +serde_json = "1" +sha2 = "0.10" +tracing = "0.1" + +[dev-dependencies] +regex = "1" +tempfile = "3" + +[lints] +workspace = true diff --git a/crates/shared/cortex/storage-sqlite/README.md b/crates/shared/cortex/storage-sqlite/README.md new file mode 100644 index 00000000..c9d16b07 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/README.md @@ -0,0 +1,27 @@ +# cortex-storage-sqlite + +`cortex-storage-sqlite` is Cortex's reusable SQLite persistence adapter. It +owns connection pooling, schema migrations, transactional writes, query/FTS +projections, retention and storage-budget enforcement, graph persistence, event +and incident persistence, heartbeat persistence, and observatory tables. + +The dependency direction is one-way: this crate may depend on +`cortex-domain`, `cortex-ingest-core`, and the pure `cortex-inventory` snapshot +contract; those crates do not depend on SQLite. Product runtime configuration, +transport DTOs, scanner implementations, collectors, and application services +are not storage dependencies. Normalized scanner events cross the boundary via +storage-neutral input contracts. + +## Compatibility + +The extraction baseline is Cortex commit +`7edf23fadb94650c2d2a2f9c80111fb44319eea8`. Migration ordering, +`KNOWN_SCHEMA_VERSION`, PRAGMA behavior, write-lock coordination, and donor +database fixtures are parity contracts. Raw SQLite row types remain adapter +details unless they are explicitly documented as storage/query projections. + +Application-facing persistence capabilities that are already consumed by donor +services are explicit storage ports, including error-signature state, notification +outbox/firings, stream health, LLM invocation persistence, observatory paging, +pattern-row queries, and a closed-enum PRAGMA diagnostics API. Pure graph +confidence math is intentionally owned by `cortex-domain` instead of SQLite. diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory.rs new file mode 100644 index 00000000..ac72fea6 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory.rs @@ -0,0 +1,497 @@ +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::fmt; +use std::str::FromStr; + +#[path = "agent_observatory_commits.rs"] +mod commits; +pub use commits::{ + GitCommitReachabilityUpdate, GitCommitUpsert, get_git_commit, list_git_commits, + reconcile_git_commits, upsert_git_commits, +}; +#[path = "agent_observatory_sources.rs"] +mod sources; +pub use sources::{ + AgentHookSourceRow, AgentLlmSourceRow, AgentMcpSourceRow, AgentSkillSourceRow, AgentSourceKind, + AgentSourcePage, AgentSourceRecord, page_agent_sources, +}; + +#[path = "agent_observatory_projection.rs"] +mod projection; +pub use projection::{ + AgentActorRow, AgentActorUpsert, AgentProjectionOutboxInput, AgentProjectionOutboxRow, + AgentProjectionRunMatch, AgentProjectionWorktreeRef, AgentProjectionWriteInput, + AgentProjectionWriteResult, AgentRunEventUpsert, AgentRunUpsert, AgentWorktreeEvidenceUpsert, + find_active_projection_worktree, find_unique_overlapping_projection_run, + find_unique_projection_run_by_session, write_agent_projection, +}; + +#[path = "agent_observatory_observations.rs"] +mod observations; +pub use observations::{ + RepositoryObservationInput, list_repository_observations, + record_repository_observations_if_changed, +}; + +#[path = "agent_observatory_queries.rs"] +mod queries; +pub use queries::{ + RepositoryReconcileResult, RepositoryUpsert, RepositoryWorktreeUpsert, get_repository_by_key, + get_worktree_by_key, list_repository_worktrees, mark_repository_removed, mark_worktree_removed, + reconcile_repository, +}; + +use crate::pool::{DbPool, write_lock}; +use anyhow::{Context, Result}; +use rusqlite::TransactionBehavior; + +/// Read or initialize the durable cursor for an observatory projection source. +/// +/// Cursor initialization is a write and therefore participates in the same +/// process-wide write coordination as every other SQLite mutation. +pub fn projection_cursor(pool: &DbPool, source_name: &str) -> Result { + let _write_guard = write_lock(); + let connection = pool.get().context("acquire database connection")?; + connection.execute( + "INSERT OR IGNORE INTO agent_projection_cursors + (cursor_type, source_name, cursor_value) VALUES ('source', ?1, '')", + [source_name], + )?; + Ok(connection.query_row( + "SELECT cursor_value FROM agent_projection_cursors + WHERE cursor_type = 'source' AND source_name = ?1", + [source_name], + |row| row.get(0), + )?) +} + +/// Advance an initialized observatory projection source cursor. +pub fn advance_projection_cursor(pool: &DbPool, source_name: &str, cursor: &str) -> Result<()> { + let _write_guard = write_lock(); + let connection = pool.get().context("acquire database connection")?; + let changed = connection.execute( + "UPDATE agent_projection_cursors + SET cursor_value = ?2, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + WHERE cursor_type = 'source' AND source_name = ?1", + rusqlite::params![source_name, cursor], + )?; + anyhow::ensure!(changed == 1, "projection cursor is not initialized"); + Ok(()) +} + +/// Record the latest health status for an observatory projection worker. +pub fn record_projection_health( + pool: &DbPool, + worker: &str, + status: &str, + detail: &str, +) -> Result<()> { + let _write_guard = write_lock(); + let connection = pool.get().context("acquire database connection")?; + connection.execute( + "INSERT INTO agent_projection_cursors (cursor_type, source_name, cursor_value) + VALUES ('health', ?1, json_object('status', ?2, 'detail', ?3, 'attempts', 1)) + ON CONFLICT(cursor_type, source_name) DO UPDATE SET + cursor_value = json_object( + 'status', ?2, 'detail', ?3, + 'attempts', COALESCE(json_extract(agent_projection_cursors.cursor_value, '$.attempts'), 0) + 1 + ), + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')", + rusqlite::params![worker, status, detail], + )?; + Ok(()) +} + +/// Read the JSON health state for an observatory projection worker. +pub fn projection_health(pool: &DbPool, worker: &str) -> Result> { + use rusqlite::OptionalExtension; + let connection = pool.get().context("acquire database connection")?; + Ok(connection + .query_row( + "SELECT cursor_value FROM agent_projection_cursors + WHERE cursor_type = 'health' AND source_name = ?1", + [worker], + |row| row.get(0), + ) + .optional()?) +} + +#[derive(Debug, Clone, PartialEq)] +pub struct GitRepositoryReconcileResult { + pub topology: RepositoryReconcileResult, + pub commits: Vec, + pub observations: Vec, +} + +/// Atomically publishes one Git observer snapshot. Readers can never observe a +/// commit import without its matching topology and observation rows. +pub fn reconcile_git_repository_snapshot( + pool: &DbPool, + repository: &RepositoryUpsert, + worktrees: &[RepositoryWorktreeUpsert], + commits: &[GitCommitUpsert], + reachability: &[GitCommitReachabilityUpdate], + observations: &[RepositoryObservationInput], + observed_at: &str, +) -> Result { + reconcile_git_repository_snapshot_with( + pool, + repository, + worktrees, + commits, + reachability, + observed_at, + |_| Ok(observations.to_vec()), + ) +} + +pub fn reconcile_git_repository_snapshot_with( + pool: &DbPool, + repository: &RepositoryUpsert, + worktrees: &[RepositoryWorktreeUpsert], + commits: &[GitCommitUpsert], + reachability: &[GitCommitReachabilityUpdate], + observed_at: &str, + build_observations: F, +) -> Result +where + F: FnOnce(&RepositoryReconcileResult) -> Result>, +{ + queries::validate_reconcile_repository(repository, worktrees, observed_at)?; + commits::validate_reconcile_git_commits(commits, reachability, observed_at)?; + let _write_guard = write_lock(); + let mut connection = pool.get().context("acquire database connection")?; + let tx = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let topology = queries::reconcile_repository_tx(&tx, repository, worktrees, observed_at)?; + let commits = commits::reconcile_git_commits_tx( + &tx, + &repository.repository_key, + commits, + reachability, + observed_at, + )?; + let observation_inputs = build_observations(&topology)?; + observations::validate_repository_observations( + &repository.repository_key, + &observation_inputs, + observed_at, + )?; + let observations = observations::record_repository_observations_if_changed_tx( + &tx, + &repository.repository_key, + &observation_inputs, + observed_at, + )?; + tx.commit()?; + Ok(GitRepositoryReconcileResult { + topology, + commits, + observations, + }) +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct EnumParseError { + type_name: &'static str, + value: String, +} + +impl EnumParseError { + pub(crate) fn new(type_name: &'static str, value: &str) -> Self { + Self { + type_name, + value: value.to_string(), + } + } +} + +impl fmt::Display for EnumParseError { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + write!( + formatter, + "invalid {} value: {}", + self.type_name, self.value + ) + } +} + +impl std::error::Error for EnumParseError {} + +macro_rules! string_enum { + ($name:ident { $($variant:ident => $value:literal),+ $(,)? }) => { + #[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] + pub enum $name { $($variant),+ } + + impl $name { + pub const ALL: &'static [Self] = &[$(Self::$variant),+]; + pub const fn as_str(self) -> &'static str { + match self { $(Self::$variant => $value),+ } + } + } + + impl fmt::Display for $name { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } + } + + impl FromStr for $name { + type Err = EnumParseError; + fn from_str(value: &str) -> Result { + match value { + $($value => Ok(Self::$variant),)+ + _ => Err(EnumParseError::new(stringify!($name), value)), + } + } + } + + impl Serialize for $name { + fn serialize(&self, serializer: S) -> Result + where S: Serializer { + serializer.serialize_str(self.as_str()) + } + } + + impl<'de> Deserialize<'de> for $name { + fn deserialize(deserializer: D) -> Result + where D: Deserializer<'de> { + let value = String::deserialize(deserializer)?; + value.parse().map_err(serde::de::Error::custom) + } + } + }; +} + +string_enum!(RunStatus { + Starting => "starting", Active => "active", Waiting => "waiting", Idle => "idle", + Stale => "stale", Completed => "completed", Failed => "failed", Abandoned => "abandoned", +}); + +string_enum!(AgentEventKind { + Lifecycle => "lifecycle", Transcript => "transcript", Command => "command", + ShellHistory => "shell_history", GitStatus => "git_status", GitHead => "git_head", + GitCommit => "git_commit", FileOperation => "file_operation", Mcp => "mcp", Hook => "hook", + Skill => "skill", Llm => "llm", OtlpLog => "otlp_log", OtlpSpan => "otlp_span", + OtlpMetric => "otlp_metric", Heartbeat => "heartbeat", Error => "error", + ProviderEvent => "provider_event", +}); + +string_enum!(EvidenceTrustLevel { + Verified => "verified", Claimed => "claimed", Correlated => "correlated", + Inferred => "inferred", Refuted => "refuted", +}); + +string_enum!(RepositoryObservationKind { + Discovered => "discovered", Status => "status", Head => "head", Branch => "branch", + WorktreeAdded => "worktree_added", WorktreeRemoved => "worktree_removed", + OverflowReconcile => "overflow_reconcile", PeriodicReconcile => "periodic_reconcile", + Error => "error", +}); + +string_enum!(StreamEventName { + RunCreated => "run.created", RunUpdated => "run.updated", RunStatus => "run.status", + RunEvent => "run.event", WorktreeUpdated => "worktree.updated", + RepositoryUpdated => "repository.updated", TelemetryUpdated => "telemetry.updated", + ObservatoryReset => "observatory.reset", +}); + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RepositoryRow { + pub id: i64, + pub repository_key: String, + pub hostname: String, + pub common_git_dir: String, + pub primary_path: String, + pub display_name: String, + pub remote_url_hash: Option, + pub first_seen_at: String, + pub last_seen_at: String, + pub removed_at: Option, + pub metadata_json: String, + pub created_at: String, + pub updated_at: String, +} + +#[allow(clippy::struct_excessive_bools)] +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RepositoryWorktreeRow { + pub id: i64, + pub worktree_key: String, + pub repository_id: i64, + pub hostname: String, + pub path: String, + pub git_dir: String, + pub branch_ref: Option, + pub branch_name: Option, + pub head_sha: Option, + pub upstream_ref: Option, + pub detached: bool, + pub bare: bool, + pub locked: bool, + pub lock_reason: Option, + pub prunable: bool, + pub prune_reason: Option, + pub dirty: bool, + pub staged_count: i64, + pub unstaged_count: i64, + pub untracked_count: i64, + pub ahead: Option, + pub behind: Option, + pub status_hash: Option, + pub first_seen_at: String, + pub last_seen_at: String, + pub removed_at: Option, + pub created_at: String, + pub updated_at: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct RepositoryObservationRow { + pub id: i64, + pub observation_key: String, + pub repository_id: i64, + pub worktree_id: Option, + pub observed_at: String, + pub observation_kind: RepositoryObservationKind, + pub old_head_sha: Option, + pub new_head_sha: Option, + pub summary: String, + pub payload_json: String, + pub created_at: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct GitCommitRow { + pub id: i64, + pub repository_id: i64, + pub sha: String, + pub parent_shas_json: String, + pub author_name: Option, + pub author_email_hash: Option, + pub authored_at: Option, + pub committed_at: Option, + pub subject: String, + pub changed_files: Option, + pub insertions: Option, + pub deletions: Option, + pub changed_paths_json: String, + pub first_observed_at: String, + pub last_observed_at: String, + pub reachable: bool, + pub metadata_json: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AgentRunRow { + pub id: i64, + pub run_key: String, + pub native_session_id: String, + pub tool: String, + pub provider_tool: Option, + pub hostname: String, + pub parent_run_id: Option, + pub previous_run_id: Option, + pub primary_worktree_id: Option, + pub transcript_path: Option, + pub process_id: Option, + pub status: RunStatus, + pub status_reason: String, + pub status_observed_at: String, + pub started_at: String, + pub last_activity_at: String, + pub ended_at: Option, + pub first_source_log_id: Option, + pub last_source_log_id: Option, + pub last_event_id: Option, + pub event_count: i64, + pub error_count: i64, + pub primary_branch: Option, + pub start_head_sha: Option, + pub current_head_sha: Option, + pub projection_version: i64, + pub freshness_json: String, + pub metadata_json: String, + pub created_at: String, + pub updated_at: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AgentRunEventRow { + pub id: i64, + pub event_key: String, + pub run_id: i64, + pub actor_id: Option, + pub worktree_id: Option, + pub commit_id: Option, + pub observed_at: String, + pub ingested_at: String, + pub event_kind: AgentEventKind, + pub source_kind: String, + pub source_id: String, + pub source_log_id: Option, + pub provider_sequence: Option, + pub trace_id: Option, + pub span_id: Option, + pub severity: String, + pub title: String, + pub summary: String, + pub payload_json: String, + pub content_scrubbed: bool, + pub created_at: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AgentRunWorktreeEvidenceRow { + pub id: i64, + pub relation_key: String, + pub run_id: i64, + pub worktree_id: i64, + pub evidence_kind: String, + pub evidence_source: String, + pub trust_level: EvidenceTrustLevel, + pub confidence: f64, + pub is_primary: bool, + pub first_seen_at: String, + pub last_seen_at: String, + pub metadata_json: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct AgentRunCommitEvidenceRow { + pub id: i64, + pub relation_key: String, + pub run_id: i64, + pub commit_id: i64, + pub worktree_id: Option, + pub evidence_kind: String, + pub evidence_source: String, + pub trust_level: EvidenceTrustLevel, + pub confidence: f64, + pub observed_at: String, + pub metadata_json: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct ProjectionCursorRow { + pub source_name: String, + pub last_source_id: i64, + pub source_max_id: i64, + pub projection_version: i64, + pub last_success_at: Option, + pub last_error_at: Option, + pub last_error: Option, + pub retry_count: i64, + pub updated_at: String, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct StreamOutboxRow { + pub id: i64, + pub event_name: StreamEventName, + pub entity_type: String, + pub entity_key: String, + pub run_id: Option, + pub payload_json: String, + pub created_at: String, + pub expires_at: String, +} diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_commits.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_commits.rs new file mode 100644 index 00000000..94d58b0f --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_commits.rs @@ -0,0 +1,309 @@ +//! Transactional exact Git commit persistence. + +use super::GitCommitRow; +use crate::pool::{DbPool, write_lock}; +use anyhow::{Context, Result, bail}; +use rusqlite::{Connection, OptionalExtension, Row, Transaction, TransactionBehavior, params}; +use serde_json::Value; +use std::collections::HashSet; + +const COMMIT_COLUMNS: &str = + "id, repository_id, sha, parent_shas_json, author_name, author_email_hash, + authored_at, committed_at, subject, changed_files, insertions, deletions, + changed_paths_json, first_observed_at, last_observed_at, reachable, metadata_json"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GitCommitReachabilityUpdate { + pub sha: String, + pub reachable: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GitCommitUpsert { + pub sha: String, + pub parent_shas_json: String, + pub author_name: Option, + pub author_email_hash: Option, + pub authored_at: Option, + pub committed_at: Option, + pub subject: String, + pub changed_files: Option, + pub insertions: Option, + pub deletions: Option, + pub changed_paths_json: String, + pub reachable: bool, + pub metadata_json: String, +} + +fn commit_row(row: &Row<'_>) -> rusqlite::Result { + Ok(GitCommitRow { + id: row.get(0)?, + repository_id: row.get(1)?, + sha: row.get(2)?, + parent_shas_json: row.get(3)?, + author_name: row.get(4)?, + author_email_hash: row.get(5)?, + authored_at: row.get(6)?, + committed_at: row.get(7)?, + subject: row.get(8)?, + changed_files: row.get(9)?, + insertions: row.get(10)?, + deletions: row.get(11)?, + changed_paths_json: row.get(12)?, + first_observed_at: row.get(13)?, + last_observed_at: row.get(14)?, + reachable: row.get(15)?, + metadata_json: row.get(16)?, + }) +} + +fn valid_object_id(value: &str) -> bool { + matches!(value.len(), 40 | 64) && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn parse_json(value: &str, field: &str) -> Result { + serde_json::from_str(value).with_context(|| format!("{field} must be valid JSON")) +} + +fn validate_timestamp(value: Option<&str>, field: &str) -> Result<()> { + if let Some(value) = value { + chrono::DateTime::parse_from_rfc3339(value) + .with_context(|| format!("invalid {field}: {value}"))?; + } + Ok(()) +} + +fn validate_commit(input: &GitCommitUpsert) -> Result<()> { + if !valid_object_id(&input.sha) { + bail!("sha must be a 40- or 64-byte hex object ID"); + } + let parents = parse_json(&input.parent_shas_json, "parent_shas_json")?; + let Value::Array(parents) = parents else { + bail!("parent_shas_json must be a JSON array"); + }; + for parent in parents { + let Some(parent) = parent.as_str() else { + bail!("parent_shas_json entries must be strings"); + }; + if !valid_object_id(parent) { + bail!("parent_shas_json contains an invalid object ID"); + } + } + let paths = parse_json(&input.changed_paths_json, "changed_paths_json")?; + if !paths.is_array() { + bail!("changed_paths_json must be a JSON array"); + } + let metadata = parse_json(&input.metadata_json, "metadata_json")?; + if !metadata.is_object() { + bail!("metadata_json must be a JSON object"); + } + validate_timestamp(input.authored_at.as_deref(), "authored_at")?; + validate_timestamp(input.committed_at.as_deref(), "committed_at")?; + for (field, value) in [ + ("changed_files", input.changed_files), + ("insertions", input.insertions), + ("deletions", input.deletions), + ] { + if value.is_some_and(|value| value < 0) { + bail!("{field} must be non-negative"); + } + } + if input + .author_email_hash + .as_deref() + .is_some_and(|value| value.trim().is_empty()) + { + bail!("author_email_hash must be non-empty when present"); + } + Ok(()) +} + +fn validate_observed_at(observed_at: &str) -> Result<()> { + chrono::DateTime::parse_from_rfc3339(observed_at) + .with_context(|| format!("invalid observed_at: {observed_at}"))?; + Ok(()) +} + +fn repository_id(conn: &Connection, repository_key: &str) -> Result { + if repository_key.trim().is_empty() { + bail!("repository_key must be non-empty"); + } + conn.query_row( + "SELECT id FROM repositories WHERE repository_key = ?1", + [repository_key], + |row| row.get(0), + ) + .optional()? + .with_context(|| format!("repository not found for key {repository_key}")) +} + +fn commit_by_sha(conn: &Connection, repository_id: i64, sha: &str) -> Result> { + let sql = format!( + "SELECT {COMMIT_COLUMNS} FROM git_commits + WHERE repository_id = ?1 AND sha = ?2" + ); + conn.query_row(&sql, params![repository_id, sha], commit_row) + .optional() + .context("query Git commit by SHA") +} + +pub fn reconcile_git_commits( + pool: &DbPool, + repository_key: &str, + commits: &[GitCommitUpsert], + reachability: &[GitCommitReachabilityUpdate], + observed_at: &str, +) -> Result> { + validate_reconcile_git_commits(commits, reachability, observed_at)?; + if commits.is_empty() && reachability.is_empty() { + return Ok(Vec::new()); + } + let _write_guard = write_lock(); + let mut conn = pool.get().context("acquire database connection")?; + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let rows = reconcile_git_commits_tx(&tx, repository_key, commits, reachability, observed_at)?; + tx.commit()?; + Ok(rows) +} + +pub(super) fn validate_reconcile_git_commits( + commits: &[GitCommitUpsert], + reachability: &[GitCommitReachabilityUpdate], + observed_at: &str, +) -> Result<()> { + validate_observed_at(observed_at)?; + let mut shas = HashSet::new(); + for commit in commits { + validate_commit(commit)?; + if !shas.insert(commit.sha.as_str()) { + bail!("duplicate commit SHA in batch"); + } + } + let mut update_shas = HashSet::new(); + for update in reachability { + if !valid_object_id(&update.sha) { + bail!("reachability SHA must be a 40- or 64-byte hex object ID"); + } + if !update_shas.insert(update.sha.as_str()) { + bail!("duplicate reachability SHA in batch"); + } + } + Ok(()) +} + +pub(super) fn reconcile_git_commits_tx( + tx: &Transaction<'_>, + repository_key: &str, + commits: &[GitCommitUpsert], + reachability: &[GitCommitReachabilityUpdate], + observed_at: &str, +) -> Result> { + let repository_id = repository_id(tx, repository_key)?; + let mut rows = Vec::with_capacity(commits.len()); + for commit in commits { + tx.execute( + "INSERT INTO git_commits + (repository_id, sha, parent_shas_json, author_name, author_email_hash, + authored_at, committed_at, subject, changed_files, insertions, deletions, + changed_paths_json, first_observed_at, last_observed_at, reachable, + metadata_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?13, + ?14, ?15) + ON CONFLICT(repository_id, sha) DO UPDATE SET + parent_shas_json = excluded.parent_shas_json, + author_name = COALESCE(excluded.author_name, git_commits.author_name), + author_email_hash = COALESCE( + excluded.author_email_hash, git_commits.author_email_hash + ), + authored_at = COALESCE(excluded.authored_at, git_commits.authored_at), + committed_at = COALESCE(excluded.committed_at, git_commits.committed_at), + subject = excluded.subject, + changed_files = excluded.changed_files, + insertions = excluded.insertions, + deletions = excluded.deletions, + changed_paths_json = excluded.changed_paths_json, + last_observed_at = excluded.last_observed_at, + reachable = excluded.reachable, + metadata_json = excluded.metadata_json", + params![ + repository_id, + commit.sha, + commit.parent_shas_json, + commit.author_name, + commit.author_email_hash, + commit.authored_at, + commit.committed_at, + commit.subject, + commit.changed_files, + commit.insertions, + commit.deletions, + commit.changed_paths_json, + observed_at, + commit.reachable, + commit.metadata_json, + ], + )?; + rows.push( + commit_by_sha(tx, repository_id, &commit.sha)? + .context("commit missing after upsert")?, + ); + } + for update in reachability { + let changed = tx.execute( + "UPDATE git_commits + SET reachable = ?3, last_observed_at = ?4 + WHERE repository_id = ?1 AND sha = ?2", + params![repository_id, update.sha, update.reachable, observed_at], + )?; + if changed != 1 { + bail!( + "Git commit not found for reachability update: {}", + update.sha + ); + } + } + Ok(rows) +} + +pub fn upsert_git_commits( + pool: &DbPool, + repository_key: &str, + commits: &[GitCommitUpsert], + observed_at: &str, +) -> Result> { + reconcile_git_commits(pool, repository_key, commits, &[], observed_at) +} + +pub fn get_git_commit( + pool: &DbPool, + repository_id: i64, + sha: &str, +) -> Result> { + if repository_id <= 0 { + bail!("repository_id must be positive"); + } + if !valid_object_id(sha) { + bail!("sha must be a 40- or 64-byte hex object ID"); + } + let conn = pool.get().context("acquire database connection")?; + commit_by_sha(&conn, repository_id, sha) +} + +pub fn list_git_commits(pool: &DbPool, repository_id: i64) -> Result> { + if repository_id <= 0 { + bail!("repository_id must be positive"); + } + let conn = pool.get().context("acquire database connection")?; + let sql = format!( + "SELECT {COMMIT_COLUMNS} FROM git_commits + WHERE repository_id = ?1 ORDER BY id" + ); + conn.prepare(&sql)? + .query_map([repository_id], commit_row)? + .collect::>() + .context("list Git commits") +} + +#[cfg(test)] +#[path = "agent_observatory_commits_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_commits_tests.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_commits_tests.rs new file mode 100644 index 00000000..0b06968c --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_commits_tests.rs @@ -0,0 +1,192 @@ +use super::{ + GitCommitReachabilityUpdate, GitCommitUpsert, get_git_commit, list_git_commits, + reconcile_git_commits, upsert_git_commits, +}; +use crate::agent_observatory::{RepositoryUpsert, reconcile_repository}; +use crate::config::StorageConfig; +use crate::init_pool; + +const SHA_ONE: &str = "0123456789012345678901234567890123456789"; +const SHA_TWO: &str = "abcdefabcdefabcdefabcdefabcdefabcdefabcd"; + +fn repository() -> RepositoryUpsert { + RepositoryUpsert { + repository_key: "repo-key".to_string(), + hostname: "devhost".to_string(), + common_git_dir: "/workspace/cortex/.git".to_string(), + primary_path: "/workspace/cortex".to_string(), + display_name: "cortex".to_string(), + remote_url_hash: None, + metadata_json: "{}".to_string(), + } +} + +fn commit(sha: &str, parents: &str, subject: &str) -> GitCommitUpsert { + GitCommitUpsert { + sha: sha.to_string(), + parent_shas_json: parents.to_string(), + author_name: Some("Cortex Fixture".to_string()), + author_email_hash: Some("sha256:fixture".to_string()), + authored_at: Some("2026-08-04T13:00:00.000Z".to_string()), + committed_at: Some("2026-08-04T13:00:00.000Z".to_string()), + subject: subject.to_string(), + changed_files: Some(2), + insertions: Some(3), + deletions: Some(1), + changed_paths_json: r#"[{"path_hex":"7372632f6c69622e7273"}]"#.to_string(), + reachable: true, + metadata_json: r#"{"binary_files":0}"#.to_string(), + } +} + +#[test] +fn commit_upserts_preserve_identity_first_seen_order_and_exact_metadata() { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test(dir.path().join("commits.db"))).unwrap(); + let repo = reconcile_repository(&pool, &repository(), &[], "2026-08-04T13:00:00.000Z") + .unwrap() + .repository; + + let first = upsert_git_commits( + &pool, + "repo-key", + &[ + commit(SHA_ONE, "[]", "one"), + commit(SHA_TWO, &format!(r#"["{SHA_ONE}"]"#), "two"), + ], + "2026-08-04T13:01:00.000Z", + ) + .unwrap(); + assert_eq!( + first.iter().map(|row| row.sha.as_str()).collect::>(), + vec![SHA_ONE, SHA_TWO] + ); + assert_eq!(first[0].parent_shas_json, "[]"); + assert_eq!(first[1].parent_shas_json, format!(r#"["{SHA_ONE}"]"#)); + assert_eq!(first[0].changed_files, Some(2)); + assert_eq!(first[0].insertions, Some(3)); + assert_eq!(first[0].deletions, Some(1)); + assert!(first.iter().all(|row| row.reachable)); + + let first_id = first[0].id; + let first_seen = first[0].first_observed_at.clone(); + let mut enriched = commit(SHA_ONE, "[]", "one enriched"); + enriched.changed_files = Some(4); + enriched.metadata_json = r#"{"binary_files":1}"#.to_string(); + let second = + upsert_git_commits(&pool, "repo-key", &[enriched], "2026-08-04T13:02:00.000Z").unwrap(); + assert_eq!(second[0].id, first_id); + assert_eq!(second[0].first_observed_at, first_seen); + assert_eq!(second[0].last_observed_at, "2026-08-04T13:02:00.000Z"); + assert_eq!(second[0].subject, "one enriched"); + assert_eq!(second[0].changed_files, Some(4)); + + let listed = list_git_commits(&pool, repo.id).unwrap(); + assert_eq!(listed.len(), 2); + assert_eq!(listed[0].id, first_id); + assert_eq!(listed[1].sha, SHA_TWO); + assert_eq!( + get_git_commit(&pool, repo.id, SHA_ONE).unwrap().unwrap(), + second[0] + ); +} + +#[test] +fn invalid_commit_batch_rolls_back_without_partial_rows() { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test(dir.path().join("rollback.db"))).unwrap(); + let repo = reconcile_repository(&pool, &repository(), &[], "2026-08-04T13:00:00.000Z") + .unwrap() + .repository; + let mut invalid = commit(SHA_TWO, "[]", "invalid"); + invalid.changed_paths_json = "{".to_string(); + let error = upsert_git_commits( + &pool, + "repo-key", + &[commit(SHA_ONE, "[]", "valid"), invalid], + "2026-08-04T13:01:00.000Z", + ) + .unwrap_err(); + assert!(error.to_string().contains("changed_paths_json")); + assert!(list_git_commits(&pool, repo.id).unwrap().is_empty()); +} + +#[test] +fn reachability_updates_are_atomic_and_preserve_commit_history() { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test(dir.path().join("reachability.db"))).unwrap(); + let repository = reconcile_repository(&pool, &repository(), &[], "2026-08-04T14:00:00.000Z") + .unwrap() + .repository; + let initial = upsert_git_commits( + &pool, + "repo-key", + &[ + commit(SHA_ONE, "[]", "one"), + commit(SHA_TWO, &format!(r#"["{SHA_ONE}"]"#), "two"), + ], + "2026-08-04T14:01:00.000Z", + ) + .unwrap(); + let second_id = initial[1].id; + let second_first_seen = initial[1].first_observed_at.clone(); + + let updated = reconcile_git_commits( + &pool, + "repo-key", + &[], + &[GitCommitReachabilityUpdate { + sha: SHA_TWO.to_string(), + reachable: false, + }], + "2026-08-04T14:02:00.000Z", + ) + .unwrap(); + assert!(updated.is_empty()); + let unreachable = get_git_commit(&pool, repository.id, SHA_TWO) + .unwrap() + .unwrap(); + assert_eq!(unreachable.id, second_id); + assert_eq!(unreachable.first_observed_at, second_first_seen); + assert_eq!(unreachable.last_observed_at, "2026-08-04T14:02:00.000Z"); + assert!(!unreachable.reachable); + + reconcile_git_commits( + &pool, + "repo-key", + &[], + &[GitCommitReachabilityUpdate { + sha: SHA_TWO.to_string(), + reachable: true, + }], + "2026-08-04T14:03:00.000Z", + ) + .unwrap(); + assert!( + get_git_commit(&pool, repository.id, SHA_TWO) + .unwrap() + .unwrap() + .reachable + ); + + let sha_three = "3333333333333333333333333333333333333333"; + let missing = "4444444444444444444444444444444444444444"; + let error = reconcile_git_commits( + &pool, + "repo-key", + &[commit(sha_three, "[]", "three")], + &[GitCommitReachabilityUpdate { + sha: missing.to_string(), + reachable: false, + }], + "2026-08-04T14:04:00.000Z", + ) + .unwrap_err(); + assert!(error.to_string().contains(missing)); + assert!( + get_git_commit(&pool, repository.id, sha_three) + .unwrap() + .is_none() + ); + assert_eq!(list_git_commits(&pool, repository.id).unwrap().len(), 2); +} diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_observations.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_observations.rs new file mode 100644 index 00000000..6bedc206 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_observations.rs @@ -0,0 +1,304 @@ +//! Append-only repository observation persistence. + +use super::{RepositoryObservationKind, RepositoryObservationRow}; +use crate::pool::{DbPool, write_lock}; +use anyhow::{Context, Result, bail}; +use cortex_domain::observatory_identity::event_key; +use rusqlite::types::Type; +use rusqlite::{OptionalExtension, Row, Transaction, TransactionBehavior, params}; +use sha2::{Digest, Sha256}; +use std::collections::HashSet; +use std::str::FromStr; + +const OBSERVATION_COLUMNS: &str = + "id, observation_key, repository_id, worktree_id, observed_at, observation_kind, + old_head_sha, new_head_sha, summary, payload_json, created_at"; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RepositoryObservationInput { + pub worktree_key: Option, + pub observation_kind: RepositoryObservationKind, + pub new_head_sha: Option, + pub summary: String, + pub payload_json: String, +} + +fn observation_row(row: &Row<'_>) -> rusqlite::Result { + let kind: String = row.get(5)?; + let observation_kind = RepositoryObservationKind::from_str(&kind).map_err(|error| { + rusqlite::Error::FromSqlConversionFailure(5, Type::Text, Box::new(error)) + })?; + Ok(RepositoryObservationRow { + id: row.get(0)?, + observation_key: row.get(1)?, + repository_id: row.get(2)?, + worktree_id: row.get(3)?, + observed_at: row.get(4)?, + observation_kind, + old_head_sha: row.get(6)?, + new_head_sha: row.get(7)?, + summary: row.get(8)?, + payload_json: row.get(9)?, + created_at: row.get(10)?, + }) +} + +fn valid_object_id(value: &str) -> bool { + matches!(value.len(), 40 | 64) && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn validate_input(input: &RepositoryObservationInput) -> Result<()> { + if input + .worktree_key + .as_deref() + .is_some_and(|value| value.trim().is_empty()) + { + bail!("worktree_key must be non-empty when present"); + } + serde_json::from_str::(&input.payload_json) + .context("observation payload_json must be valid JSON")?; + if input.observation_kind == RepositoryObservationKind::Head { + let head = input + .new_head_sha + .as_deref() + .context("head observation requires new_head_sha")?; + if !valid_object_id(head) { + bail!("head observation new_head_sha must be a 40- or 64-byte hex object ID"); + } + } else if let Some(head) = input.new_head_sha.as_deref() + && !valid_object_id(head) + { + bail!("new_head_sha must be a 40- or 64-byte hex object ID"); + } + Ok(()) +} + +fn repository_id(tx: &Transaction<'_>, repository_key: &str) -> Result { + tx.query_row( + "SELECT id FROM repositories WHERE repository_key = ?1", + [repository_key], + |row| row.get(0), + ) + .optional()? + .with_context(|| format!("repository not found for key {repository_key}")) +} + +fn worktree_id( + tx: &Transaction<'_>, + repository_id: i64, + worktree_key: Option<&str>, +) -> Result> { + let Some(worktree_key) = worktree_key else { + return Ok(None); + }; + let row: Option<(i64, i64)> = tx + .query_row( + "SELECT id, repository_id FROM repository_worktrees WHERE worktree_key = ?1", + [worktree_key], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional()?; + let (id, owner_repository_id) = + row.with_context(|| format!("worktree not found for key {worktree_key}"))?; + if owner_repository_id != repository_id { + bail!("worktree {worktree_key} does not belong to repository"); + } + Ok(Some(id)) +} + +fn latest_observation( + tx: &Transaction<'_>, + repository_id: i64, + worktree_id: Option, + kind: RepositoryObservationKind, +) -> Result> { + let sql = format!( + "SELECT {OBSERVATION_COLUMNS} + FROM repository_observations + WHERE repository_id = ?1 + AND (worktree_id = ?2 OR (worktree_id IS NULL AND ?2 IS NULL)) + AND observation_kind = ?3 + ORDER BY id DESC + LIMIT 1" + ); + tx.query_row( + &sql, + params![repository_id, worktree_id, kind.as_str()], + observation_row, + ) + .optional() + .context("query latest repository observation") +} + +fn observation_by_key( + tx: &Transaction<'_>, + observation_key: &str, +) -> Result { + let sql = format!( + "SELECT {OBSERVATION_COLUMNS} + FROM repository_observations + WHERE observation_key = ?1" + ); + tx.query_row(&sql, [observation_key], observation_row) + .context("query inserted repository observation") +} + +fn hash_component(hasher: &mut Sha256, value: &str) { + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(value.as_bytes()); +} + +fn deterministic_key( + repository_key: &str, + input: &RepositoryObservationInput, + previous_key: Option<&str>, + old_head_sha: Option<&str>, +) -> Result { + let mut hasher = Sha256::new(); + for value in [ + repository_key, + input.worktree_key.as_deref().unwrap_or(""), + input.observation_kind.as_str(), + previous_key.unwrap_or(""), + old_head_sha.unwrap_or(""), + input.new_head_sha.as_deref().unwrap_or(""), + input.summary.as_str(), + input.payload_json.as_str(), + ] { + hash_component(&mut hasher, value); + } + let digest = format!("{:x}", hasher.finalize()); + event_key( + "repository_observations", + &digest, + input.observation_kind.as_str(), + ) + .context("build repository observation key") +} + +fn state_is_unchanged( + latest: &RepositoryObservationRow, + input: &RepositoryObservationInput, +) -> bool { + if matches!( + input.observation_kind, + RepositoryObservationKind::WorktreeAdded | RepositoryObservationKind::WorktreeRemoved + ) { + return false; + } + if input.observation_kind == RepositoryObservationKind::Head { + return latest.new_head_sha == input.new_head_sha; + } + latest.new_head_sha == input.new_head_sha + && latest.summary == input.summary + && latest.payload_json == input.payload_json +} + +pub fn record_repository_observations_if_changed( + pool: &DbPool, + repository_key: &str, + inputs: &[RepositoryObservationInput], + observed_at: &str, +) -> Result> { + validate_repository_observations(repository_key, inputs, observed_at)?; + let _write_guard = write_lock(); + let mut connection = pool.get().context("acquire database connection")?; + let tx = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let inserted = + record_repository_observations_if_changed_tx(&tx, repository_key, inputs, observed_at)?; + tx.commit()?; + Ok(inserted) +} + +pub(super) fn validate_repository_observations( + repository_key: &str, + inputs: &[RepositoryObservationInput], + observed_at: &str, +) -> Result<()> { + if repository_key.trim().is_empty() { + bail!("repository_key must be non-empty"); + } + chrono::DateTime::parse_from_rfc3339(observed_at) + .with_context(|| format!("invalid observed_at: {observed_at}"))?; + let mut identities = HashSet::new(); + for input in inputs { + validate_input(input)?; + if !identities.insert((input.worktree_key.as_deref(), input.observation_kind)) { + bail!("duplicate worktree/kind in observation batch"); + } + } + + Ok(()) +} + +pub(super) fn record_repository_observations_if_changed_tx( + tx: &Transaction<'_>, + repository_key: &str, + inputs: &[RepositoryObservationInput], + observed_at: &str, +) -> Result> { + let repository_id = repository_id(tx, repository_key)?; + let mut inserted = Vec::new(); + + for input in inputs { + let worktree_id = worktree_id(tx, repository_id, input.worktree_key.as_deref())?; + let latest = latest_observation(tx, repository_id, worktree_id, input.observation_kind)?; + if latest + .as_ref() + .is_some_and(|row| state_is_unchanged(row, input)) + { + continue; + } + let old_head_sha = (input.observation_kind == RepositoryObservationKind::Head) + .then(|| latest.as_ref().and_then(|row| row.new_head_sha.clone())) + .flatten(); + let key = deterministic_key( + repository_key, + input, + latest.as_ref().map(|row| row.observation_key.as_str()), + old_head_sha.as_deref(), + )?; + tx.execute( + "INSERT INTO repository_observations + (observation_key, repository_id, worktree_id, observed_at, + observation_kind, old_head_sha, new_head_sha, summary, payload_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9)", + params![ + key, + repository_id, + worktree_id, + observed_at, + input.observation_kind.as_str(), + old_head_sha, + input.new_head_sha, + input.summary, + input.payload_json, + ], + )?; + inserted.push(observation_by_key(tx, &key)?); + } + + Ok(inserted) +} + +pub fn list_repository_observations( + pool: &DbPool, + repository_id: i64, +) -> Result> { + let connection = pool.get().context("acquire database connection")?; + let sql = format!( + "SELECT {OBSERVATION_COLUMNS} + FROM repository_observations + WHERE repository_id = ?1 + ORDER BY id" + ); + connection + .prepare(&sql)? + .query_map([repository_id], observation_row)? + .collect::>() + .context("list repository observations") +} + +#[cfg(test)] +#[path = "agent_observatory_observations_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_observations_tests.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_observations_tests.rs new file mode 100644 index 00000000..52d9b48b --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_observations_tests.rs @@ -0,0 +1,223 @@ +use super::{ + RepositoryObservationInput, list_repository_observations, + record_repository_observations_if_changed, +}; +use crate::agent_observatory::{ + RepositoryObservationKind, RepositoryUpsert, RepositoryWorktreeUpsert, reconcile_repository, +}; +use crate::config::StorageConfig; +use crate::init_pool; +use std::collections::HashSet; + +const SHA_ONE: &str = "0123456789012345678901234567890123456789"; +const SHA_TWO: &str = "abcdefabcdefabcdefabcdefabcdefabcdefabcd"; + +fn repository() -> RepositoryUpsert { + RepositoryUpsert { + repository_key: "repo-key".to_string(), + hostname: "devhost".to_string(), + common_git_dir: "/workspace/cortex/.git".to_string(), + primary_path: "/workspace/cortex".to_string(), + display_name: "cortex".to_string(), + remote_url_hash: None, + metadata_json: r#"{"source":"test"}"#.to_string(), + } +} + +fn worktree() -> RepositoryWorktreeUpsert { + RepositoryWorktreeUpsert { + worktree_key: "worktree-key".to_string(), + hostname: "devhost".to_string(), + path: "/workspace/cortex".to_string(), + git_dir: "/workspace/cortex/.git".to_string(), + branch_ref: Some("refs/heads/main".to_string()), + branch_name: Some("main".to_string()), + head_sha: Some(SHA_ONE.to_string()), + upstream_ref: None, + detached: false, + bare: false, + locked: false, + lock_reason: None, + prunable: false, + prune_reason: None, + dirty: false, + staged_count: 0, + unstaged_count: 0, + untracked_count: 0, + ahead: None, + behind: None, + status_hash: Some("status-one".to_string()), + } +} + +fn input( + worktree_key: Option<&str>, + observation_kind: RepositoryObservationKind, + new_head_sha: Option<&str>, + summary: &str, + payload_json: &str, +) -> RepositoryObservationInput { + RepositoryObservationInput { + worktree_key: worktree_key.map(str::to_string), + observation_kind, + new_head_sha: new_head_sha.map(str::to_string), + summary: summary.to_string(), + payload_json: payload_json.to_string(), + } +} + +#[test] +fn observation_batch_records_only_state_changes_and_chains_head_transitions() { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test(dir.path().join("observations.db"))).unwrap(); + let topology = reconcile_repository( + &pool, + &repository(), + &[worktree()], + "2026-08-03T12:00:00.000Z", + ) + .unwrap(); + + let initial = vec![ + input( + None, + RepositoryObservationKind::Discovered, + None, + "repository discovered", + r#"{"primary_path":"/workspace/cortex"}"#, + ), + input( + Some("worktree-key"), + RepositoryObservationKind::Status, + None, + "worktree status changed", + r#"{"dirty":false}"#, + ), + input( + Some("worktree-key"), + RepositoryObservationKind::Head, + Some(SHA_ONE), + "worktree HEAD changed", + r#"{"head_sha":"0123456789012345678901234567890123456789"}"#, + ), + ]; + let inserted = record_repository_observations_if_changed( + &pool, + "repo-key", + &initial, + "2026-08-03T12:00:00.000Z", + ) + .unwrap(); + assert_eq!(inserted.len(), 3); + + let unchanged = record_repository_observations_if_changed( + &pool, + "repo-key", + &initial, + "2026-08-03T12:01:00.000Z", + ) + .unwrap(); + assert!(unchanged.is_empty()); + + let changed = record_repository_observations_if_changed( + &pool, + "repo-key", + &[ + input( + Some("worktree-key"), + RepositoryObservationKind::Status, + None, + "worktree status changed", + r#"{"dirty":true}"#, + ), + input( + Some("worktree-key"), + RepositoryObservationKind::Head, + Some(SHA_TWO), + "worktree HEAD changed", + r#"{"head_sha":"abcdefabcdefabcdefabcdefabcdefabcdefabcd"}"#, + ), + ], + "2026-08-03T12:02:00.000Z", + ) + .unwrap(); + assert_eq!(changed.len(), 2); + + let reverted = record_repository_observations_if_changed( + &pool, + "repo-key", + &[input( + Some("worktree-key"), + RepositoryObservationKind::Head, + Some(SHA_ONE), + "worktree HEAD changed", + r#"{"head_sha":"0123456789012345678901234567890123456789"}"#, + )], + "2026-08-03T12:03:00.000Z", + ) + .unwrap(); + assert_eq!(reverted.len(), 1); + + let rows = list_repository_observations(&pool, topology.repository.id).unwrap(); + assert_eq!(rows.len(), 6); + assert_eq!( + rows.iter() + .map(|row| row.observation_key.as_str()) + .collect::>() + .len(), + rows.len() + ); + let heads = rows + .iter() + .filter(|row| row.observation_kind == RepositoryObservationKind::Head) + .collect::>(); + assert_eq!(heads.len(), 3); + assert_eq!(heads[0].old_head_sha, None); + assert_eq!(heads[0].new_head_sha.as_deref(), Some(SHA_ONE)); + assert_eq!(heads[1].old_head_sha.as_deref(), Some(SHA_ONE)); + assert_eq!(heads[1].new_head_sha.as_deref(), Some(SHA_TWO)); + assert_eq!(heads[2].old_head_sha.as_deref(), Some(SHA_TWO)); + assert_eq!(heads[2].new_head_sha.as_deref(), Some(SHA_ONE)); +} + +#[test] +fn invalid_observation_batch_rolls_back_every_insert() { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test(dir.path().join("rollback.db"))).unwrap(); + let topology = reconcile_repository( + &pool, + &repository(), + &[worktree()], + "2026-08-03T12:00:00.000Z", + ) + .unwrap(); + + let error = record_repository_observations_if_changed( + &pool, + "repo-key", + &[ + input( + None, + RepositoryObservationKind::Discovered, + None, + "repository discovered", + "{}", + ), + input( + Some("missing-worktree"), + RepositoryObservationKind::Status, + None, + "worktree status changed", + "{}", + ), + ], + "2026-08-03T12:01:00.000Z", + ) + .unwrap_err(); + assert!(error.to_string().contains("missing-worktree")); + assert!( + list_repository_observations(&pool, topology.repository.id) + .unwrap() + .is_empty() + ); +} diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection.rs new file mode 100644 index 00000000..a897a9c2 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection.rs @@ -0,0 +1,310 @@ +//! Atomic Agent Observatory projector persistence. + +#[path = "agent_observatory_projection_types.rs"] +mod types; +pub(super) use types::AgentProjectionWriteFault; +pub use types::{ + AgentActorRow, AgentActorUpsert, AgentProjectionOutboxInput, AgentProjectionOutboxRow, + AgentProjectionWriteInput, AgentProjectionWriteResult, AgentRunEventUpsert, AgentRunUpsert, + AgentWorktreeEvidenceUpsert, +}; + +#[path = "agent_observatory_projection_counters.rs"] +mod counters; +#[path = "agent_observatory_projection_lookup.rs"] +mod lookup; +pub use lookup::{ + AgentProjectionRunMatch, AgentProjectionWorktreeRef, find_active_projection_worktree, + find_unique_overlapping_projection_run, find_unique_projection_run_by_session, +}; +#[path = "agent_observatory_projection_refs.rs"] +mod refs; +#[path = "agent_observatory_projection_sql.rs"] +mod sql; +#[path = "agent_observatory_projection_tie_break.rs"] +mod tie_break; + +use crate::pool::{DbPool, write_lock}; +use anyhow::{Context, Result, bail}; +use cortex_domain::observatory_identity::{actor_key, canonical_tool, event_key, run_key}; +use rusqlite::TransactionBehavior; +use serde_json::Value; +use sha2::{Digest, Sha256}; + +fn required(value: &str, field: &str) -> Result<()> { + if value.trim().is_empty() { + bail!("{field} must be non-empty"); + } + Ok(()) +} + +fn timestamp(value: &str, field: &str) -> Result<()> { + chrono::DateTime::parse_from_rfc3339(value) + .with_context(|| format!("invalid {field}: {value}"))?; + Ok(()) +} + +fn optional_timestamp(value: Option<&str>, field: &str) -> Result<()> { + if let Some(value) = value { + timestamp(value, field)?; + } + Ok(()) +} + +fn json_object(value: &str, field: &str) -> Result<()> { + let parsed: Value = + serde_json::from_str(value).with_context(|| format!("{field} must be valid JSON"))?; + if !parsed.is_object() { + bail!("{field} must be a JSON object"); + } + Ok(()) +} + +fn valid_object_id(value: &str) -> bool { + matches!(value.len(), 40 | 64) && value.bytes().all(|byte| byte.is_ascii_hexdigit()) +} + +fn optional_object_id(value: Option<&str>, field: &str) -> Result<()> { + if value.is_some_and(|value| !valid_object_id(value)) { + bail!("{field} must be a 40- or 64-byte hex object ID"); + } + Ok(()) +} + +fn validate_run(input: &AgentRunUpsert) -> Result<()> { + run_key(&input.hostname, &input.tool, &input.native_session_id)?; + required(&input.status_reason, "status_reason")?; + timestamp(&input.status_observed_at, "status_observed_at")?; + timestamp(&input.started_at, "started_at")?; + timestamp(&input.last_activity_at, "last_activity_at")?; + optional_timestamp(input.ended_at.as_deref(), "ended_at")?; + if input.projection_version <= 0 { + bail!("projection_version must be positive"); + } + optional_object_id(input.start_head_sha.as_deref(), "start_head_sha")?; + optional_object_id(input.current_head_sha.as_deref(), "current_head_sha")?; + json_object(&input.freshness_json, "freshness_json")?; + json_object(&input.metadata_json, "run metadata_json")?; + for (field, value) in [ + ("parent_run_key", input.parent_run_key.as_deref()), + ("previous_run_key", input.previous_run_key.as_deref()), + ( + "primary_worktree_key", + input.primary_worktree_key.as_deref(), + ), + ] { + if value.is_some_and(|value| value.trim().is_empty()) { + bail!("{field} must be non-empty when present"); + } + } + Ok(()) +} + +fn validate_actor(input: &AgentActorUpsert) -> Result<()> { + required(&input.native_actor_id, "native_actor_id")?; + optional_timestamp(input.started_at.as_deref(), "actor started_at")?; + optional_timestamp(input.last_activity_at.as_deref(), "actor last_activity_at")?; + optional_timestamp(input.ended_at.as_deref(), "actor ended_at")?; + json_object(&input.metadata_json, "actor metadata_json") +} + +fn validate_evidence(input: &AgentWorktreeEvidenceUpsert) -> Result<()> { + required(&input.worktree_key, "evidence worktree_key")?; + required(&input.evidence_kind, "evidence_kind")?; + required(&input.evidence_source, "evidence_source")?; + if !input.confidence.is_finite() || !(0.0..=1.0).contains(&input.confidence) { + bail!("confidence must be between 0.0 and 1.0"); + } + timestamp(&input.first_seen_at, "evidence first_seen_at")?; + timestamp(&input.last_seen_at, "evidence last_seen_at")?; + json_object(&input.metadata_json, "evidence metadata_json") +} + +fn validate_event(input: &AgentRunEventUpsert) -> Result<()> { + event_key( + &input.source_kind, + &input.source_id, + &input.projection_variant, + )?; + if input + .worktree_key + .as_deref() + .is_some_and(|value| value.trim().is_empty()) + { + bail!("event worktree_key must be non-empty when present"); + } + timestamp(&input.observed_at, "event observed_at")?; + timestamp(&input.ingested_at, "event ingested_at")?; + if input.source_log_id.is_some_and(|value| value <= 0) { + bail!("source_log_id must be positive when present"); + } + if input.provider_sequence.is_some_and(|value| value < 0) { + bail!("provider_sequence must be non-negative when present"); + } + required(&input.severity, "severity")?; + json_object(&input.payload_json, "event payload_json") +} + +fn validate_outbox(input: &AgentProjectionOutboxInput) -> Result<()> { + timestamp(&input.expires_at, "outbox expires_at")?; + json_object(&input.payload_json, "outbox payload_json") +} + +fn validate_input(input: &AgentProjectionWriteInput) -> Result<()> { + validate_run(&input.run)?; + if let Some(actor) = &input.actor { + validate_actor(actor)?; + } + if let Some(evidence) = &input.worktree_evidence { + validate_evidence(evidence)?; + } + validate_event(&input.event)?; + validate_outbox(&input.outbox) +} + +fn digest_key(namespace: &str, values: &[&str]) -> String { + let mut hasher = Sha256::new(); + for value in values { + hasher.update((value.len() as u64).to_be_bytes()); + hasher.update(value.as_bytes()); + } + format!("v1:{namespace}:{:x}", hasher.finalize()) +} + +fn evidence_key(run_key: &str, input: &AgentWorktreeEvidenceUpsert) -> String { + digest_key( + "run_worktree_evidence", + &[ + run_key, + &input.worktree_key, + &input.evidence_kind, + &input.evidence_source, + ], + ) +} + +fn outbox_key(input: &AgentProjectionWriteInput) -> Result { + let bytes = serde_json::to_vec(input).context("serialize projection input fingerprint")?; + Ok(format!("v1:projection_outbox:{:x}", Sha256::digest(bytes))) +} + +fn write_inner( + pool: &DbPool, + input: &AgentProjectionWriteInput, + fault: Option, +) -> Result { + validate_input(input)?; + let canonical_tool = canonical_tool(&input.run.tool)?; + let durable_run_key = run_key( + &input.run.hostname, + &canonical_tool, + &input.run.native_session_id, + )?; + let durable_actor_key = input + .actor + .as_ref() + .map(|actor| actor_key(&durable_run_key, &actor.native_actor_id)) + .transpose()?; + let durable_event_key = event_key( + &input.event.source_kind, + &input.event.source_id, + &input.event.projection_variant, + )?; + let durable_evidence_key = input + .worktree_evidence + .as_ref() + .map(|evidence| evidence_key(&durable_run_key, evidence)); + + let _write_guard = write_lock(); + let mut connection = pool.get().context("acquire database connection")?; + let tx = connection.transaction_with_behavior(TransactionBehavior::Immediate)?; + let refs = sql::resolve_run_refs(&tx, &input.run)?; + let (mut run, run_changed) = + sql::upsert_run(&tx, &durable_run_key, &canonical_tool, &input.run, &refs)?; + + let (actor, actor_changed) = match (&input.actor, durable_actor_key.as_deref()) { + (Some(actor), Some(key)) => { + let (row, changed) = sql::upsert_actor(&tx, key, run.id, actor)?; + (Some(row), changed) + } + (None, None) => (None, false), + _ => unreachable!("actor input and key are constructed together"), + }; + + let (worktree_evidence, evidence_changed) = + match (&input.worktree_evidence, durable_evidence_key.as_deref()) { + (Some(evidence), Some(key)) => { + let worktree_id = sql::worktree_id(&tx, &evidence.worktree_key)?; + let (row, changed) = sql::upsert_evidence(&tx, key, run.id, worktree_id, evidence)?; + (Some(row), changed) + } + (None, None) => (None, false), + _ => unreachable!("evidence input and key are constructed together"), + }; + + let event_worktree_id = input + .event + .worktree_key + .as_deref() + .map(|key| sql::worktree_id(&tx, key)) + .transpose()?; + let (event, event_inserted) = sql::insert_event( + &tx, + &durable_event_key, + run.id, + actor.as_ref().map(|row| row.id), + event_worktree_id, + &input.event, + )?; + + if fault == Some(AgentProjectionWriteFault::AfterEventInsert) { + bail!("injected failure after event insert"); + } + + if event_inserted { + run = counters::apply_event_counters(&tx, run.id, &event)?; + } + let materialized_state_changed = + run_changed || actor_changed || evidence_changed || event_inserted; + let outbox = if materialized_state_changed { + Some(sql::insert_outbox( + &tx, + &outbox_key(input)?, + run.id, + &input.outbox, + )?) + } else { + None + }; + tx.commit()?; + + Ok(AgentProjectionWriteResult { + run, + actor, + worktree_evidence, + event, + event_inserted, + materialized_state_changed, + outbox, + }) +} + +pub fn write_agent_projection( + pool: &DbPool, + input: &AgentProjectionWriteInput, +) -> Result { + write_inner(pool, input, None) +} + +#[cfg(test)] +pub(super) fn write_agent_projection_with_fault( + pool: &DbPool, + input: &AgentProjectionWriteInput, + fault: AgentProjectionWriteFault, +) -> Result { + write_inner(pool, input, Some(fault)) +} + +#[cfg(test)] +#[path = "agent_observatory_projection_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_counters.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_counters.rs new file mode 100644 index 00000000..a6e2f284 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_counters.rs @@ -0,0 +1,40 @@ +//! Run counter updates for atomic Agent Observatory projection writes. + +use super::super::{AgentEventKind, AgentRunEventRow, AgentRunRow}; +use super::sql; +use anyhow::Result; +use rusqlite::{Transaction, params}; + +pub(super) fn apply_event_counters( + tx: &Transaction<'_>, + run_id: i64, + event: &AgentRunEventRow, +) -> Result { + let error_increment = i64::from(event.event_kind == AgentEventKind::Error); + tx.execute( + "UPDATE agent_runs SET last_event_id=?1, event_count=event_count+1, + error_count=error_count+?2, + first_source_log_id=CASE + WHEN ?3 IS NULL THEN first_source_log_id + WHEN first_source_log_id IS NULL OR first_source_log_id > ?3 THEN ?3 + ELSE first_source_log_id END, + last_source_log_id=CASE + WHEN ?3 IS NULL THEN last_source_log_id + WHEN last_source_log_id IS NULL OR last_source_log_id < ?3 THEN ?3 + ELSE last_source_log_id END, + last_activity_at=MAX(last_activity_at, ?4), + updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now') WHERE id=?5", + params![ + event.id, + error_increment, + event.source_log_id, + event.observed_at, + run_id + ], + )?; + sql::run_by_id(tx, run_id) +} + +#[cfg(test)] +#[path = "agent_observatory_projection_counters_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_counters_tests.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_counters_tests.rs new file mode 100644 index 00000000..49658cb5 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_counters_tests.rs @@ -0,0 +1,72 @@ +use super::super::{ + AgentProjectionOutboxInput, AgentProjectionWriteInput, AgentRunEventUpsert, AgentRunUpsert, + write_agent_projection, +}; +use crate::agent_observatory::{AgentEventKind, RunStatus, StreamEventName}; +use crate::{StorageConfig, init_pool}; + +#[test] +fn projection_write_increments_event_and_error_counters_atomically() { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig { + db_path: dir.path().join("counters.db"), + pool_size: 1, + wal_mode: false, + ..StorageConfig::default() + }) + .unwrap(); + let input = AgentProjectionWriteInput { + run: AgentRunUpsert { + native_session_id: "session".into(), + tool: "Claude".into(), + provider_tool: None, + hostname: "dookie".into(), + parent_run_key: None, + previous_run_key: None, + primary_worktree_key: None, + transcript_path: None, + process_id: None, + status: RunStatus::Active, + status_reason: "test".into(), + status_observed_at: "2026-08-18T00:00:00Z".into(), + started_at: "2026-08-18T00:00:00Z".into(), + last_activity_at: "2026-08-18T00:00:01Z".into(), + ended_at: None, + primary_branch: None, + start_head_sha: None, + current_head_sha: None, + projection_version: 1, + freshness_json: "{}".into(), + metadata_json: "{}".into(), + }, + actor: None, + worktree_evidence: None, + event: AgentRunEventUpsert { + source_kind: "ai_logs".into(), + source_id: "1".into(), + projection_variant: "error".into(), + worktree_key: None, + observed_at: "2026-08-18T00:00:01Z".into(), + ingested_at: "2026-08-18T00:00:01Z".into(), + event_kind: AgentEventKind::Error, + source_log_id: None, + provider_sequence: None, + trace_id: None, + span_id: None, + severity: "err".into(), + title: "error".into(), + summary: "error".into(), + payload_json: "{}".into(), + content_scrubbed: true, + }, + outbox: AgentProjectionOutboxInput { + event_name: StreamEventName::RunEvent, + expires_at: "2026-08-19T00:00:00Z".into(), + payload_json: "{}".into(), + }, + }; + let result = write_agent_projection(&pool, &input).unwrap(); + assert_eq!(result.run.event_count, 1); + assert_eq!(result.run.error_count, 1); + assert_eq!(result.run.last_event_id, Some(result.event.id)); +} diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_lookup.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_lookup.rs new file mode 100644 index 00000000..e2ee9586 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_lookup.rs @@ -0,0 +1,147 @@ +//! Read-side lookups used by Agent Observatory source projectors. + +use super::super::AgentRunRow; +use super::sql; +use crate::pool::DbPool; +use anyhow::{Context, Result, bail}; +use cortex_domain::observatory_identity::canonical_tool; +use rusqlite::params; +use std::path::{Component, Path}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AgentProjectionWorktreeRef { + pub id: i64, + pub worktree_key: String, + pub path: String, +} + +#[derive(Debug, Clone, PartialEq)] +pub enum AgentProjectionRunMatch { + None, + Unique(Box), + Ambiguous, +} + +fn canonical_absolute(path: &str) -> bool { + let path = Path::new(path); + path.is_absolute() + && path.components().all(|component| { + matches!( + component, + Component::Prefix(_) | Component::RootDir | Component::Normal(_) + ) + }) +} + +pub fn find_active_projection_worktree( + pool: &DbPool, + hostname: &str, + path: &str, +) -> Result> { + if hostname.trim().is_empty() { + bail!("hostname must be non-empty"); + } + if !canonical_absolute(path) { + bail!("worktree path must be canonical and absolute"); + } + let connection = pool.get().context("acquire database connection")?; + let mut statement = connection.prepare( + "SELECT id, worktree_key, path FROM repository_worktrees + WHERE hostname = ?1 + AND removed_at IS NULL + AND ( + path = ?2 + OR path = '/' + OR ( + length(?2) > length(path) + AND substr(?2, 1, length(path)) = path + AND substr(?2, length(path) + 1, 1) = '/' + ) + ) + ORDER BY length(path) DESC, id + LIMIT 1", + )?; + let rows = statement + .query_map(params![hostname.trim(), path], |row| { + Ok(AgentProjectionWorktreeRef { + id: row.get(0)?, + worktree_key: row.get(1)?, + path: row.get(2)?, + }) + })? + .collect::>>()?; + Ok(rows.into_iter().next()) +} + +pub fn find_unique_overlapping_projection_run( + pool: &DbPool, + hostname: &str, + observed_at: &str, +) -> Result { + if hostname.trim().is_empty() { + bail!("hostname must be non-empty"); + } + chrono::DateTime::parse_from_rfc3339(observed_at) + .with_context(|| format!("invalid observed_at: {observed_at}"))?; + let connection = pool.get().context("acquire database connection")?; + let mut statement = connection.prepare( + "SELECT id FROM agent_runs + WHERE hostname = ?1 + AND started_at <= ?2 + AND last_activity_at >= ?2 + AND status IN ('starting','active','waiting','idle','stale') + ORDER BY id LIMIT 2", + )?; + let ids = statement + .query_map(params![hostname.trim(), observed_at], |row| { + row.get::<_, i64>(0) + })? + .collect::>>()?; + match ids.as_slice() { + [] => Ok(AgentProjectionRunMatch::None), + [id] => Ok(AgentProjectionRunMatch::Unique(Box::new(sql::run_by_id( + &connection, + *id, + )?))), + _ => Ok(AgentProjectionRunMatch::Ambiguous), + } +} + +pub fn find_unique_projection_run_by_session( + pool: &DbPool, + tool: &str, + session_id: &str, +) -> Result { + if tool.trim().is_empty() { + bail!("tool must be non-empty"); + } + if session_id.trim().is_empty() { + bail!("session_id must be non-empty"); + } + let canonical_tool = canonical_tool(tool)?; + let connection = pool.get().context("acquire database connection")?; + let mut statement = connection.prepare( + "SELECT id FROM agent_runs + WHERE tool = ?1 + AND native_session_id = ?2 + AND status IN ('starting','active','waiting','idle','stale') + ORDER BY id LIMIT 2", + )?; + let ids = statement + .query_map(params![canonical_tool, session_id.trim()], |row| { + row.get::<_, i64>(0) + })? + .collect::>>()?; + match ids.as_slice() { + [] => Ok(AgentProjectionRunMatch::None), + [id] => Ok(AgentProjectionRunMatch::Unique(Box::new(sql::run_by_id( + &connection, + *id, + )?))), + _ => Ok(AgentProjectionRunMatch::Ambiguous), + } +} + +#[cfg(test)] +#[path = "agent_observatory_projection_lookup_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_lookup_tests.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_lookup_tests.rs new file mode 100644 index 00000000..2ef65190 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_lookup_tests.rs @@ -0,0 +1,9 @@ +use super::*; + +#[test] +fn canonical_worktree_paths_must_be_absolute_without_parent_segments() { + assert!(canonical_absolute("/workspace/soma")); + assert!(canonical_absolute("/workspace/./soma")); + assert!(!canonical_absolute("workspace/soma")); + assert!(!canonical_absolute("/workspace/../soma")); +} diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_refs.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_refs.rs new file mode 100644 index 00000000..0fc460fa --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_refs.rs @@ -0,0 +1,50 @@ +//! Foreign-key resolution for Agent Observatory projection writes. + +use super::sql::run_id; +use super::types::AgentRunUpsert; +use anyhow::{Context, Result}; +use rusqlite::{OptionalExtension, Transaction}; + +pub(super) struct RunRefs { + pub parent_run_id: Option, + pub previous_run_id: Option, + pub primary_worktree_id: Option, +} + +fn required_run_id(tx: &Transaction<'_>, key: &str) -> Result { + run_id(tx, key)?.with_context(|| format!("run not found for key {key}")) +} + +pub(super) fn worktree_id(tx: &Transaction<'_>, key: &str) -> Result { + tx.query_row( + "SELECT id FROM repository_worktrees WHERE worktree_key = ?1 AND removed_at IS NULL", + [key], + |row| row.get(0), + ) + .optional()? + .with_context(|| format!("worktree not found for key {key}")) +} + +pub(super) fn resolve_run_refs(tx: &Transaction<'_>, input: &AgentRunUpsert) -> Result { + Ok(RunRefs { + parent_run_id: input + .parent_run_key + .as_deref() + .map(|key| required_run_id(tx, key)) + .transpose()?, + previous_run_id: input + .previous_run_key + .as_deref() + .map(|key| required_run_id(tx, key)) + .transpose()?, + primary_worktree_id: input + .primary_worktree_key + .as_deref() + .map(|key| worktree_id(tx, key)) + .transpose()?, + }) +} + +#[cfg(test)] +#[path = "agent_observatory_projection_refs_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_refs_tests.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_refs_tests.rs new file mode 100644 index 00000000..12659a68 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_refs_tests.rs @@ -0,0 +1,20 @@ +use super::*; +use crate::{StorageConfig, init_pool}; + +#[test] +fn missing_worktree_reference_returns_actionable_error() { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig { + db_path: dir.path().join("refs.db"), + pool_size: 1, + wal_mode: false, + ..StorageConfig::default() + }) + .unwrap(); + let mut conn = pool.get().unwrap(); + let tx = conn.transaction().unwrap(); + let error = worktree_id(&tx, "missing-worktree") + .unwrap_err() + .to_string(); + assert!(error.contains("worktree not found for key missing-worktree")); +} diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_sql.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_sql.rs new file mode 100644 index 00000000..365ab500 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_sql.rs @@ -0,0 +1,505 @@ +//! SQL primitives for one atomic Agent Observatory projection write. + +use super::super::{AgentRunEventRow, AgentRunRow, AgentRunWorktreeEvidenceRow}; +use anyhow::{Context, Result, bail}; +use rusqlite::types::Type; +use rusqlite::{Connection, OptionalExtension, Row, Transaction, params}; +use std::str::FromStr; + +pub(super) use super::refs::{RunRefs, resolve_run_refs, worktree_id}; +use super::types::{ + AgentActorRow, AgentActorUpsert, AgentProjectionOutboxInput, AgentProjectionOutboxRow, + AgentRunEventUpsert, AgentRunUpsert, AgentWorktreeEvidenceUpsert, +}; + +const RUN_COLUMNS: &str = "id, run_key, native_session_id, tool, provider_tool, hostname, + parent_run_id, previous_run_id, primary_worktree_id, transcript_path, process_id, status, + status_reason, status_observed_at, started_at, last_activity_at, ended_at, + first_source_log_id, last_source_log_id, last_event_id, event_count, error_count, + primary_branch, start_head_sha, current_head_sha, projection_version, freshness_json, + metadata_json, created_at, updated_at"; +const ACTOR_COLUMNS: &str = "id, actor_key, run_id, native_actor_id, actor_type, display_name, + started_at, last_activity_at, ended_at, metadata_json"; +const EVIDENCE_COLUMNS: &str = "id, relation_key, run_id, worktree_id, evidence_kind, + evidence_source, trust_level, confidence, is_primary, first_seen_at, last_seen_at, + metadata_json"; +const EVENT_COLUMNS: &str = "id, event_key, run_id, actor_id, worktree_id, commit_id, + observed_at, ingested_at, event_kind, source_kind, source_id, source_log_id, + provider_sequence, trace_id, span_id, severity, title, summary, payload_json, + content_scrubbed, created_at"; +const OUTBOX_COLUMNS: &str = "id, outbox_key, run_id, stream_event_type, expires_at, + payload_json, created_at"; + +fn enum_value(row: &Row<'_>, index: usize, _name: &'static str) -> rusqlite::Result +where + T: FromStr, + T::Err: std::error::Error + Send + Sync + 'static, +{ + let value: String = row.get(index)?; + value.parse().map_err(|error| { + rusqlite::Error::FromSqlConversionFailure(index, Type::Text, Box::new(error)) + }) +} + +fn run_row(row: &Row<'_>) -> rusqlite::Result { + Ok(AgentRunRow { + id: row.get(0)?, + run_key: row.get(1)?, + native_session_id: row.get(2)?, + tool: row.get(3)?, + provider_tool: row.get(4)?, + hostname: row.get(5)?, + parent_run_id: row.get(6)?, + previous_run_id: row.get(7)?, + primary_worktree_id: row.get(8)?, + transcript_path: row.get(9)?, + process_id: row.get(10)?, + status: enum_value(row, 11, "status")?, + status_reason: row.get(12)?, + status_observed_at: row.get(13)?, + started_at: row.get(14)?, + last_activity_at: row.get(15)?, + ended_at: row.get(16)?, + first_source_log_id: row.get(17)?, + last_source_log_id: row.get(18)?, + last_event_id: row.get(19)?, + event_count: row.get(20)?, + error_count: row.get(21)?, + primary_branch: row.get(22)?, + start_head_sha: row.get(23)?, + current_head_sha: row.get(24)?, + projection_version: row.get(25)?, + freshness_json: row.get(26)?, + metadata_json: row.get(27)?, + created_at: row.get(28)?, + updated_at: row.get(29)?, + }) +} + +fn actor_row(row: &Row<'_>) -> rusqlite::Result { + Ok(AgentActorRow { + id: row.get(0)?, + actor_key: row.get(1)?, + run_id: row.get(2)?, + native_actor_id: row.get(3)?, + actor_type: row.get(4)?, + display_name: row.get(5)?, + started_at: row.get(6)?, + last_activity_at: row.get(7)?, + ended_at: row.get(8)?, + metadata_json: row.get(9)?, + }) +} + +fn evidence_row(row: &Row<'_>) -> rusqlite::Result { + Ok(AgentRunWorktreeEvidenceRow { + id: row.get(0)?, + relation_key: row.get(1)?, + run_id: row.get(2)?, + worktree_id: row.get(3)?, + evidence_kind: row.get(4)?, + evidence_source: row.get(5)?, + trust_level: enum_value(row, 6, "trust_level")?, + confidence: row.get(7)?, + is_primary: row.get(8)?, + first_seen_at: row.get(9)?, + last_seen_at: row.get(10)?, + metadata_json: row.get(11)?, + }) +} + +fn event_row(row: &Row<'_>) -> rusqlite::Result { + Ok(AgentRunEventRow { + id: row.get(0)?, + event_key: row.get(1)?, + run_id: row.get(2)?, + actor_id: row.get(3)?, + worktree_id: row.get(4)?, + commit_id: row.get(5)?, + observed_at: row.get(6)?, + ingested_at: row.get(7)?, + event_kind: enum_value(row, 8, "event_kind")?, + source_kind: row.get(9)?, + source_id: row.get(10)?, + source_log_id: row.get(11)?, + provider_sequence: row.get(12)?, + trace_id: row.get(13)?, + span_id: row.get(14)?, + severity: row.get(15)?, + title: row.get(16)?, + summary: row.get(17)?, + payload_json: row.get(18)?, + content_scrubbed: row.get(19)?, + created_at: row.get(20)?, + }) +} + +fn outbox_row(row: &Row<'_>) -> rusqlite::Result { + Ok(AgentProjectionOutboxRow { + id: row.get(0)?, + outbox_key: row.get(1)?, + run_id: row.get(2)?, + event_name: enum_value(row, 3, "stream_event_type")?, + expires_at: row.get(4)?, + payload_json: row.get(5)?, + created_at: row.get(6)?, + }) +} + +pub(super) fn run_id(tx: &Transaction<'_>, key: &str) -> Result> { + tx.query_row( + "SELECT id FROM agent_runs WHERE run_key = ?1", + [key], + |row| row.get(0), + ) + .optional() + .context("query run ID") +} + +pub(super) fn upsert_run( + tx: &Transaction<'_>, + key: &str, + canonical_tool: &str, + input: &AgentRunUpsert, + refs: &RunRefs, +) -> Result<(AgentRunRow, bool)> { + let select_sql = format!("SELECT {RUN_COLUMNS} FROM agent_runs WHERE run_key = ?1"); + let existing = tx.query_row(&select_sql, [key], run_row).optional()?; + let activity_wins = existing.as_ref().is_none_or(|row| { + super::tie_break::run_activity_wins(input, refs.primary_worktree_id, row) + }); + let status_wins = existing + .as_ref() + .is_none_or(|row| super::tie_break::run_status_wins(input, row)); + tx.execute( + "INSERT INTO agent_runs + (run_key, native_session_id, tool, provider_tool, hostname, parent_run_id, + previous_run_id, primary_worktree_id, transcript_path, process_id, status, + status_reason, status_observed_at, started_at, last_activity_at, ended_at, + primary_branch, start_head_sha, current_head_sha, projection_version, + freshness_json, metadata_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, + ?15, ?16, ?17, ?18, ?19, ?20, ?21, ?22) + ON CONFLICT(run_key) DO UPDATE SET + provider_tool=CASE + WHEN ?23 + THEN COALESCE(excluded.provider_tool, agent_runs.provider_tool) + ELSE agent_runs.provider_tool END, + parent_run_id=COALESCE(agent_runs.parent_run_id, excluded.parent_run_id), + previous_run_id=COALESCE(agent_runs.previous_run_id, excluded.previous_run_id), + primary_worktree_id=CASE + WHEN ?23 + THEN COALESCE(excluded.primary_worktree_id, agent_runs.primary_worktree_id) + ELSE agent_runs.primary_worktree_id END, + transcript_path=CASE + WHEN ?23 + THEN COALESCE(excluded.transcript_path, agent_runs.transcript_path) + ELSE agent_runs.transcript_path END, + process_id=CASE + WHEN ?23 + THEN COALESCE(excluded.process_id, agent_runs.process_id) + ELSE agent_runs.process_id END, + status=CASE + WHEN ?24 + THEN excluded.status ELSE agent_runs.status END, + status_reason=CASE + WHEN ?24 + THEN excluded.status_reason ELSE agent_runs.status_reason END, + status_observed_at=MAX(agent_runs.status_observed_at, excluded.status_observed_at), + started_at=MIN(agent_runs.started_at, excluded.started_at), + last_activity_at=MAX(agent_runs.last_activity_at, excluded.last_activity_at), + ended_at=CASE + WHEN excluded.ended_at IS NULL THEN agent_runs.ended_at + WHEN agent_runs.ended_at IS NULL THEN excluded.ended_at + ELSE MAX(agent_runs.ended_at, excluded.ended_at) END, + primary_branch=CASE + WHEN ?23 + THEN COALESCE(excluded.primary_branch, agent_runs.primary_branch) + ELSE agent_runs.primary_branch END, + start_head_sha=COALESCE(agent_runs.start_head_sha, excluded.start_head_sha), + current_head_sha=CASE + WHEN ?23 + THEN COALESCE(excluded.current_head_sha, agent_runs.current_head_sha) + ELSE agent_runs.current_head_sha END, + projection_version=MAX(agent_runs.projection_version, excluded.projection_version), + freshness_json=CASE + WHEN ?23 + THEN excluded.freshness_json ELSE agent_runs.freshness_json END, + metadata_json=CASE + WHEN ?23 + THEN excluded.metadata_json ELSE agent_runs.metadata_json END, + updated_at=strftime('%Y-%m-%dT%H:%M:%fZ','now') + WHERE (?23 AND ( + agent_runs.provider_tool IS NOT + COALESCE(excluded.provider_tool, agent_runs.provider_tool) + OR agent_runs.primary_worktree_id IS NOT + COALESCE(excluded.primary_worktree_id, agent_runs.primary_worktree_id) + OR agent_runs.transcript_path IS NOT + COALESCE(excluded.transcript_path, agent_runs.transcript_path) + OR agent_runs.process_id IS NOT + COALESCE(excluded.process_id, agent_runs.process_id) + OR agent_runs.primary_branch IS NOT + COALESCE(excluded.primary_branch, agent_runs.primary_branch) + OR agent_runs.current_head_sha IS NOT + COALESCE(excluded.current_head_sha, agent_runs.current_head_sha) + OR agent_runs.freshness_json IS NOT excluded.freshness_json + OR agent_runs.metadata_json IS NOT excluded.metadata_json)) + OR (?24 AND ( + agent_runs.status IS NOT excluded.status + OR agent_runs.status_reason IS NOT excluded.status_reason + OR agent_runs.status_observed_at IS NOT excluded.status_observed_at)) + OR (agent_runs.parent_run_id IS NULL AND excluded.parent_run_id IS NOT NULL) + OR (agent_runs.previous_run_id IS NULL AND excluded.previous_run_id IS NOT NULL) + OR agent_runs.started_at > excluded.started_at + OR agent_runs.last_activity_at < excluded.last_activity_at + OR (excluded.ended_at IS NOT NULL AND ( + agent_runs.ended_at IS NULL OR agent_runs.ended_at < excluded.ended_at)) + OR (agent_runs.start_head_sha IS NULL AND excluded.start_head_sha IS NOT NULL) + OR agent_runs.projection_version < excluded.projection_version", + params![ + key, + input.native_session_id, + canonical_tool, + input.provider_tool, + input.hostname, + refs.parent_run_id, + refs.previous_run_id, + refs.primary_worktree_id, + input.transcript_path, + input.process_id, + input.status.as_str(), + input.status_reason, + input.status_observed_at, + input.started_at, + input.last_activity_at, + input.ended_at, + input.primary_branch, + input.start_head_sha, + input.current_head_sha, + input.projection_version, + input.freshness_json, + input.metadata_json, + activity_wins, + status_wins + ], + )?; + let changed = tx.changes() > 0; + let row = tx.query_row(&select_sql, [key], run_row)?; + if row.hostname != input.hostname + || row.tool != canonical_tool + || row.native_session_id != input.native_session_id + { + bail!("run identity conflict for key {key}"); + } + Ok((row, changed)) +} + +pub(super) fn upsert_actor( + tx: &Transaction<'_>, + key: &str, + run_id: i64, + input: &AgentActorUpsert, +) -> Result<(AgentActorRow, bool)> { + let select_sql = format!("SELECT {ACTOR_COLUMNS} FROM agent_run_actors WHERE actor_key = ?1"); + let existing = tx.query_row(&select_sql, [key], actor_row).optional()?; + let actor_wins = existing + .as_ref() + .is_none_or(|row| super::tie_break::actor_wins(input, row)); + tx.execute( + "INSERT INTO agent_run_actors + (actor_key, run_id, native_actor_id, actor_type, display_name, started_at, + last_activity_at, ended_at, metadata_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9) + ON CONFLICT(actor_key) DO UPDATE SET + actor_type=CASE WHEN ?10 THEN excluded.actor_type ELSE agent_run_actors.actor_type END, + display_name=CASE WHEN ?10 THEN excluded.display_name ELSE agent_run_actors.display_name END, + started_at=CASE WHEN agent_run_actors.started_at IS NULL THEN excluded.started_at WHEN excluded.started_at IS NULL THEN agent_run_actors.started_at ELSE MIN(agent_run_actors.started_at, excluded.started_at) END, + last_activity_at=CASE WHEN agent_run_actors.last_activity_at IS NULL THEN excluded.last_activity_at WHEN excluded.last_activity_at IS NULL THEN agent_run_actors.last_activity_at ELSE MAX(agent_run_actors.last_activity_at, excluded.last_activity_at) END, + ended_at=CASE WHEN agent_run_actors.ended_at IS NULL THEN excluded.ended_at WHEN excluded.ended_at IS NULL THEN agent_run_actors.ended_at ELSE MAX(agent_run_actors.ended_at, excluded.ended_at) END, + metadata_json=CASE WHEN ?10 THEN excluded.metadata_json ELSE agent_run_actors.metadata_json END + WHERE (?10 AND ( + agent_run_actors.actor_type IS NOT excluded.actor_type + OR agent_run_actors.display_name IS NOT excluded.display_name + OR agent_run_actors.metadata_json IS NOT excluded.metadata_json)) + OR (excluded.started_at IS NOT NULL AND ( + agent_run_actors.started_at IS NULL OR agent_run_actors.started_at > excluded.started_at)) + OR (excluded.last_activity_at IS NOT NULL AND ( + agent_run_actors.last_activity_at IS NULL OR agent_run_actors.last_activity_at < excluded.last_activity_at)) + OR (excluded.ended_at IS NOT NULL AND ( + agent_run_actors.ended_at IS NULL OR agent_run_actors.ended_at < excluded.ended_at))", + params![ + key, + run_id, + input.native_actor_id, + input.actor_type, + input.display_name, + input.started_at, + input.last_activity_at, + input.ended_at, + input.metadata_json, + actor_wins + ], + )?; + let changed = tx.changes() > 0; + let row = tx.query_row(&select_sql, [key], actor_row)?; + if row.run_id != run_id || row.native_actor_id != input.native_actor_id { + bail!("actor identity conflict for key {key}"); + } + Ok((row, changed)) +} + +pub(super) fn upsert_evidence( + tx: &Transaction<'_>, + key: &str, + run_id: i64, + worktree_id: i64, + input: &AgentWorktreeEvidenceUpsert, +) -> Result<(AgentRunWorktreeEvidenceRow, bool)> { + let select_sql = + format!("SELECT {EVIDENCE_COLUMNS} FROM agent_run_worktrees WHERE relation_key = ?1"); + let existing = tx.query_row(&select_sql, [key], evidence_row).optional()?; + let evidence_wins = existing + .as_ref() + .is_none_or(|row| super::tie_break::evidence_wins(input, row)); + tx.execute( + "INSERT INTO agent_run_worktrees + (relation_key, run_id, worktree_id, evidence_kind, evidence_source, trust_level, + confidence, is_primary, first_seen_at, last_seen_at, metadata_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11) + ON CONFLICT(relation_key) DO UPDATE SET + trust_level=CASE WHEN ?12 THEN excluded.trust_level ELSE agent_run_worktrees.trust_level END, + confidence=CASE WHEN ?12 THEN excluded.confidence ELSE agent_run_worktrees.confidence END, + is_primary=CASE WHEN ?12 THEN excluded.is_primary ELSE agent_run_worktrees.is_primary END, + first_seen_at=MIN(agent_run_worktrees.first_seen_at, excluded.first_seen_at), + last_seen_at=MAX(agent_run_worktrees.last_seen_at, excluded.last_seen_at), + metadata_json=CASE WHEN ?12 THEN excluded.metadata_json ELSE agent_run_worktrees.metadata_json END + WHERE (?12 AND ( + agent_run_worktrees.trust_level IS NOT excluded.trust_level + OR agent_run_worktrees.confidence IS NOT excluded.confidence + OR agent_run_worktrees.is_primary IS NOT excluded.is_primary + OR agent_run_worktrees.metadata_json IS NOT excluded.metadata_json)) + OR agent_run_worktrees.first_seen_at > excluded.first_seen_at + OR agent_run_worktrees.last_seen_at < excluded.last_seen_at", + params![ + key, + run_id, + worktree_id, + input.evidence_kind, + input.evidence_source, + input.trust_level.as_str(), + input.confidence, + input.is_primary, + input.first_seen_at, + input.last_seen_at, + input.metadata_json, + evidence_wins + ], + )?; + let changed = tx.changes() > 0; + let row = tx.query_row(&select_sql, [key], evidence_row)?; + if row.run_id != run_id + || row.worktree_id != worktree_id + || row.evidence_kind != input.evidence_kind + || row.evidence_source != input.evidence_source + { + bail!("worktree evidence identity conflict for key {key}"); + } + Ok((row, changed)) +} + +pub(super) fn insert_event( + tx: &Transaction<'_>, + key: &str, + run_id: i64, + actor_id: Option, + worktree_id: Option, + input: &AgentRunEventUpsert, +) -> Result<(AgentRunEventRow, bool)> { + tx.execute( + "INSERT INTO agent_run_events + (event_key, run_id, actor_id, worktree_id, observed_at, ingested_at, event_kind, + source_kind, source_id, source_log_id, provider_sequence, trace_id, span_id, + severity, title, summary, payload_json, content_scrubbed) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, + ?15, ?16, ?17, ?18) + ON CONFLICT(event_key) DO NOTHING", + params![ + key, + run_id, + actor_id, + worktree_id, + input.observed_at, + input.ingested_at, + input.event_kind.as_str(), + input.source_kind, + input.source_id, + input.source_log_id, + input.provider_sequence, + input.trace_id, + input.span_id, + input.severity, + input.title, + input.summary, + input.payload_json, + input.content_scrubbed + ], + )?; + let inserted = tx.changes() > 0; + let sql = format!("SELECT {EVENT_COLUMNS} FROM agent_run_events WHERE event_key = ?1"); + let row = tx.query_row(&sql, [key], event_row)?; + if row.run_id != run_id + || row.source_kind != input.source_kind + || row.source_id != input.source_id + || row.event_kind != input.event_kind + || row.actor_id != actor_id + || row.worktree_id != worktree_id + || row.observed_at != input.observed_at + || row.ingested_at != input.ingested_at + || row.source_log_id != input.source_log_id + || row.provider_sequence != input.provider_sequence + || row.trace_id != input.trace_id + || row.span_id != input.span_id + || row.severity != input.severity + || row.title != input.title + || row.summary != input.summary + || row.payload_json != input.payload_json + || row.content_scrubbed != input.content_scrubbed + { + bail!("event identity conflict for key {key}"); + } + Ok((row, inserted)) +} + +pub(super) fn run_by_id(connection: &Connection, run_id: i64) -> Result { + let sql = format!("SELECT {RUN_COLUMNS} FROM agent_runs WHERE id = ?1"); + connection + .query_row(&sql, [run_id], run_row) + .context("query run by ID") +} + +pub(super) fn insert_outbox( + tx: &Transaction<'_>, + key: &str, + run_id: i64, + input: &AgentProjectionOutboxInput, +) -> Result { + tx.execute( + "INSERT INTO agent_stream_outbox + (outbox_key, run_id, stream_event_type, expires_at, payload_json) + VALUES (?1, ?2, ?3, ?4, ?5) ON CONFLICT(outbox_key) DO NOTHING", + params![ + key, + run_id, + input.event_name.as_str(), + input.expires_at, + input.payload_json + ], + )?; + let sql = format!("SELECT {OUTBOX_COLUMNS} FROM agent_stream_outbox WHERE outbox_key = ?1"); + tx.query_row(&sql, [key], outbox_row) + .context("query projection outbox row") +} + +#[cfg(test)] +#[path = "agent_observatory_projection_sql_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_sql_tests.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_sql_tests.rs new file mode 100644 index 00000000..666bf4c6 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_sql_tests.rs @@ -0,0 +1,20 @@ +use super::*; +use crate::agent_observatory::RunStatus; + +#[test] +fn enum_value_maps_valid_and_invalid_database_text() { + let conn = rusqlite::Connection::open_in_memory().unwrap(); + let status = conn + .query_row("SELECT 'active'", [], |row| { + enum_value::(row, 0, "status") + }) + .unwrap(); + assert_eq!(status, RunStatus::Active); + let invalid = conn.query_row("SELECT 'not-a-status'", [], |row| { + enum_value::(row, 0, "status") + }); + assert!(matches!( + invalid, + Err(rusqlite::Error::FromSqlConversionFailure(..)) + )); +} diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_tests.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_tests.rs new file mode 100644 index 00000000..959c10b4 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_tests.rs @@ -0,0 +1,403 @@ +use super::{ + AgentActorUpsert, AgentProjectionOutboxInput, AgentProjectionWriteFault, + AgentProjectionWriteInput, AgentRunEventUpsert, AgentRunUpsert, AgentWorktreeEvidenceUpsert, + write_agent_projection, write_agent_projection_with_fault, +}; +use crate::agent_observatory::{ + AgentEventKind, EvidenceTrustLevel, GitCommitUpsert, RepositoryObservationInput, + RepositoryObservationKind, RepositoryUpsert, RepositoryWorktreeUpsert, RunStatus, + StreamEventName, get_repository_by_key, reconcile_git_repository_snapshot_with, + reconcile_repository, +}; +use crate::config::StorageConfig; +use crate::init_pool; +use cortex_domain::observatory_identity::{actor_key, event_key, run_key}; +use rusqlite::Connection; + +const STARTED_AT: &str = "2026-08-05T12:00:00.000Z"; +const EVENT_AT: &str = "2026-08-05T12:00:01.000Z"; +const EXPIRES_AT: &str = "2026-08-06T12:00:01.000Z"; +const HEAD_SHA: &str = "0123456789012345678901234567890123456789"; + +fn repository() -> RepositoryUpsert { + RepositoryUpsert { + repository_key: "repo-key".to_string(), + hostname: "devhost".to_string(), + common_git_dir: "/workspace/cortex/.git".to_string(), + primary_path: "/workspace/cortex".to_string(), + display_name: "cortex".to_string(), + remote_url_hash: None, + metadata_json: "{}".to_string(), + } +} + +fn worktree() -> RepositoryWorktreeUpsert { + RepositoryWorktreeUpsert { + worktree_key: "worktree-key".to_string(), + hostname: "devhost".to_string(), + path: "/workspace/cortex".to_string(), + git_dir: "/workspace/cortex/.git".to_string(), + branch_ref: Some("refs/heads/main".to_string()), + branch_name: Some("main".to_string()), + head_sha: Some(HEAD_SHA.to_string()), + upstream_ref: None, + detached: false, + bare: false, + locked: false, + lock_reason: None, + prunable: false, + prune_reason: None, + dirty: false, + staged_count: 0, + unstaged_count: 0, + untracked_count: 0, + ahead: None, + behind: None, + status_hash: Some("clean".to_string()), + } +} + +fn input() -> AgentProjectionWriteInput { + AgentProjectionWriteInput { + run: AgentRunUpsert { + native_session_id: "session-one".to_string(), + tool: "Claude".to_string(), + provider_tool: Some("claude-code".to_string()), + hostname: "devhost".to_string(), + parent_run_key: None, + previous_run_key: None, + primary_worktree_key: Some("worktree-key".to_string()), + transcript_path: Some("/workspace/cortex/session.jsonl".to_string()), + process_id: Some("4242".to_string()), + status: RunStatus::Active, + status_reason: "provider activity".to_string(), + status_observed_at: EVENT_AT.to_string(), + started_at: STARTED_AT.to_string(), + last_activity_at: EVENT_AT.to_string(), + ended_at: None, + primary_branch: Some("main".to_string()), + start_head_sha: Some(HEAD_SHA.to_string()), + current_head_sha: Some(HEAD_SHA.to_string()), + projection_version: 1, + freshness_json: r#"{"transcript":"fresh"}"#.to_string(), + metadata_json: r#"{"provider":"claude"}"#.to_string(), + }, + actor: Some(AgentActorUpsert { + native_actor_id: "main".to_string(), + actor_type: Some("primary".to_string()), + display_name: Some("Main agent".to_string()), + started_at: Some(STARTED_AT.to_string()), + last_activity_at: Some(EVENT_AT.to_string()), + ended_at: None, + metadata_json: "{}".to_string(), + }), + worktree_evidence: Some(AgentWorktreeEvidenceUpsert { + worktree_key: "worktree-key".to_string(), + evidence_kind: "cwd".to_string(), + evidence_source: "ai_logs:42".to_string(), + trust_level: EvidenceTrustLevel::Verified, + confidence: 1.0, + is_primary: true, + first_seen_at: EVENT_AT.to_string(), + last_seen_at: EVENT_AT.to_string(), + metadata_json: "{}".to_string(), + }), + event: AgentRunEventUpsert { + source_kind: "ai_logs".to_string(), + source_id: "42".to_string(), + projection_variant: "transcript".to_string(), + worktree_key: Some("worktree-key".to_string()), + observed_at: EVENT_AT.to_string(), + ingested_at: EVENT_AT.to_string(), + event_kind: AgentEventKind::Transcript, + source_log_id: Some(42), + provider_sequence: Some(7), + trace_id: None, + span_id: None, + severity: "info".to_string(), + title: "Assistant response".to_string(), + summary: "Projected transcript event".to_string(), + payload_json: r#"{"role":"assistant"}"#.to_string(), + content_scrubbed: true, + }, + outbox: AgentProjectionOutboxInput { + event_name: StreamEventName::RunEvent, + expires_at: EXPIRES_AT.to_string(), + payload_json: r#"{"changed":["event_count","last_activity_at"]}"#.to_string(), + }, + } +} + +fn setup() -> (tempfile::TempDir, crate::DbPool) { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test(dir.path().join("projection.db"))).unwrap(); + reconcile_repository(&pool, &repository(), &[worktree()], STARTED_AT).unwrap(); + (dir, pool) +} + +fn table_count(connection: &Connection, table: &str) -> i64 { + connection + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .unwrap() +} + +fn assert_projection_counts(pool: &crate::DbPool, expected: [i64; 5]) { + let connection = pool.get().unwrap(); + let actual = [ + table_count(&connection, "agent_runs"), + table_count(&connection, "agent_run_actors"), + table_count(&connection, "agent_run_worktrees"), + table_count(&connection, "agent_run_events"), + table_count(&connection, "agent_stream_outbox"), + ]; + assert_eq!(actual, expected); +} + +#[test] +fn injected_failure_after_event_insert_rolls_back_everything_and_retry_is_idempotent() { + let (_dir, pool) = setup(); + let input = input(); + + let error = write_agent_projection_with_fault( + &pool, + &input, + AgentProjectionWriteFault::AfterEventInsert, + ) + .unwrap_err(); + assert!(error.to_string().contains("after event insert")); + assert_projection_counts(&pool, [0, 0, 0, 0, 0]); + + let written = write_agent_projection(&pool, &input).unwrap(); + assert!(written.materialized_state_changed); + assert!(written.event_inserted); + assert_eq!(written.run.event_count, 1); + assert_eq!(written.run.error_count, 0); + assert_eq!(written.run.first_source_log_id, Some(42)); + assert_eq!(written.run.last_source_log_id, Some(42)); + assert_eq!(written.run.last_event_id, Some(written.event.id)); + assert_eq!( + written.run.run_key, + run_key("devhost", "claude", "session-one").unwrap() + ); + let actor = written.actor.as_ref().unwrap(); + assert_eq!( + actor.actor_key, + actor_key(&written.run.run_key, "main").unwrap() + ); + assert_eq!(written.event.actor_id, Some(actor.id)); + assert_eq!( + written.event.worktree_id, + Some(written.worktree_evidence.as_ref().unwrap().worktree_id) + ); + assert_eq!( + written.event.event_key, + event_key("ai_logs", "42", "transcript").unwrap() + ); + let outbox = written.outbox.as_ref().unwrap(); + assert_eq!(outbox.run_id, written.run.id); + assert_eq!(outbox.event_name, StreamEventName::RunEvent); + assert!(outbox.outbox_key.starts_with("v1:projection_outbox:")); + assert_projection_counts(&pool, [1, 1, 1, 1, 1]); + + let replay = write_agent_projection(&pool, &input).unwrap(); + assert!(!replay.materialized_state_changed); + assert!(!replay.event_inserted); + assert_eq!(replay.run.id, written.run.id); + assert_eq!(replay.actor.as_ref().unwrap().id, actor.id); + assert_eq!( + replay.worktree_evidence.as_ref().unwrap().id, + written.worktree_evidence.as_ref().unwrap().id + ); + assert_eq!(replay.event.id, written.event.id); + assert!(replay.outbox.is_none()); + assert_projection_counts(&pool, [1, 1, 1, 1, 1]); +} + +#[test] +fn material_run_update_emits_one_outbox_without_double_counting_the_event() { + let (_dir, pool) = setup(); + let initial_input = input(); + let initial = write_agent_projection(&pool, &initial_input).unwrap(); + let initial_outbox_key = initial.outbox.as_ref().unwrap().outbox_key.clone(); + + let mut updated_input = initial_input.clone(); + updated_input.run.status = RunStatus::Waiting; + updated_input.run.status_reason = "awaiting user input".to_string(); + updated_input.run.status_observed_at = "2026-08-05T12:00:10.000Z".to_string(); + updated_input.run.last_activity_at = "2026-08-05T12:00:10.000Z".to_string(); + updated_input.outbox.event_name = StreamEventName::RunStatus; + updated_input.outbox.payload_json = r#"{"status":"waiting"}"#.to_string(); + + let updated = write_agent_projection(&pool, &updated_input).unwrap(); + assert!(updated.materialized_state_changed); + assert!(!updated.event_inserted); + assert_eq!(updated.run.status, RunStatus::Waiting); + assert_eq!(updated.run.event_count, 1); + assert_eq!(updated.run.error_count, 0); + let updated_outbox = updated.outbox.as_ref().unwrap(); + assert_eq!(updated_outbox.event_name, StreamEventName::RunStatus); + assert_ne!(updated_outbox.outbox_key, initial_outbox_key); + assert_projection_counts(&pool, [1, 1, 1, 1, 2]); + + let replay = write_agent_projection(&pool, &updated_input).unwrap(); + assert!(!replay.materialized_state_changed); + assert!(!replay.event_inserted); + assert!(replay.outbox.is_none()); + assert_eq!(replay.run.event_count, 1); + assert_projection_counts(&pool, [1, 1, 1, 1, 2]); +} + +#[test] +fn missing_worktree_reference_rolls_back_run_actor_and_event() { + let (_dir, pool) = setup(); + let mut invalid = input(); + invalid.run.primary_worktree_key = Some("missing-worktree".to_string()); + + let error = write_agent_projection(&pool, &invalid).unwrap_err(); + assert!(error.to_string().contains("missing-worktree")); + assert_projection_counts(&pool, [0, 0, 0, 0, 0]); +} + +#[test] +fn reverse_replay_cannot_regress_actor_or_worktree_evidence() { + let (_dir, pool) = setup(); + let mut newest = input(); + newest.actor.as_mut().unwrap().display_name = Some("Newest actor".to_string()); + newest.actor.as_mut().unwrap().last_activity_at = Some("2026-08-05T12:05:00.000Z".to_string()); + newest.worktree_evidence.as_mut().unwrap().last_seen_at = + "2026-08-05T12:05:00.000Z".to_string(); + newest.worktree_evidence.as_mut().unwrap().trust_level = EvidenceTrustLevel::Verified; + write_agent_projection(&pool, &newest).unwrap(); + + let mut older = input(); + older.event.source_id = "43".to_string(); + older.event.source_log_id = Some(43); + older.actor.as_mut().unwrap().display_name = Some("Stale actor".to_string()); + older.actor.as_mut().unwrap().last_activity_at = Some(STARTED_AT.to_string()); + older.worktree_evidence.as_mut().unwrap().last_seen_at = STARTED_AT.to_string(); + older.worktree_evidence.as_mut().unwrap().trust_level = EvidenceTrustLevel::Inferred; + let replay = write_agent_projection(&pool, &older).unwrap(); + assert_eq!( + replay.actor.unwrap().display_name.as_deref(), + Some("Newest actor") + ); + let evidence = replay.worktree_evidence.unwrap(); + assert_eq!(evidence.last_seen_at, "2026-08-05T12:05:00.000Z"); + assert_eq!(evidence.trust_level, EvidenceTrustLevel::Verified); +} + +#[test] +fn event_key_collision_rejects_any_fingerprint_mismatch() { + let (_dir, pool) = setup(); + let original = input(); + write_agent_projection(&pool, &original).unwrap(); + let mut collision = original; + collision.event.trace_id = Some("0123456789abcdef0123456789abcdef".to_string()); + let error = write_agent_projection(&pool, &collision).unwrap_err(); + assert!(error.to_string().contains("event identity conflict")); + assert_projection_counts(&pool, [1, 1, 1, 1, 1]); +} + +#[test] +fn git_snapshot_failure_after_topology_and_commits_rolls_back_all_stages() { + let directory = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test( + directory.path().join("atomic-git.db"), + )) + .unwrap(); + let commit = GitCommitUpsert { + sha: HEAD_SHA.to_string(), + parent_shas_json: "[]".to_string(), + author_name: None, + author_email_hash: None, + authored_at: Some(STARTED_AT.to_string()), + committed_at: Some(STARTED_AT.to_string()), + subject: "atomic snapshot".to_string(), + changed_files: Some(1), + insertions: Some(1), + deletions: Some(0), + changed_paths_json: "[]".to_string(), + reachable: true, + metadata_json: "{}".to_string(), + }; + let error = reconcile_git_repository_snapshot_with( + &pool, + &repository(), + &[worktree()], + &[commit], + &[], + STARTED_AT, + |_| { + Ok(vec![RepositoryObservationInput { + worktree_key: Some("missing-worktree".to_string()), + observation_kind: RepositoryObservationKind::Status, + new_head_sha: None, + summary: "injected failure".to_string(), + payload_json: "{}".to_string(), + }]) + }, + ) + .unwrap_err(); + assert!(error.to_string().contains("missing-worktree")); + assert!(get_repository_by_key(&pool, "repo-key").unwrap().is_none()); + let connection = pool.get().unwrap(); + assert_eq!(table_count(&connection, "git_commits"), 0); + assert_eq!(table_count(&connection, "repository_observations"), 0); +} + +#[test] +fn equal_timestamp_a_b_a_replay_converges_for_all_materialized_state() { + for variant in ["run", "actor", "evidence"] { + let (_dir, pool) = setup(); + let original = input(); + write_agent_projection(&pool, &original).unwrap(); + let mut conflicting = original.clone(); + match variant { + "run" => conflicting.run.metadata_json = r#"{"provider":"other"}"#.to_string(), + "actor" => { + conflicting.actor.as_mut().unwrap().display_name = Some("Other actor".to_string()); + } + "evidence" => conflicting.worktree_evidence.as_mut().unwrap().confidence = 0.5, + _ => unreachable!(), + } + write_agent_projection(&pool, &conflicting).unwrap(); + let outbox_before = table_count(&pool.get().unwrap(), "agent_stream_outbox"); + let replay = write_agent_projection(&pool, &original).unwrap(); + assert!(!replay.materialized_state_changed, "{variant}"); + assert!(!replay.event_inserted, "{variant}"); + assert!(replay.outbox.is_none(), "{variant}"); + assert_eq!( + table_count(&pool.get().unwrap(), "agent_stream_outbox"), + outbox_before, + "{variant}" + ); + assert_eq!(table_count(&pool.get().unwrap(), "agent_run_events"), 1); + } +} + +#[test] +fn older_different_replay_is_a_true_noop_without_outbox() { + let (_dir, pool) = setup(); + let original = input(); + write_agent_projection(&pool, &original).unwrap(); + let mut older = original.clone(); + older.run.last_activity_at = STARTED_AT.to_string(); + older.run.status_observed_at = STARTED_AT.to_string(); + older.run.status = RunStatus::Waiting; + older.run.status_reason = "stale".to_string(); + older.run.metadata_json = r#"{"provider":"stale"}"#.to_string(); + older.actor.as_mut().unwrap().last_activity_at = Some(STARTED_AT.to_string()); + older.actor.as_mut().unwrap().display_name = Some("Stale actor".to_string()); + older.actor.as_mut().unwrap().metadata_json = r#"{"stale":true}"#.to_string(); + older.worktree_evidence.as_mut().unwrap().last_seen_at = STARTED_AT.to_string(); + older.worktree_evidence.as_mut().unwrap().confidence = 0.25; + older.worktree_evidence.as_mut().unwrap().metadata_json = r#"{"stale":true}"#.to_string(); + let replay = write_agent_projection(&pool, &older).unwrap(); + assert!(!replay.materialized_state_changed); + assert!(!replay.event_inserted); + assert!(replay.outbox.is_none()); + assert_eq!(replay.run.event_count, 1); + assert_projection_counts(&pool, [1, 1, 1, 1, 1]); +} diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_tie_break.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_tie_break.rs new file mode 100644 index 00000000..a101db8e --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_tie_break.rs @@ -0,0 +1,83 @@ +//! Stable total-order tie-breaks for equal-freshness projection state. + +use super::{AgentActorRow, AgentActorUpsert, AgentRunUpsert, AgentWorktreeEvidenceUpsert}; +use crate::agent_observatory::{AgentRunRow, AgentRunWorktreeEvidenceRow}; + +pub(super) fn run_activity_wins( + input: &AgentRunUpsert, + primary_worktree_id: Option, + row: &AgentRunRow, +) -> bool { + input.last_activity_at > row.last_activity_at + || (input.last_activity_at == row.last_activity_at + && format!( + "{:?}|{:?}|{:?}|{:?}|{:?}|{:?}|{}|{}", + input.provider_tool, + primary_worktree_id, + input.transcript_path, + input.process_id, + input.primary_branch, + input.current_head_sha, + input.freshness_json, + input.metadata_json + ) > format!( + "{:?}|{:?}|{:?}|{:?}|{:?}|{:?}|{}|{}", + row.provider_tool, + row.primary_worktree_id, + row.transcript_path, + row.process_id, + row.primary_branch, + row.current_head_sha, + row.freshness_json, + row.metadata_json + )) +} + +pub(super) fn run_status_wins(input: &AgentRunUpsert, row: &AgentRunRow) -> bool { + input.status_observed_at > row.status_observed_at + || (input.status_observed_at == row.status_observed_at + && format!("{}|{}", input.status.as_str(), input.status_reason) + > format!("{}|{}", row.status.as_str(), row.status_reason)) +} + +pub(super) fn actor_wins(input: &AgentActorUpsert, row: &AgentActorRow) -> bool { + let incoming_time = input + .last_activity_at + .as_ref() + .or(input.started_at.as_ref()); + let existing_time = row.last_activity_at.as_ref().or(row.started_at.as_ref()); + incoming_time > existing_time + || (incoming_time == existing_time + && format!( + "{:?}|{:?}|{}", + input.actor_type, input.display_name, input.metadata_json + ) > format!( + "{:?}|{:?}|{}", + row.actor_type, row.display_name, row.metadata_json + )) +} + +pub(super) fn evidence_wins( + input: &AgentWorktreeEvidenceUpsert, + row: &AgentRunWorktreeEvidenceRow, +) -> bool { + input.last_seen_at > row.last_seen_at + || (input.last_seen_at == row.last_seen_at + && format!( + "{}|{}|{}|{}", + input.trust_level.as_str(), + input.confidence, + input.is_primary, + input.metadata_json + ) > format!( + "{}|{}|{}|{}", + row.trust_level.as_str(), + row.confidence, + row.is_primary, + row.metadata_json + )) +} + +#[cfg(test)] +#[path = "agent_observatory_projection_tie_break_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_tie_break_tests.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_tie_break_tests.rs new file mode 100644 index 00000000..e45dc244 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_tie_break_tests.rs @@ -0,0 +1,33 @@ +use super::*; + +#[test] +fn actor_tie_break_prefers_newer_activity_then_stable_payload_order() { + let row = AgentActorRow { + id: 1, + actor_key: "actor".into(), + run_id: 1, + native_actor_id: "main".into(), + actor_type: Some("primary".into()), + display_name: Some("Agent".into()), + started_at: Some("2026-08-18T00:00:00Z".into()), + last_activity_at: Some("2026-08-18T00:01:00Z".into()), + ended_at: None, + metadata_json: "{}".into(), + }; + let newer = AgentActorUpsert { + native_actor_id: "main".into(), + actor_type: Some("primary".into()), + display_name: Some("Agent".into()), + started_at: row.started_at.clone(), + last_activity_at: Some("2026-08-18T00:02:00Z".into()), + ended_at: None, + metadata_json: "{}".into(), + }; + assert!(actor_wins(&newer, &row)); + + let older = AgentActorUpsert { + last_activity_at: Some("2026-08-18T00:00:30Z".into()), + ..newer + }; + assert!(!actor_wins(&older, &row)); +} diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_types.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_types.rs new file mode 100644 index 00000000..60251f0c --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_types.rs @@ -0,0 +1,137 @@ +//! Public inputs and materialized rows for atomic Agent Observatory projection writes. + +use super::super::{ + AgentEventKind, AgentRunEventRow, AgentRunRow, AgentRunWorktreeEvidenceRow, EvidenceTrustLevel, + RunStatus, StreamEventName, +}; +use serde::Serialize; + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct AgentRunUpsert { + pub native_session_id: String, + pub tool: String, + pub provider_tool: Option, + pub hostname: String, + pub parent_run_key: Option, + pub previous_run_key: Option, + pub primary_worktree_key: Option, + pub transcript_path: Option, + pub process_id: Option, + pub status: RunStatus, + pub status_reason: String, + pub status_observed_at: String, + pub started_at: String, + pub last_activity_at: String, + pub ended_at: Option, + pub primary_branch: Option, + pub start_head_sha: Option, + pub current_head_sha: Option, + pub projection_version: i64, + pub freshness_json: String, + pub metadata_json: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct AgentActorUpsert { + pub native_actor_id: String, + pub actor_type: Option, + pub display_name: Option, + pub started_at: Option, + pub last_activity_at: Option, + pub ended_at: Option, + pub metadata_json: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct AgentWorktreeEvidenceUpsert { + pub worktree_key: String, + pub evidence_kind: String, + pub evidence_source: String, + pub trust_level: EvidenceTrustLevel, + pub confidence: f64, + pub is_primary: bool, + pub first_seen_at: String, + pub last_seen_at: String, + pub metadata_json: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct AgentRunEventUpsert { + pub source_kind: String, + pub source_id: String, + pub projection_variant: String, + pub worktree_key: Option, + pub observed_at: String, + pub ingested_at: String, + pub event_kind: AgentEventKind, + pub source_log_id: Option, + pub provider_sequence: Option, + pub trace_id: Option, + pub span_id: Option, + pub severity: String, + pub title: String, + pub summary: String, + pub payload_json: String, + pub content_scrubbed: bool, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct AgentProjectionOutboxInput { + pub event_name: StreamEventName, + pub expires_at: String, + pub payload_json: String, +} + +#[derive(Debug, Clone, PartialEq, Serialize)] +pub struct AgentProjectionWriteInput { + pub run: AgentRunUpsert, + pub actor: Option, + pub worktree_evidence: Option, + pub event: AgentRunEventUpsert, + pub outbox: AgentProjectionOutboxInput, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct AgentActorRow { + pub id: i64, + pub actor_key: String, + pub run_id: i64, + pub native_actor_id: String, + pub actor_type: Option, + pub display_name: Option, + pub started_at: Option, + pub last_activity_at: Option, + pub ended_at: Option, + pub metadata_json: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AgentProjectionOutboxRow { + pub id: i64, + pub outbox_key: String, + pub run_id: i64, + pub event_name: StreamEventName, + pub expires_at: String, + pub payload_json: String, + pub created_at: String, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct AgentProjectionWriteResult { + pub run: AgentRunRow, + pub actor: Option, + pub worktree_evidence: Option, + pub event: AgentRunEventRow, + pub event_inserted: bool, + pub materialized_state_changed: bool, + pub outbox: Option, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) enum AgentProjectionWriteFault { + AfterEventInsert, +} + +#[cfg(test)] +#[path = "agent_observatory_projection_types_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_types_tests.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_types_tests.rs new file mode 100644 index 00000000..4818d746 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_projection_types_tests.rs @@ -0,0 +1,18 @@ +use super::*; + +#[test] +fn actor_upsert_serializes_stable_public_fields() { + let actor = AgentActorUpsert { + native_actor_id: "main".into(), + actor_type: Some("primary".into()), + display_name: Some("Main agent".into()), + started_at: Some("2026-08-18T00:00:00Z".into()), + last_activity_at: Some("2026-08-18T00:01:00Z".into()), + ended_at: None, + metadata_json: "{}".into(), + }; + let value = serde_json::to_value(actor).unwrap(); + assert_eq!(value["native_actor_id"], "main"); + assert_eq!(value["actor_type"], "primary"); + assert_eq!(value["metadata_json"], "{}"); +} diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_queries.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_queries.rs new file mode 100644 index 00000000..207378b4 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_queries.rs @@ -0,0 +1,523 @@ +//! Transactional repository and worktree persistence for Agent Observatory. + +use super::{RepositoryRow, RepositoryWorktreeRow}; +use crate::pool::{DbPool, write_lock}; +use anyhow::{Context, Result, bail}; +use rusqlite::{Connection, OptionalExtension, Row, Transaction, TransactionBehavior, params}; +use std::collections::HashSet; +use std::path::{Component, Path}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RepositoryUpsert { + pub repository_key: String, + pub hostname: String, + pub common_git_dir: String, + pub primary_path: String, + pub display_name: String, + pub remote_url_hash: Option, + pub metadata_json: String, +} + +#[allow(clippy::struct_excessive_bools)] +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct RepositoryWorktreeUpsert { + pub worktree_key: String, + pub hostname: String, + pub path: String, + pub git_dir: String, + pub branch_ref: Option, + pub branch_name: Option, + pub head_sha: Option, + pub upstream_ref: Option, + pub detached: bool, + pub bare: bool, + pub locked: bool, + pub lock_reason: Option, + pub prunable: bool, + pub prune_reason: Option, + pub dirty: bool, + pub staged_count: i64, + pub unstaged_count: i64, + pub untracked_count: i64, + pub ahead: Option, + pub behind: Option, + pub status_hash: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct RepositoryReconcileResult { + pub repository: RepositoryRow, + pub worktrees: Vec, + pub removed_worktree_ids: Vec, +} + +const REPOSITORY_BY_KEY_SQL: &str = + "SELECT id, repository_key, hostname, common_git_dir, primary_path, display_name, + remote_url_hash, first_seen_at, last_seen_at, removed_at, metadata_json, + created_at, updated_at + FROM repositories WHERE repository_key = ?1"; + +const WORKTREE_BY_KEY_SQL: &str = + "SELECT id, worktree_key, repository_id, hostname, path, git_dir, branch_ref, + branch_name, head_sha, upstream_ref, detached, bare, locked, lock_reason, + prunable, prune_reason, dirty, staged_count, unstaged_count, untracked_count, + ahead, behind, status_hash, first_seen_at, last_seen_at, removed_at, + created_at, updated_at + FROM repository_worktrees WHERE worktree_key = ?1"; + +const WORKTREE_LIST_ALL_SQL: &str = + "SELECT id, worktree_key, repository_id, hostname, path, git_dir, branch_ref, + branch_name, head_sha, upstream_ref, detached, bare, locked, lock_reason, + prunable, prune_reason, dirty, staged_count, unstaged_count, untracked_count, + ahead, behind, status_hash, first_seen_at, last_seen_at, removed_at, + created_at, updated_at + FROM repository_worktrees + WHERE repository_id = ?1 + ORDER BY path, id"; + +const WORKTREE_LIST_ACTIVE_SQL: &str = + "SELECT id, worktree_key, repository_id, hostname, path, git_dir, branch_ref, + branch_name, head_sha, upstream_ref, detached, bare, locked, lock_reason, + prunable, prune_reason, dirty, staged_count, unstaged_count, untracked_count, + ahead, behind, status_hash, first_seen_at, last_seen_at, removed_at, + created_at, updated_at + FROM repository_worktrees + WHERE repository_id = ?1 AND removed_at IS NULL + ORDER BY path, id"; + +fn repository_row(row: &Row<'_>) -> rusqlite::Result { + Ok(RepositoryRow { + id: row.get(0)?, + repository_key: row.get(1)?, + hostname: row.get(2)?, + common_git_dir: row.get(3)?, + primary_path: row.get(4)?, + display_name: row.get(5)?, + remote_url_hash: row.get(6)?, + first_seen_at: row.get(7)?, + last_seen_at: row.get(8)?, + removed_at: row.get(9)?, + metadata_json: row.get(10)?, + created_at: row.get(11)?, + updated_at: row.get(12)?, + }) +} + +fn worktree_row(row: &Row<'_>) -> rusqlite::Result { + Ok(RepositoryWorktreeRow { + id: row.get(0)?, + worktree_key: row.get(1)?, + repository_id: row.get(2)?, + hostname: row.get(3)?, + path: row.get(4)?, + git_dir: row.get(5)?, + branch_ref: row.get(6)?, + branch_name: row.get(7)?, + head_sha: row.get(8)?, + upstream_ref: row.get(9)?, + detached: row.get(10)?, + bare: row.get(11)?, + locked: row.get(12)?, + lock_reason: row.get(13)?, + prunable: row.get(14)?, + prune_reason: row.get(15)?, + dirty: row.get(16)?, + staged_count: row.get(17)?, + unstaged_count: row.get(18)?, + untracked_count: row.get(19)?, + ahead: row.get(20)?, + behind: row.get(21)?, + status_hash: row.get(22)?, + first_seen_at: row.get(23)?, + last_seen_at: row.get(24)?, + removed_at: row.get(25)?, + created_at: row.get(26)?, + updated_at: row.get(27)?, + }) +} + +fn required(value: &str, field: &str) -> Result<()> { + if value.trim().is_empty() { + bail!("{field} must be non-empty"); + } + Ok(()) +} + +fn canonical_absolute_path(value: &str, field: &str) -> Result<()> { + required(value, field)?; + let path = Path::new(value); + if !path.is_absolute() { + bail!("{field} must be an absolute canonical path"); + } + if path + .components() + .any(|component| matches!(component, Component::CurDir | Component::ParentDir)) + { + bail!("{field} must not contain dot path components"); + } + Ok(()) +} + +fn validate_repository(input: &RepositoryUpsert, observed_at: &str) -> Result<()> { + required(&input.repository_key, "repository_key")?; + required(&input.hostname, "hostname")?; + required(&input.display_name, "display_name")?; + required(observed_at, "observed_at")?; + chrono::DateTime::parse_from_rfc3339(observed_at) + .with_context(|| format!("invalid observed_at: {observed_at}"))?; + canonical_absolute_path(&input.common_git_dir, "common_git_dir")?; + canonical_absolute_path(&input.primary_path, "primary_path")?; + serde_json::from_str::(&input.metadata_json) + .context("metadata_json must be valid JSON")?; + Ok(()) +} + +fn validate_worktree( + repository: &RepositoryUpsert, + input: &RepositoryWorktreeUpsert, +) -> Result<()> { + required(&input.worktree_key, "worktree_key")?; + required(&input.hostname, "worktree hostname")?; + if input.hostname != repository.hostname { + bail!("worktree hostname must match repository hostname"); + } + canonical_absolute_path(&input.path, "worktree path")?; + canonical_absolute_path(&input.git_dir, "worktree git_dir")?; + if input.staged_count < 0 || input.unstaged_count < 0 || input.untracked_count < 0 { + bail!("worktree change counts must be non-negative"); + } + if input.ahead.is_some_and(|value| value < 0) || input.behind.is_some_and(|value| value < 0) { + bail!("worktree ahead/behind counts must be non-negative"); + } + Ok(()) +} + +fn repository_by_key(conn: &Connection, repository_key: &str) -> Result> { + conn.query_row(REPOSITORY_BY_KEY_SQL, [repository_key], repository_row) + .optional() + .context("query repository by key") +} + +fn worktree_by_key(conn: &Connection, worktree_key: &str) -> Result> { + conn.query_row(WORKTREE_BY_KEY_SQL, [worktree_key], worktree_row) + .optional() + .context("query worktree by key") +} + +fn list_worktrees( + conn: &Connection, + repository_id: i64, + include_removed: bool, +) -> Result> { + let sql = if include_removed { + WORKTREE_LIST_ALL_SQL + } else { + WORKTREE_LIST_ACTIVE_SQL + }; + conn.prepare(sql)? + .query_map([repository_id], worktree_row)? + .collect::>() + .context("list repository worktrees") +} + +fn upsert_repository_tx( + tx: &Transaction<'_>, + input: &RepositoryUpsert, + observed_at: &str, +) -> Result { + let identity: Option<(String, String)> = tx + .query_row( + "SELECT hostname, common_git_dir FROM repositories WHERE repository_key = ?1", + [&input.repository_key], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional()?; + if let Some((hostname, common_git_dir)) = identity + && (hostname != input.hostname || common_git_dir != input.common_git_dir) + { + bail!("repository identity fields cannot change"); + } + + let conflicting_key: Option = tx + .query_row( + "SELECT repository_key FROM repositories + WHERE hostname = ?1 AND common_git_dir = ?2 AND repository_key <> ?3", + params![input.hostname, input.common_git_dir, input.repository_key], + |row| row.get(0), + ) + .optional()?; + if conflicting_key.is_some() { + bail!("hostname/common_git_dir already belongs to another repository key"); + } + + tx.execute( + "INSERT INTO repositories + (repository_key, hostname, common_git_dir, primary_path, display_name, + remote_url_hash, first_seen_at, last_seen_at, removed_at, metadata_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?7, NULL, ?8) + ON CONFLICT(repository_key) DO UPDATE SET + primary_path = excluded.primary_path, + display_name = excluded.display_name, + remote_url_hash = excluded.remote_url_hash, + last_seen_at = excluded.last_seen_at, + removed_at = NULL, + metadata_json = excluded.metadata_json, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')", + params![ + input.repository_key, + input.hostname, + input.common_git_dir, + input.primary_path, + input.display_name, + input.remote_url_hash, + observed_at, + input.metadata_json, + ], + )?; + + repository_by_key(tx, &input.repository_key)?.context("repository missing after upsert") +} + +fn upsert_worktree_tx( + tx: &Transaction<'_>, + repository_id: i64, + input: &RepositoryWorktreeUpsert, + observed_at: &str, +) -> Result { + let identity: Option<(i64, String, String)> = tx + .query_row( + "SELECT repository_id, hostname, path + FROM repository_worktrees WHERE worktree_key = ?1", + [&input.worktree_key], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional()?; + if let Some((existing_repository_id, hostname, path)) = identity + && (existing_repository_id != repository_id + || hostname != input.hostname + || path != input.path) + { + bail!("worktree identity fields cannot change"); + } + + let conflicting_key: Option = tx + .query_row( + "SELECT worktree_key FROM repository_worktrees + WHERE hostname = ?1 AND path = ?2 AND worktree_key <> ?3", + params![input.hostname, input.path, input.worktree_key], + |row| row.get(0), + ) + .optional()?; + if conflicting_key.is_some() { + bail!("hostname/path already belongs to another worktree key"); + } + + tx.execute( + "INSERT INTO repository_worktrees + (worktree_key, repository_id, hostname, path, git_dir, branch_ref, branch_name, + head_sha, upstream_ref, detached, bare, locked, lock_reason, prunable, + prune_reason, dirty, staged_count, unstaged_count, untracked_count, ahead, + behind, status_hash, first_seen_at, last_seen_at, removed_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, + ?15, ?16, ?17, ?18, ?19, ?20, ?21, ?22, ?23, ?23, NULL) + ON CONFLICT(worktree_key) DO UPDATE SET + git_dir = excluded.git_dir, + branch_ref = excluded.branch_ref, + branch_name = excluded.branch_name, + head_sha = excluded.head_sha, + upstream_ref = excluded.upstream_ref, + detached = excluded.detached, + bare = excluded.bare, + locked = excluded.locked, + lock_reason = excluded.lock_reason, + prunable = excluded.prunable, + prune_reason = excluded.prune_reason, + dirty = excluded.dirty, + staged_count = excluded.staged_count, + unstaged_count = excluded.unstaged_count, + untracked_count = excluded.untracked_count, + ahead = excluded.ahead, + behind = excluded.behind, + status_hash = excluded.status_hash, + last_seen_at = excluded.last_seen_at, + removed_at = NULL, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')", + params![ + input.worktree_key, + repository_id, + input.hostname, + input.path, + input.git_dir, + input.branch_ref, + input.branch_name, + input.head_sha, + input.upstream_ref, + input.detached, + input.bare, + input.locked, + input.lock_reason, + input.prunable, + input.prune_reason, + input.dirty, + input.staged_count, + input.unstaged_count, + input.untracked_count, + input.ahead, + input.behind, + input.status_hash, + observed_at, + ], + )?; + + worktree_by_key(tx, &input.worktree_key)?.context("worktree missing after upsert") +} + +pub fn reconcile_repository( + pool: &DbPool, + repository: &RepositoryUpsert, + worktrees: &[RepositoryWorktreeUpsert], + observed_at: &str, +) -> Result { + validate_reconcile_repository(repository, worktrees, observed_at)?; + let _write_guard = write_lock(); + let mut conn = pool.get().context("acquire database connection")?; + let tx = conn.transaction_with_behavior(TransactionBehavior::Immediate)?; + let result = reconcile_repository_tx(&tx, repository, worktrees, observed_at)?; + tx.commit()?; + Ok(result) +} + +pub(super) fn validate_reconcile_repository( + repository: &RepositoryUpsert, + worktrees: &[RepositoryWorktreeUpsert], + observed_at: &str, +) -> Result<()> { + validate_repository(repository, observed_at)?; + let mut keys = HashSet::new(); + let mut paths = HashSet::new(); + for worktree in worktrees { + validate_worktree(repository, worktree)?; + if !keys.insert(worktree.worktree_key.as_str()) { + bail!("duplicate worktree key in reconciliation"); + } + if !paths.insert((worktree.hostname.as_str(), worktree.path.as_str())) { + bail!("duplicate hostname/path in reconciliation"); + } + } + + Ok(()) +} + +pub(super) fn reconcile_repository_tx( + tx: &Transaction<'_>, + repository: &RepositoryUpsert, + worktrees: &[RepositoryWorktreeUpsert], + observed_at: &str, +) -> Result { + let keys = worktrees + .iter() + .map(|worktree| worktree.worktree_key.as_str()) + .collect::>(); + let repository_row = upsert_repository_tx(tx, repository, observed_at)?; + + let mut active_worktrees = Vec::with_capacity(worktrees.len()); + for worktree in worktrees { + active_worktrees.push(upsert_worktree_tx( + tx, + repository_row.id, + worktree, + observed_at, + )?); + } + + let existing_active: Vec<(i64, String)> = tx + .prepare( + "SELECT id, worktree_key FROM repository_worktrees + WHERE repository_id = ?1 AND removed_at IS NULL + ORDER BY id", + )? + .query_map([repository_row.id], |row| Ok((row.get(0)?, row.get(1)?)))? + .collect::>()?; + let mut removed_worktree_ids = Vec::new(); + for (id, key) in existing_active { + if !keys.contains(key.as_str()) { + tx.execute( + "UPDATE repository_worktrees + SET removed_at = COALESCE(removed_at, ?2), + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + WHERE id = ?1", + params![id, observed_at], + )?; + removed_worktree_ids.push(id); + } + } + + active_worktrees.sort_by(|left, right| left.path.cmp(&right.path).then(left.id.cmp(&right.id))); + Ok(RepositoryReconcileResult { + repository: repository_row, + worktrees: active_worktrees, + removed_worktree_ids, + }) +} + +pub fn get_repository_by_key(pool: &DbPool, repository_key: &str) -> Result> { + let conn = pool.get().context("acquire database connection")?; + repository_by_key(&conn, repository_key) +} + +pub fn get_worktree_by_key( + pool: &DbPool, + worktree_key: &str, +) -> Result> { + let conn = pool.get().context("acquire database connection")?; + worktree_by_key(&conn, worktree_key) +} + +pub fn list_repository_worktrees( + pool: &DbPool, + repository_id: i64, + include_removed: bool, +) -> Result> { + let conn = pool.get().context("acquire database connection")?; + list_worktrees(&conn, repository_id, include_removed) +} + +pub fn mark_repository_removed( + pool: &DbPool, + repository_key: &str, + removed_at: &str, +) -> Result { + required(repository_key, "repository_key")?; + required(removed_at, "removed_at")?; + chrono::DateTime::parse_from_rfc3339(removed_at) + .with_context(|| format!("invalid removed_at: {removed_at}"))?; + let _write_guard = write_lock(); + let conn = pool.get().context("acquire database connection")?; + Ok(conn.execute( + "UPDATE repositories + SET removed_at = COALESCE(removed_at, ?2), + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + WHERE repository_key = ?1", + params![repository_key, removed_at], + )? > 0) +} + +pub fn mark_worktree_removed(pool: &DbPool, worktree_key: &str, removed_at: &str) -> Result { + required(worktree_key, "worktree_key")?; + required(removed_at, "removed_at")?; + chrono::DateTime::parse_from_rfc3339(removed_at) + .with_context(|| format!("invalid removed_at: {removed_at}"))?; + let _write_guard = write_lock(); + let conn = pool.get().context("acquire database connection")?; + Ok(conn.execute( + "UPDATE repository_worktrees + SET removed_at = COALESCE(removed_at, ?2), + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + WHERE worktree_key = ?1", + params![worktree_key, removed_at], + )? > 0) +} + +#[cfg(test)] +#[path = "agent_observatory_queries_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_queries_tests.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_queries_tests.rs new file mode 100644 index 00000000..00ce5254 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_queries_tests.rs @@ -0,0 +1,253 @@ +use super::{ + RepositoryUpsert, RepositoryWorktreeUpsert, get_repository_by_key, get_worktree_by_key, + list_repository_worktrees, mark_repository_removed, mark_worktree_removed, + reconcile_repository, +}; +use crate::config::StorageConfig; +use crate::init_pool; + +fn repository(key: &str, display_name: &str) -> RepositoryUpsert { + RepositoryUpsert { + repository_key: key.to_string(), + hostname: "devhost".to_string(), + common_git_dir: format!("/workspace/{key}/.git"), + primary_path: format!("/workspace/{key}"), + display_name: display_name.to_string(), + remote_url_hash: Some(format!("hash-{key}")), + metadata_json: "{\"source\":\"fixture\"}".to_string(), + } +} + +fn worktree(key: &str, path: &str, branch: &str) -> RepositoryWorktreeUpsert { + RepositoryWorktreeUpsert { + worktree_key: key.to_string(), + hostname: "devhost".to_string(), + path: path.to_string(), + git_dir: format!("{path}/.git"), + branch_ref: Some(format!("refs/heads/{branch}")), + branch_name: Some(branch.to_string()), + head_sha: Some("0123456789012345678901234567890123456789".to_string()), + upstream_ref: Some(format!("refs/remotes/origin/{branch}")), + detached: false, + bare: false, + locked: false, + lock_reason: None, + prunable: false, + prune_reason: None, + dirty: false, + staged_count: 0, + unstaged_count: 0, + untracked_count: 0, + ahead: Some(0), + behind: Some(0), + status_hash: Some(format!("status-{key}")), + } +} + +#[test] +fn reconcile_create_update_remove_and_reappear_preserves_identity_history() { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test(dir.path().join("reconcile.db"))).unwrap(); + let first_seen = "2026-08-02T16:00:00.000Z"; + let second_seen = "2026-08-02T16:01:00.000Z"; + let removed_at = "2026-08-02T16:02:00.000Z"; + let reappeared_at = "2026-08-02T16:03:00.000Z"; + + let repo = repository("cortex", "Cortex"); + let primary = worktree("cortex-main", "/workspace/cortex", "main"); + let feature = worktree( + "cortex-feature", + "/workspace/cortex/.worktrees/feature", + "feature", + ); + let created = reconcile_repository( + &pool, + &repo, + &[primary.clone(), feature.clone()], + first_seen, + ) + .unwrap(); + assert_eq!(created.worktrees.len(), 2); + assert!(created.removed_worktree_ids.is_empty()); + let repository_id = created.repository.id; + let repository_first_seen = created.repository.first_seen_at.clone(); + let primary_id = get_worktree_by_key(&pool, "cortex-main") + .unwrap() + .unwrap() + .id; + let feature_before = get_worktree_by_key(&pool, "cortex-feature") + .unwrap() + .unwrap(); + + let mut updated_repo = repo.clone(); + updated_repo.primary_path = "/workspace/cortex-renamed".to_string(); + updated_repo.display_name = "Cortex Prime".to_string(); + updated_repo.remote_url_hash = None; + updated_repo.metadata_json = "{\"source\":\"second-reconcile\"}".to_string(); + let mut updated_primary = primary.clone(); + updated_primary.branch_name = Some("trunk".to_string()); + updated_primary.branch_ref = Some("refs/heads/trunk".to_string()); + updated_primary.dirty = true; + updated_primary.staged_count = 2; + updated_primary.unstaged_count = 3; + updated_primary.untracked_count = 4; + updated_primary.ahead = Some(5); + updated_primary.behind = None; + updated_primary.status_hash = Some("status-updated".to_string()); + + let updated = + reconcile_repository(&pool, &updated_repo, &[updated_primary], second_seen).unwrap(); + assert_eq!(updated.repository.id, repository_id); + assert_eq!(updated.repository.first_seen_at, repository_first_seen); + assert_eq!(updated.repository.last_seen_at, second_seen); + assert_eq!(updated.repository.primary_path, "/workspace/cortex-renamed"); + assert_eq!(updated.repository.display_name, "Cortex Prime"); + assert_eq!(updated.repository.remote_url_hash, None); + assert_eq!(updated.removed_worktree_ids, vec![feature_before.id]); + + let primary_after = get_worktree_by_key(&pool, "cortex-main").unwrap().unwrap(); + assert_eq!(primary_after.id, primary_id); + assert_eq!(primary_after.first_seen_at, first_seen); + assert_eq!(primary_after.last_seen_at, second_seen); + assert_eq!(primary_after.branch_name.as_deref(), Some("trunk")); + assert!(primary_after.dirty); + assert_eq!(primary_after.staged_count, 2); + assert_eq!(primary_after.unstaged_count, 3); + assert_eq!(primary_after.untracked_count, 4); + assert_eq!(primary_after.ahead, Some(5)); + assert_eq!(primary_after.behind, None); + + let feature_removed = get_worktree_by_key(&pool, "cortex-feature") + .unwrap() + .unwrap(); + assert_eq!(feature_removed.id, feature_before.id); + assert_eq!(feature_removed.first_seen_at, first_seen); + assert_eq!(feature_removed.last_seen_at, first_seen); + assert_eq!(feature_removed.removed_at.as_deref(), Some(second_seen)); + assert_eq!( + list_repository_worktrees(&pool, repository_id, false) + .unwrap() + .len(), + 1 + ); + assert_eq!( + list_repository_worktrees(&pool, repository_id, true) + .unwrap() + .len(), + 2 + ); + + assert!(mark_repository_removed(&pool, "cortex", removed_at).unwrap()); + assert!(mark_repository_removed(&pool, "cortex", reappeared_at).unwrap()); + let repository_removed = get_repository_by_key(&pool, "cortex").unwrap().unwrap(); + assert_eq!(repository_removed.removed_at.as_deref(), Some(removed_at)); + + let reappeared = reconcile_repository( + &pool, + &updated_repo, + &[primary.clone(), feature.clone()], + reappeared_at, + ) + .unwrap(); + assert_eq!(reappeared.repository.id, repository_id); + assert_eq!(reappeared.repository.first_seen_at, first_seen); + assert_eq!(reappeared.repository.removed_at, None); + assert_eq!(reappeared.repository.last_seen_at, reappeared_at); + + let feature_after = get_worktree_by_key(&pool, "cortex-feature") + .unwrap() + .unwrap(); + assert_eq!(feature_after.id, feature_before.id); + assert_eq!(feature_after.first_seen_at, first_seen); + assert_eq!(feature_after.removed_at, None); + assert_eq!(feature_after.last_seen_at, reappeared_at); + + assert!(mark_worktree_removed(&pool, "cortex-feature", removed_at).unwrap()); + assert!(mark_worktree_removed(&pool, "cortex-feature", reappeared_at).unwrap()); + assert_eq!( + get_worktree_by_key(&pool, "cortex-feature") + .unwrap() + .unwrap() + .removed_at + .as_deref(), + Some(removed_at) + ); +} + +#[test] +fn reconcile_rolls_back_repository_when_worktree_upsert_fails() { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test(dir.path().join("rollback.db"))).unwrap(); + let observed_at = "2026-08-02T16:00:00.000Z"; + + reconcile_repository( + &pool, + &repository("one", "One"), + &[worktree("shared-key", "/workspace/one", "main")], + observed_at, + ) + .unwrap(); + + let conflict = worktree("shared-key", "/workspace/two", "main"); + assert!( + reconcile_repository(&pool, &repository("two", "Two"), &[conflict], observed_at,).is_err() + ); + assert!(get_repository_by_key(&pool, "two").unwrap().is_none()); + assert_eq!( + get_worktree_by_key(&pool, "shared-key") + .unwrap() + .unwrap() + .path, + "/workspace/one" + ); +} + +#[test] +fn canonical_path_validation_rejects_relative_and_parent_paths_without_writes() { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test(dir.path().join("paths.db"))).unwrap(); + let observed_at = "2026-08-02T16:00:00.000Z"; + + let mut relative = repository("relative", "Relative"); + relative.primary_path = "workspace/relative".to_string(); + assert!(reconcile_repository(&pool, &relative, &[], observed_at).is_err()); + assert!(get_repository_by_key(&pool, "relative").unwrap().is_none()); + + let mut parent = repository("parent", "Parent"); + parent.common_git_dir = "/workspace/parent/../other/.git".to_string(); + assert!(reconcile_repository(&pool, &parent, &[], observed_at).is_err()); + assert!(get_repository_by_key(&pool, "parent").unwrap().is_none()); + + let repo = repository("valid", "Valid"); + let mut invalid_worktree = worktree("invalid-wt", "/workspace/valid", "main"); + invalid_worktree.git_dir = "/workspace/valid/../escape/.git".to_string(); + assert!(reconcile_repository(&pool, &repo, &[invalid_worktree], observed_at).is_err()); + assert!(get_repository_by_key(&pool, "valid").unwrap().is_none()); +} + +#[test] +fn parameterized_upserts_preserve_sql_metacharacters_as_data() { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test(dir.path().join("parameters.db"))).unwrap(); + let observed_at = "2026-08-02T16:00:00.000Z"; + let display_name = "Cortex'); DROP TABLE repositories; --"; + let repo = repository("quoted", display_name); + + reconcile_repository(&pool, &repo, &[], observed_at).unwrap(); + assert_eq!( + get_repository_by_key(&pool, "quoted") + .unwrap() + .unwrap() + .display_name, + display_name + ); + + reconcile_repository( + &pool, + &repository("second", "Still exists"), + &[], + observed_at, + ) + .unwrap(); + assert!(get_repository_by_key(&pool, "second").unwrap().is_some()); +} diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_sources.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_sources.rs new file mode 100644 index 00000000..6da9d998 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_sources.rs @@ -0,0 +1,366 @@ +//! Bounded source-table pages for Agent Observatory projection. + +use crate::DbPool; +use anyhow::{Context, Result, bail}; +use rusqlite::params; +use serde::{Deserialize, Serialize}; + +const MAX_PAGE: usize = 500; + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum AgentSourceKind { + Mcp, + Hook, + Skill, + Llm, +} + +impl AgentSourceKind { + #[must_use] + pub const fn as_str(self) -> &'static str { + match self { + Self::Mcp => "mcp_events", + Self::Hook => "hook_events", + Self::Skill => "skill_events", + Self::Llm => "llm_invocations", + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AgentMcpSourceRow { + pub cursor_id: i64, + pub call_log_id: Option, + pub result_log_id: Option, + pub ai_tool: String, + pub ai_project: Option, + pub ai_session_id: Option, + pub hostname: String, + pub timestamp: String, + pub turn_id: Option, + pub call_id: String, + pub tool_name: String, + pub mcp_server: Option, + pub mcp_tool: Option, + pub event_kind: String, + pub status: Option, + pub duration_ms: Option, + pub is_error: Option, + pub arguments_json: Option, + pub output_preview: Option, + pub error_text: Option, + pub metadata_json: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AgentHookSourceRow { + pub cursor_id: i64, + pub log_id: Option, + pub ai_tool: String, + pub ai_project: Option, + pub ai_session_id: Option, + pub hostname: String, + pub timestamp: String, + pub hook_event: String, + pub hook_name: Option, + pub hook_source: Option, + pub hook_command: Option, + pub status: String, + pub exit_code: Option, + pub duration_ms: Option, + pub stdout_preview: Option, + pub stderr_preview: Option, + pub persisted_output_path: Option, + pub trusted_hash: Option, + pub evidence_kind: String, + pub metadata_json: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AgentSkillSourceRow { + pub cursor_id: i64, + pub log_id: i64, + pub ai_tool: String, + pub ai_project: Option, + pub ai_session_id: Option, + pub hostname: String, + pub timestamp: String, + pub skill_name: String, + pub skill_plugin: Option, + pub event_kind: String, + pub evidence_kind: String, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AgentLlmSourceRow { + pub id: String, + pub started_at: String, + pub finished_at: Option, + pub duration_ms: Option, + pub caller_surface: String, + pub action: String, + pub provider: String, + pub model: Option, + pub program: Option, + pub incident_id: Option, + pub ai_tool: Option, + pub ai_project: Option, + pub ai_session_id: Option, + pub evidence_counts_json: Option, + pub prompt_bytes: Option, + pub output_bytes: Option, + pub status: String, + pub error: Option, + pub metadata_json: Option, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum AgentSourceRecord { + Mcp(AgentMcpSourceRow), + Hook(AgentHookSourceRow), + Skill(AgentSkillSourceRow), + Llm(AgentLlmSourceRow), +} + +impl AgentSourceRecord { + pub(crate) fn next_cursor(&self) -> String { + match self { + Self::Mcp(row) => row.cursor_id.to_string(), + Self::Hook(row) => row.cursor_id.to_string(), + Self::Skill(row) => row.cursor_id.to_string(), + Self::Llm(row) => serde_json::to_string(&LlmCursor { + started_at: row.started_at.clone(), + id: row.id.clone(), + }) + .expect("LLM cursor serialization cannot fail"), + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AgentSourcePage { + pub records: Vec, + pub next_cursor: String, + pub truncated: bool, +} + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +struct LlmCursor { + started_at: String, + id: String, +} + +fn validate_page(limit: usize) -> Result { + if limit == 0 || limit > MAX_PAGE { + bail!("limit must be between 1 and {MAX_PAGE}"); + } + i64::try_from(limit + 1).context("page limit exceeds SQLite integer range") +} + +fn numeric_cursor(cursor: &str) -> Result { + if cursor.is_empty() { + return Ok(0); + } + let value = cursor + .parse::() + .with_context(|| format!("invalid numeric source cursor: {cursor}"))?; + if value < 0 { + bail!("numeric source cursor must be non-negative"); + } + Ok(value) +} + +fn llm_cursor(cursor: &str) -> Result> { + if cursor.is_empty() { + return Ok(None); + } + let cursor: LlmCursor = serde_json::from_str(cursor).context("invalid LLM source cursor")?; + if cursor.id.is_empty() || chrono::DateTime::parse_from_rfc3339(&cursor.started_at).is_err() { + bail!("invalid LLM source cursor fields"); + } + Ok(Some(cursor)) +} + +fn mcp_page(conn: &rusqlite::Connection, after: i64, limit: i64) -> Result> { + let mut stmt = conn.prepare( + "SELECT id, call_log_id, result_log_id, ai_tool, ai_project, ai_session_id, + hostname, timestamp, turn_id, call_id, tool_name, mcp_server, mcp_tool, + event_kind, status, duration_ms, is_error, arguments_json, output_preview, + error_text, metadata_json + FROM ai_mcp_events WHERE id > ?1 ORDER BY id LIMIT ?2", + )?; + Ok(stmt + .query_map(params![after, limit], |row| { + Ok(AgentSourceRecord::Mcp(AgentMcpSourceRow { + cursor_id: row.get(0)?, + call_log_id: row.get(1)?, + result_log_id: row.get(2)?, + ai_tool: row.get(3)?, + ai_project: row.get(4)?, + ai_session_id: row.get(5)?, + hostname: row.get(6)?, + timestamp: row.get(7)?, + turn_id: row.get(8)?, + call_id: row.get(9)?, + tool_name: row.get(10)?, + mcp_server: row.get(11)?, + mcp_tool: row.get(12)?, + event_kind: row.get(13)?, + status: row.get(14)?, + duration_ms: row.get(15)?, + is_error: row.get(16)?, + arguments_json: row.get(17)?, + output_preview: row.get(18)?, + error_text: row.get(19)?, + metadata_json: row.get(20)?, + })) + })? + .collect::>()?) +} + +fn hook_page( + conn: &rusqlite::Connection, + after: i64, + limit: i64, +) -> Result> { + let mut stmt = conn.prepare( + "SELECT id, log_id, ai_tool, ai_project, ai_session_id, hostname, timestamp, + hook_event, hook_name, hook_source, hook_command, status, exit_code, + duration_ms, stdout_preview, stderr_preview, persisted_output_path, + trusted_hash, evidence_kind, metadata_json + FROM ai_hook_events WHERE id > ?1 ORDER BY id LIMIT ?2", + )?; + Ok(stmt + .query_map(params![after, limit], |row| { + Ok(AgentSourceRecord::Hook(AgentHookSourceRow { + cursor_id: row.get(0)?, + log_id: row.get(1)?, + ai_tool: row.get(2)?, + ai_project: row.get(3)?, + ai_session_id: row.get(4)?, + hostname: row.get(5)?, + timestamp: row.get(6)?, + hook_event: row.get(7)?, + hook_name: row.get(8)?, + hook_source: row.get(9)?, + hook_command: row.get(10)?, + status: row.get(11)?, + exit_code: row.get(12)?, + duration_ms: row.get(13)?, + stdout_preview: row.get(14)?, + stderr_preview: row.get(15)?, + persisted_output_path: row.get(16)?, + trusted_hash: row.get(17)?, + evidence_kind: row.get(18)?, + metadata_json: row.get(19)?, + })) + })? + .collect::>()?) +} + +fn skill_page( + conn: &rusqlite::Connection, + after: i64, + limit: i64, +) -> Result> { + let mut stmt = conn.prepare( + "SELECT id, log_id, ai_tool, ai_project, ai_session_id, hostname, timestamp, + skill_name, skill_plugin, event_kind, evidence_kind + FROM ai_skill_events WHERE id > ?1 ORDER BY id LIMIT ?2", + )?; + Ok(stmt + .query_map(params![after, limit], |row| { + Ok(AgentSourceRecord::Skill(AgentSkillSourceRow { + cursor_id: row.get(0)?, + log_id: row.get(1)?, + ai_tool: row.get(2)?, + ai_project: row.get(3)?, + ai_session_id: row.get(4)?, + hostname: row.get(5)?, + timestamp: row.get(6)?, + skill_name: row.get(7)?, + skill_plugin: row.get(8)?, + event_kind: row.get(9)?, + evidence_kind: row.get(10)?, + })) + })? + .collect::>()?) +} + +fn llm_page( + conn: &rusqlite::Connection, + after: Option<&LlmCursor>, + limit: i64, +) -> Result> { + let mut stmt = conn.prepare( + "SELECT id, started_at, finished_at, duration_ms, caller_surface, action, + provider, model, program, incident_id, ai_tool, ai_project, ai_session_id, + evidence_counts_json, prompt_bytes, output_bytes, status, error, metadata_json + FROM llm_invocations + WHERE ?1 IS NULL + OR started_at > ?1 + OR (started_at = ?1 AND id > ?2) + ORDER BY started_at, id + LIMIT ?3", + )?; + let after_started_at = after.map(|cursor| cursor.started_at.as_str()); + let after_id = after.map(|cursor| cursor.id.as_str()); + Ok(stmt + .query_map(params![after_started_at, after_id, limit], |row| { + Ok(AgentSourceRecord::Llm(AgentLlmSourceRow { + id: row.get(0)?, + started_at: row.get(1)?, + finished_at: row.get(2)?, + duration_ms: row.get(3)?, + caller_surface: row.get(4)?, + action: row.get(5)?, + provider: row.get(6)?, + model: row.get(7)?, + program: row.get(8)?, + incident_id: row.get(9)?, + ai_tool: row.get(10)?, + ai_project: row.get(11)?, + ai_session_id: row.get(12)?, + evidence_counts_json: row.get(13)?, + prompt_bytes: row.get(14)?, + output_bytes: row.get(15)?, + status: row.get(16)?, + error: row.get(17)?, + metadata_json: row.get(18)?, + })) + })? + .collect::>()?) +} + +pub fn page_agent_sources( + pool: &DbPool, + kind: AgentSourceKind, + after_cursor: &str, + limit: usize, +) -> Result { + let probe_limit = validate_page(limit)?; + let conn = pool.get().context("acquire database connection")?; + let mut records = match kind { + AgentSourceKind::Mcp => mcp_page(&conn, numeric_cursor(after_cursor)?, probe_limit)?, + AgentSourceKind::Hook => hook_page(&conn, numeric_cursor(after_cursor)?, probe_limit)?, + AgentSourceKind::Skill => skill_page(&conn, numeric_cursor(after_cursor)?, probe_limit)?, + AgentSourceKind::Llm => { + let after = llm_cursor(after_cursor)?; + llm_page(&conn, after.as_ref(), probe_limit)? + } + }; + let truncated = records.len() > limit; + records.truncate(limit); + let next_cursor = records + .last() + .map_or_else(|| after_cursor.to_string(), AgentSourceRecord::next_cursor); + Ok(AgentSourcePage { + records, + next_cursor, + truncated, + }) +} + +#[cfg(test)] +#[path = "agent_observatory_sources_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_sources_tests.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_sources_tests.rs new file mode 100644 index 00000000..dc1ddb78 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_sources_tests.rs @@ -0,0 +1,168 @@ +use super::{AgentSourceKind, AgentSourceRecord, page_agent_sources}; +use crate::config::StorageConfig; +use crate::{LogBatchEntry, init_pool, insert_logs_batch}; +use rusqlite::params; + +fn log_entry() -> LogBatchEntry { + LogBatchEntry { + timestamp: "2026-08-05T12:00:00.000Z".to_string(), + hostname: "devhost".to_string(), + facility: None, + severity: "info".to_string(), + app_name: Some("fixture".to_string()), + process_id: None, + message: "fixture".to_string(), + raw: "fixture".to_string(), + source_ip: "test://ao038".to_string(), + docker_checkpoint: None, + ai_tool: Some("claude".to_string()), + ai_project: Some("/workspace/cortex".to_string()), + ai_session_id: Some("session-one".to_string()), + ai_transcript_path: None, + metadata_json: Some("{}".to_string()), + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + } +} + +#[test] +fn pages_all_projection_sources_in_cursor_order_with_hard_limits() { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test(dir.path().join("sources.db"))).unwrap(); + insert_logs_batch(&pool, &[log_entry()]).unwrap(); + let conn = pool.get().unwrap(); + let log_id: i64 = conn + .query_row("SELECT MAX(id) FROM logs", [], |row| row.get(0)) + .unwrap(); + + for suffix in ["one", "two"] { + conn.execute( + "INSERT INTO ai_mcp_events + (ai_tool, ai_project, ai_session_id, hostname, timestamp, call_id, + tool_name, mcp_server, mcp_tool, event_kind, status, is_error, + arguments_json, metadata_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)", + params![ + "claude", + "/workspace/cortex", + "session-one", + "devhost", + "2026-08-05T12:00:00.000Z", + format!("call-{suffix}"), + "mcp__server__tool", + "server", + "tool", + suffix, + "ok", + 0, + "{}", + "{}", + ], + ) + .unwrap(); + conn.execute( + "INSERT INTO ai_hook_events + (ai_tool, ai_project, ai_session_id, hostname, timestamp, + hook_event, hook_name, status, evidence_kind, metadata_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", + params![ + "claude", + "/workspace/cortex", + "session-one", + "devhost", + "2026-08-05T12:00:00.000Z", + suffix, + "fixture-hook", + "success", + "runtime", + "{}", + ], + ) + .unwrap(); + conn.execute( + "INSERT INTO ai_skill_events + (log_id, ai_tool, ai_project, ai_session_id, hostname, timestamp, + skill_name, skill_plugin, event_kind, evidence_kind) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", + params![ + log_id, + "claude", + "/workspace/cortex", + "session-one", + "devhost", + "2026-08-05T12:00:00.000Z", + format!("skill-{suffix}"), + "fixture-plugin", + suffix, + "tag", + ], + ) + .unwrap(); + conn.execute( + "INSERT INTO llm_invocations + (id, started_at, finished_at, duration_ms, caller_surface, action, + provider, model, ai_tool, ai_project, ai_session_id, status, + metadata_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13)", + params![ + format!("llm-{suffix}"), + "2026-08-05T12:00:00.000Z", + "2026-08-05T12:00:01.000Z", + 1000, + "cli", + suffix, + "openai", + "gpt-test", + "claude", + "/workspace/cortex", + "session-one", + "success", + "{}", + ], + ) + .unwrap(); + } + drop(conn); + + for kind in [ + AgentSourceKind::Mcp, + AgentSourceKind::Hook, + AgentSourceKind::Skill, + AgentSourceKind::Llm, + ] { + let first = page_agent_sources(&pool, kind, "", 1).unwrap(); + assert_eq!(first.records.len(), 1); + assert!(first.truncated); + assert!(!first.next_cursor.is_empty()); + assert!(matches!( + (&kind, &first.records[0]), + (AgentSourceKind::Mcp, AgentSourceRecord::Mcp(_)) + | (AgentSourceKind::Hook, AgentSourceRecord::Hook(_)) + | (AgentSourceKind::Skill, AgentSourceRecord::Skill(_)) + | (AgentSourceKind::Llm, AgentSourceRecord::Llm(_)) + )); + if kind == AgentSourceKind::Llm { + pool.get().unwrap().execute_batch("VACUUM").unwrap(); + } + let second = page_agent_sources(&pool, kind, &first.next_cursor, 500).unwrap(); + assert_eq!(second.records.len(), 1); + assert!(!second.truncated); + assert_ne!(second.next_cursor, first.next_cursor); + let empty = page_agent_sources(&pool, kind, &second.next_cursor, 500).unwrap(); + assert!(empty.records.is_empty()); + assert_eq!(empty.next_cursor, second.next_cursor); + } +} + +#[test] +fn source_page_rejects_zero_and_over_limit_pages() { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test(dir.path().join("limits.db"))).unwrap(); + assert!(page_agent_sources(&pool, AgentSourceKind::Mcp, "", 0).is_err()); + assert!(page_agent_sources(&pool, AgentSourceKind::Mcp, "", 501).is_err()); + assert!(page_agent_sources(&pool, AgentSourceKind::Mcp, "-1", 1).is_err()); + assert!(page_agent_sources(&pool, AgentSourceKind::Llm, "not-json", 1).is_err()); +} diff --git a/crates/shared/cortex/storage-sqlite/src/agent_observatory_tests.rs b/crates/shared/cortex/storage-sqlite/src/agent_observatory_tests.rs new file mode 100644 index 00000000..5338c8eb --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/agent_observatory_tests.rs @@ -0,0 +1,112 @@ +use super::agent_observatory::{ + AgentEventKind, EvidenceTrustLevel, RepositoryObservationKind, RunStatus, StreamEventName, + advance_projection_cursor, projection_cursor, projection_health, record_projection_health, +}; +use super::otlp_metrics::MetricInstrumentKind; +use super::otlp_traces::OtelSpanRow; +use super::{StorageConfig, init_pool}; +use std::str::FromStr; + +#[test] +fn observatory_text_enums_round_trip_and_reject_unknown_values() { + for value in RunStatus::ALL { + assert_eq!(RunStatus::from_str(value.as_str()).unwrap(), *value); + } + for value in AgentEventKind::ALL { + assert_eq!(AgentEventKind::from_str(value.as_str()).unwrap(), *value); + } + for value in EvidenceTrustLevel::ALL { + assert_eq!( + EvidenceTrustLevel::from_str(value.as_str()).unwrap(), + *value + ); + } + for value in RepositoryObservationKind::ALL { + assert_eq!( + RepositoryObservationKind::from_str(value.as_str()).unwrap(), + *value + ); + } + for value in StreamEventName::ALL { + assert_eq!(StreamEventName::from_str(value.as_str()).unwrap(), *value); + } + for value in MetricInstrumentKind::ALL { + assert_eq!( + MetricInstrumentKind::from_str(value.as_str()).unwrap(), + *value + ); + } + + assert!(RunStatus::from_str("running").is_err()); + assert!(AgentEventKind::from_str("unknown").is_err()); + assert!(EvidenceTrustLevel::from_str("trusted").is_err()); + assert!(RepositoryObservationKind::from_str("poll").is_err()); + assert!(StreamEventName::from_str("run.deleted").is_err()); + assert!(MetricInstrumentKind::from_str("counter").is_err()); +} + +#[test] +fn observatory_row_structs_use_string_api_keys_and_internal_integer_ids() { + let span = OtelSpanRow { + id: 7, + trace_id: "0123456789abcdef0123456789abcdef".to_string(), + span_id: "0123456789abcdef".to_string(), + parent_span_id: None, + trace_state: None, + flags: 0, + span_name: "fixture".to_string(), + span_kind: 1, + start_time_unix_nano: 10, + end_time_unix_nano: 20, + duration_nano: 10, + status_code: 0, + status_message: None, + hostname: "fixture-host".to_string(), + service_name: None, + service_version: None, + scope_name: None, + scope_version: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + run_id: None, + resource_json: "{}".to_string(), + attributes_json: "{}".to_string(), + events_json: "[]".to_string(), + links_json: "[]".to_string(), + received_at: "2026-01-01T00:00:00.000Z".to_string(), + content_scrubbed: true, + }; + assert_eq!(span.id, 7); + assert_eq!(span.trace_id.len(), 32); +} + +#[test] +fn projection_cursor_and_health_ports_round_trip() { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig { + db_path: dir.path().join("observatory-ports.db"), + pool_size: 1, + wal_mode: false, + ..StorageConfig::default() + }) + .unwrap(); + + assert_eq!(projection_cursor(&pool, "transcript").unwrap(), ""); + advance_projection_cursor(&pool, "transcript", "42").unwrap(); + assert_eq!(projection_cursor(&pool, "transcript").unwrap(), "42"); + + assert!(projection_health(&pool, "projector").unwrap().is_none()); + record_projection_health(&pool, "projector", "ok", "first pass").unwrap(); + record_projection_health(&pool, "projector", "degraded", "retrying").unwrap(); + let health: serde_json::Value = serde_json::from_str( + projection_health(&pool, "projector") + .unwrap() + .as_deref() + .unwrap(), + ) + .unwrap(); + assert_eq!(health["status"], "degraded"); + assert_eq!(health["detail"], "retrying"); + assert_eq!(health["attempts"], 2); +} diff --git a/crates/shared/cortex/storage-sqlite/src/analytics.rs b/crates/shared/cortex/storage-sqlite/src/analytics.rs new file mode 100644 index 00000000..062bdcb7 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/analytics.rs @@ -0,0 +1,1696 @@ +//! Higher-level analytics queries layered on top of the `logs` table. +//! +//! These power MCP actions beyond raw search/tail: distinct-value enumeration +//! (`apps`, `source_ips`), time-series aggregations (`timeline`, +//! `ingest_rate`), pattern clustering (`patterns`), drill-down helpers +//! (`context`, `get`), operational health (`silent_hosts`, `clock_skew`), +//! and comparison/anomaly detection. + +use anyhow::Result; +use rusqlite::params; +use std::cmp::Reverse; +use std::collections::BTreeMap; + +use super::pool::DbPool; +use super::queries::map_row_with_raw; +use super::{ + AiProjectContext, AiProjectContextParams, AiUsageBlock, AiUsageBlocksParams, + AiUsageBlocksResult, LogEntry, +}; + +// ----------------------------------------------------------------------------- +// apps: distinct app_names with stats +// ----------------------------------------------------------------------------- + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct AppEntry { + pub app_name: String, + pub log_count: i64, + pub host_count: i64, + pub first_seen: String, + pub last_seen: String, +} + +pub struct ListAppsParams<'a> { + pub hostname: Option<&'a str>, + pub from: Option<&'a str>, + pub to: Option<&'a str>, + /// Page size. Default 500, max 5000. + pub limit: usize, + /// Page offset (number of distinct apps to skip). Default 0. + pub offset: usize, +} + +pub struct ListAppsResult { + pub apps: Vec, + /// Total distinct app names matching the filter (across all pages). + pub total: usize, +} + +pub fn list_apps(pool: &DbPool, params: &ListAppsParams<'_>) -> Result { + let conn = pool.get()?; + let limit = params.limit.clamp(1, 5_000); + let offset = params.offset; + + if params.hostname.is_none() + && params.from.is_none() + && params.to.is_none() + && inventory_backfill_complete_conn(&conn).unwrap_or(false) + { + let total = conn.query_row("SELECT COUNT(*) FROM app_inventory_stats", [], |row| { + row.get::<_, i64>(0) + })? as usize; + let mut stmt = conn.prepare(&format!( + "WITH page AS ( + SELECT app_name, log_count, first_seen, last_seen + FROM app_inventory_stats + ORDER BY last_seen DESC, app_name ASC + LIMIT {limit} OFFSET {offset} + ) + SELECT p.app_name, p.log_count, COUNT(h.hostname), p.first_seen, p.last_seen + FROM page p + LEFT JOIN app_host_inventory_stats h ON h.app_name = p.app_name + GROUP BY p.app_name, p.log_count, p.first_seen, p.last_seen + ORDER BY p.last_seen DESC, p.app_name ASC" + ))?; + let apps = stmt + .query_map([], |row| { + Ok(AppEntry { + app_name: row.get(0)?, + log_count: row.get(1)?, + host_count: row.get(2)?, + first_seen: row.get(3)?, + last_seen: row.get(4)?, + }) + })? + .collect::>>()?; + + return Ok(ListAppsResult { apps, total }); + } + + // Build the shared WHERE clause and bindings once; reuse for COUNT and data queries. + // first_seen / last_seen come from `received_at` (server clock) so they match + // how the `hosts` table is updated and aren't skewed by a misconfigured device clock. + let mut where_clause = String::from("app_name IS NOT NULL AND app_name != ''"); + let mut bindings: Vec = vec![]; + let mut idx = 1usize; + + if let Some(h) = params.hostname { + where_clause.push_str(&format!(" AND hostname = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(h.to_owned())); + idx += 1; + } + if let Some(f) = params.from { + where_clause.push_str(&format!(" AND received_at >= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(f.to_owned())); + idx += 1; + } + if let Some(t) = params.to { + where_clause.push_str(&format!(" AND received_at <= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(t.to_owned())); + idx += 1; + } + let _ = idx; + + let total = conn.query_row( + &format!("SELECT COUNT(DISTINCT app_name) FROM logs WHERE {where_clause}"), + rusqlite::params_from_iter(bindings.iter()), + |row| row.get::<_, i64>(0), + )? as usize; + + let data_sql = format!( + "SELECT app_name, COUNT(*), COUNT(DISTINCT hostname), + MIN(received_at), MAX(received_at) + FROM logs + WHERE {where_clause} + GROUP BY app_name + ORDER BY MAX(received_at) DESC, app_name ASC + LIMIT {limit} OFFSET {offset}" + ); + let mut stmt = conn.prepare(&data_sql)?; + let apps = stmt + .query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + Ok(AppEntry { + app_name: row.get(0)?, + log_count: row.get(1)?, + host_count: row.get(2)?, + first_seen: row.get(3)?, + last_seen: row.get(4)?, + }) + })? + .collect::>>()?; + + Ok(ListAppsResult { apps, total }) +} + +// ----------------------------------------------------------------------------- +// source_ips: distinct senders + hostnames seen per sender +// ----------------------------------------------------------------------------- + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct SourceIpHostBreakdown { + pub hostname: String, + pub log_count: i64, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct SourceIpEntry { + pub source_ip: String, + pub log_count: i64, + pub host_count: i64, + pub first_seen: String, + pub last_seen: String, + /// Top hostnames associated with this source_ip (capped at 10). + pub hostnames: Vec, +} + +pub struct ListSourceIpsResult { + pub source_ips: Vec, + /// Total distinct source IPs in the database (across all pages). + pub total: usize, +} + +pub struct ListSourceIpsParams { + /// Page size. Default 500, max 5000. + pub limit: usize, + /// Page offset (number of distinct IPs to skip). Default 0. + pub offset: usize, +} + +pub fn list_source_ips(pool: &DbPool, params: &ListSourceIpsParams) -> Result { + let limit = params.limit.clamp(1, 5_000); + let offset = params.offset; + + let inventory_complete = { + let conn = pool.get()?; + inventory_backfill_complete_conn(&conn).unwrap_or(false) + }; + + if inventory_complete { + let conn = pool.get()?; + let total = conn.query_row( + "SELECT COUNT(*) FROM source_ip_inventory_stats", + [], + |row| row.get::<_, i64>(0), + )? as usize; + + let mut stmt = conn.prepare(&format!( + "WITH page AS ( + SELECT source_ip, log_count, first_seen, last_seen + FROM source_ip_inventory_stats + ORDER BY log_count DESC, source_ip ASC + LIMIT {limit} OFFSET {offset} + ) + SELECT p.source_ip, p.log_count, p.first_seen, p.last_seen, + h.hostname, h.log_count, h.first_seen, h.last_seen + FROM page p + LEFT JOIN source_ip_host_inventory_stats h ON h.source_ip = p.source_ip + ORDER BY p.log_count DESC, p.source_ip ASC, h.log_count DESC, h.hostname ASC" + ))?; + let rows = stmt.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, Option>(5)?.unwrap_or(0), + )) + })?; + + let mut by_ip: BTreeMap = BTreeMap::new(); + for row in rows { + let (ip, total_count, first, last, host, host_count) = row?; + let entry = by_ip.entry(ip.clone()).or_insert_with(|| SourceIpEntry { + source_ip: ip, + log_count: total_count, + host_count: 0, + first_seen: first, + last_seen: last, + hostnames: Vec::new(), + }); + if let Some(host) = host { + entry.host_count += 1; + if entry.hostnames.len() < 10 { + entry.hostnames.push(SourceIpHostBreakdown { + hostname: host, + log_count: host_count, + }); + } + } + } + + let mut out: Vec = by_ip.into_values().collect(); + out.sort_by_key(|entry| Reverse(entry.log_count)); + return Ok(ListSourceIpsResult { + source_ips: out, + total, + }); + } + + list_source_ips_from_logs(pool, params) +} + +fn inventory_backfill_complete_conn(conn: &rusqlite::Connection) -> rusqlite::Result { + conn.query_row( + "SELECT completed_at IS NOT NULL + FROM inventory_backfill_state + WHERE name = 'app_source_inventory'", + [], + |row| row.get::<_, bool>(0), + ) +} + +fn list_source_ips_from_logs( + pool: &DbPool, + params: &ListSourceIpsParams, +) -> Result { + let limit = params.limit.clamp(1, 5_000); + let offset = params.offset; + let conn = pool.get()?; + + // Single query: the scalar subquery computes the total in one pass alongside the page data. + let mut stmt = conn.prepare(&format!( + "WITH ip_agg AS ( + SELECT source_ip, COUNT(*) AS ip_count + FROM logs + WHERE source_ip != '' + GROUP BY source_ip + ), + top_ips AS ( + SELECT source_ip + FROM ip_agg + ORDER BY ip_count DESC, source_ip ASC + LIMIT {limit} OFFSET {offset} + ) + SELECT l.source_ip, l.hostname, COUNT(*), + MIN(l.received_at), MAX(l.received_at), + (SELECT COUNT(*) FROM ip_agg) AS grand_total + FROM logs l + JOIN top_ips t ON t.source_ip = l.source_ip + GROUP BY l.source_ip, l.hostname + ORDER BY l.source_ip, COUNT(*) DESC" + ))?; + let rows = stmt.query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)?, + row.get::<_, String>(3)?, + row.get::<_, String>(4)?, + row.get::<_, i64>(5)?, + )) + })?; + let mut total = 0usize; + + let mut by_ip: BTreeMap = BTreeMap::new(); + for row in rows { + let (ip, host, count, first, last, grand_total) = row?; + if total == 0 { + total = grand_total as usize; + } + let entry = by_ip.entry(ip.clone()).or_insert_with(|| SourceIpEntry { + source_ip: ip, + log_count: 0, + host_count: 0, + first_seen: first.clone(), + last_seen: last.clone(), + hostnames: Vec::new(), + }); + entry.log_count += count; + entry.host_count += 1; + if first < entry.first_seen { + entry.first_seen = first; + } + if last > entry.last_seen { + entry.last_seen = last; + } + if entry.hostnames.len() < 10 { + entry.hostnames.push(SourceIpHostBreakdown { + hostname: host, + log_count: count, + }); + } + } + + let mut out: Vec = by_ip.into_values().collect(); + out.sort_by_key(|entry| Reverse(entry.log_count)); + Ok(ListSourceIpsResult { + source_ips: out, + total, + }) +} + +pub fn get_ai_usage_blocks( + pool: &DbPool, + params: &AiUsageBlocksParams, +) -> Result { + let conn = pool.get()?; + const DEFAULT_LIMIT: usize = 1_000; + const MAX_LIMIT: usize = 1_000; + const DEFAULT_LOOKBACK_DAYS: i64 = 30; + const BUCKET_SECS: i64 = 18_000; + let limit = params + .limit + .map(|value| value as usize) + .unwrap_or(DEFAULT_LIMIT) + .clamp(1, MAX_LIMIT); + let mut sql = format!( + "SELECT datetime((CAST(strftime('%s', timestamp) AS INTEGER) / {BUCKET_SECS}) * {BUCKET_SECS}, 'unixepoch') AS bucket_start, + datetime(((CAST(strftime('%s', timestamp) AS INTEGER) / {BUCKET_SECS}) * {BUCKET_SECS}) + {BUCKET_SECS}, 'unixepoch') AS bucket_end, + ai_project, + ai_tool, + COUNT(DISTINCT ai_session_id) AS session_count, + COUNT(*) AS event_count + FROM logs + WHERE ai_project IS NOT NULL AND ai_project != '' + AND ai_tool IS NOT NULL AND ai_tool != '' + AND ai_session_id IS NOT NULL AND ai_session_id != ''" + ); + let mut bindings: Vec = Vec::new(); + let mut idx = 1usize; + if let Some(project) = ¶ms.ai_project { + sql.push_str(&format!(" AND ai_project = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(project.clone())); + idx += 1; + } + if let Some(tool) = ¶ms.ai_tool { + sql.push_str(&format!(" AND ai_tool = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(tool.clone())); + idx += 1; + } + if let Some(from) = ¶ms.since { + sql.push_str(&format!(" AND timestamp >= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(from.clone())); + idx += 1; + } + if let Some(to) = ¶ms.until { + sql.push_str(&format!(" AND timestamp <= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(to.clone())); + } else if params.since.is_none() { + sql.push_str(&format!( + " AND timestamp >= strftime('%Y-%m-%dT%H:%M:%fZ', 'now', '-{DEFAULT_LOOKBACK_DAYS} days')" + )); + } + sql.push_str(&format!( + " GROUP BY bucket_start, bucket_end, ai_project, ai_tool + ORDER BY bucket_start ASC, ai_project ASC, ai_tool ASC + LIMIT {}", + limit + 1 + )); + + let mut stmt = conn.prepare(&sql)?; + let mut blocks = stmt + .query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + Ok(AiUsageBlock { + bucket_start: row.get(0)?, + bucket_end: row.get(1)?, + project: row.get(2)?, + tool: row.get(3)?, + session_count: row.get(4)?, + event_count: row.get(5)?, + }) + })? + .collect::>>()?; + let truncated = truncate_to_limit(&mut blocks, limit); + Ok(AiUsageBlocksResult { + total_blocks: blocks.len(), + truncated, + blocks, + }) +} + +fn truncate_to_limit(values: &mut Vec, limit: usize) -> bool { + let truncated = values.len() > limit; + values.truncate(limit); + truncated +} + +pub fn get_ai_project_context( + pool: &DbPool, + params: &AiProjectContextParams, +) -> Result { + type ProjectAggregateRow = ( + Option, + Option, + Option, + Option, + Option, + i64, + ); + let conn = pool.get()?; + let mut aggregate_sql = String::from( + "SELECT GROUP_CONCAT(DISTINCT ai_tool), + GROUP_CONCAT(DISTINCT ai_session_id), + GROUP_CONCAT(DISTINCT hostname), + MIN(timestamp), + MAX(timestamp), + COUNT(*) + FROM logs + WHERE ai_project = ?1", + ); + let mut aggregate_bindings = vec![rusqlite::types::Value::Text(params.project.clone())]; + if let Some(tool) = ¶ms.ai_tool { + aggregate_sql.push_str(" AND ai_tool = ?2"); + aggregate_bindings.push(rusqlite::types::Value::Text(tool.clone())); + } + + let (tools, sessions, hostnames, first_seen, last_seen, event_count): ProjectAggregateRow = + conn.query_row( + &aggregate_sql, + rusqlite::params_from_iter(aggregate_bindings.iter()), + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + )) + }, + )?; + + let limit = params.limit.unwrap_or(5).min(20); + let mut recent_sql = String::from( + "SELECT id, timestamp, hostname, facility, severity, + app_name, process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs + WHERE ai_project = ?1", + ); + let mut recent_bindings = vec![rusqlite::types::Value::Text(params.project.clone())]; + if let Some(tool) = ¶ms.ai_tool { + recent_sql.push_str(" AND ai_tool = ?2"); + recent_bindings.push(rusqlite::types::Value::Text(tool.clone())); + } + recent_sql.push_str(&format!( + " ORDER BY timestamp DESC, id DESC LIMIT {}", + limit + 1 + )); + let mut stmt = conn.prepare(&recent_sql)?; + let mut recent_entries = stmt + .query_map( + rusqlite::params_from_iter(recent_bindings.iter()), + super::queries::map_row, + )? + .collect::>>()?; + let recent_entries_truncated = recent_entries.len() > limit as usize; + recent_entries.truncate(limit as usize); + for entry in &mut recent_entries { + entry.message = truncate_chars(&entry.message, 256); + } + + Ok(AiProjectContext { + project: params.project.clone(), + tools: split_csv(tools), + sessions: split_csv(sessions), + hostnames: split_csv(hostnames), + first_seen, + last_seen, + event_count, + recent_entries_truncated, + recent_entries, + }) +} + +fn truncate_chars(value: &str, max: usize) -> String { + if value.chars().count() <= max { + return value.to_string(); + } + value + .chars() + .take(max.saturating_sub(1)) + .collect::() + + "…" +} + +// ----------------------------------------------------------------------------- +// timeline: bucketed counts +// ----------------------------------------------------------------------------- + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Bucket { + Minute, + Hour, + Day, + Week, + Month, +} + +impl Bucket { + pub fn parse(s: &str) -> Option { + match s { + "minute" | "min" | "m" => Some(Self::Minute), + "hour" | "h" => Some(Self::Hour), + "day" | "d" => Some(Self::Day), + "week" | "w" => Some(Self::Week), + "month" => Some(Self::Month), + _ => None, + } + } + + /// SQLite `strftime` pattern for grouping timestamps into this bucket. + /// + /// `Week` uses `%W` (Monday-based week-of-year, 00–53). Days in early + /// January that fall *before* the year's first Monday land in week `00`, + /// producing labels like `2026-W00` (bead llto.2). Bucket tests deliberately + /// avoid that range. We do NOT use ISO-8601 week-numbering (`%G-%V`, which + /// would never emit week 00) because the SQLite version bundled here does + /// not support the `%G`/`%V` specifiers. + /// + /// `pub(crate)` (bead llto.3): the `db` module is `pub(crate)`, so this is + /// only reachable within the crate — widening past crate scope serves no + /// caller. + pub(crate) fn strftime_format(self) -> &'static str { + match self { + Self::Minute => "%Y-%m-%dT%H:%M:00Z", + Self::Hour => "%Y-%m-%dT%H:00:00Z", + Self::Day => "%Y-%m-%dT00:00:00Z", + Self::Week => "%Y-W%W", + Self::Month => "%Y-%m", + } + } + + /// Whether `timeline` can answer this bucket from the `timeline_hourly` + /// rollup (bead syslog-mcp-kcvq). The rollup grain is one hour, so every + /// bucket coarser-or-equal to an hour is exactly summable from it; only + /// `Minute` is finer and must hit the live `logs` table. + pub(crate) fn served_by_hourly_rollup(self) -> bool { + match self { + Self::Minute => false, + Self::Hour | Self::Day | Self::Week | Self::Month => true, + } + } + + /// Default lookback window (days) when no explicit `from`/`to` is provided. + /// Wider buckets scan wider time ranges, so larger defaults are appropriate. + pub fn default_lookback_days(self) -> i64 { + match self { + Self::Minute => 1, + Self::Hour => 7, + Self::Day => 30, + Self::Week => 180, + Self::Month => 730, + } + } +} + +#[derive(Debug, Clone, Copy)] +pub enum TimelineGroupBy { + None, + Hostname, + Severity, + AppName, +} + +impl TimelineGroupBy { + pub fn parse(s: &str) -> Option { + match s { + "hostname" | "host" => Some(Self::Hostname), + "severity" | "sev" => Some(Self::Severity), + "app_name" | "app" => Some(Self::AppName), + _ => None, + } + } + + fn column(self) -> Option<&'static str> { + match self { + Self::None => None, + Self::Hostname => Some("hostname"), + Self::Severity => Some("severity"), + Self::AppName => Some("app_name"), + } + } +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct TimelinePoint { + pub bucket: String, + #[serde(skip_serializing_if = "Option::is_none")] + pub group: Option, + pub count: i64, +} + +#[allow(clippy::too_many_arguments)] +pub fn timeline( + pool: &DbPool, + bucket: Bucket, + group_by: TimelineGroupBy, + from: Option<&str>, + to: Option<&str>, + hostname: Option<&str>, + app_name: Option<&str>, + severity_in: Option<&[String]>, +) -> Result> { + // Hour/day/week/month read the precomputed timeline_hourly rollup (bead + // syslog-mcp-kcvq) — O(#buckets) instead of an O(#rows) strftime GROUP BY. + // Minute is finer than the rollup grain, so it stays on the live table. + // Bounded-staleness: the rollup lags ingest by up to the refresh cadence + // (TIMELINE_ROLLUP_REFRESH_SECS) plus the current partial hour. + if bucket.served_by_hourly_rollup() { + return timeline_from_rollup( + pool, + bucket, + group_by, + from, + to, + hostname, + app_name, + severity_in, + ); + } + let conn = pool.get()?; + let mut sql = format!( + "SELECT strftime('{fmt}', timestamp) AS bucket", + fmt = bucket.strftime_format() + ); + if let Some(col) = group_by.column() { + sql.push_str(&format!(", COALESCE({col}, '') AS grp")); + } + sql.push_str(", COUNT(*) FROM logs WHERE 1=1"); + + let mut bindings: Vec> = Vec::new(); + let mut idx = 1usize; + + if let Some(f) = from { + sql.push_str(&format!(" AND timestamp >= ?{idx}")); + bindings.push(Box::new(f.to_string())); + idx += 1; + } + if let Some(t) = to { + sql.push_str(&format!(" AND timestamp <= ?{idx}")); + bindings.push(Box::new(t.to_string())); + idx += 1; + } + if let Some(h) = hostname { + sql.push_str(&format!(" AND hostname = ?{idx}")); + bindings.push(Box::new(h.to_string())); + idx += 1; + } + if let Some(a) = app_name { + sql.push_str(&format!(" AND app_name = ?{idx}")); + bindings.push(Box::new(a.to_string())); + idx += 1; + } + if let Some(levels) = severity_in + && !levels.is_empty() + { + let placeholders: Vec = + (0..levels.len()).map(|i| format!("?{}", idx + i)).collect(); + sql.push_str(&format!(" AND severity IN ({})", placeholders.join(", "))); + for lvl in levels { + bindings.push(Box::new(lvl.clone())); + } + } + + if group_by.column().is_some() { + sql.push_str(" GROUP BY bucket, grp ORDER BY bucket ASC, grp ASC"); + } else { + sql.push_str(" GROUP BY bucket ORDER BY bucket ASC"); + } + + let mut stmt = conn.prepare(&sql)?; + let bind_refs: Vec<&dyn rusqlite::types::ToSql> = bindings.iter().map(|b| b.as_ref()).collect(); + let has_group = group_by.column().is_some(); + let rows = stmt.query_map(rusqlite::params_from_iter(bind_refs.iter().copied()), |r| { + if has_group { + Ok(TimelinePoint { + bucket: r.get(0)?, + group: Some(r.get::<_, String>(1)?), + count: r.get(2)?, + }) + } else { + Ok(TimelinePoint { + bucket: r.get(0)?, + group: None, + count: r.get(1)?, + }) + } + })?; + Ok(rows.collect::>>()?) +} + +/// Rollup-backed implementation of [`timeline`] for hour/day/week/month buckets. +/// +/// Reads `timeline_hourly` (grain = one hour) and re-buckets the stored ISO hour +/// string up to the requested grain via `strftime`. `SUM(event_count)` replaces +/// `COUNT(*)`. The output is identical to the live query for these buckets, +/// modulo bounded staleness (the rollup trails ingest by the refresh cadence and +/// excludes the current partial hour until the next refresh). +/// +/// `app_name` is stored as `''` for null-app rows (the PK column is NOT NULL); +/// when grouping by app_name we project `''` back to `''` so the result +/// matches the live query's `COALESCE(app_name, '')`. +#[allow(clippy::too_many_arguments)] +fn timeline_from_rollup( + pool: &DbPool, + bucket: Bucket, + group_by: TimelineGroupBy, + from: Option<&str>, + to: Option<&str>, + hostname: Option<&str>, + app_name: Option<&str>, + severity_in: Option<&[String]>, +) -> Result> { + let conn = pool.get()?; + // Re-bucket the stored hour string to the requested grain. For Hour this is + // an identity transform; the same strftime format the live path uses. + let mut sql = format!( + "SELECT strftime('{fmt}', bucket) AS b", + fmt = bucket.strftime_format() + ); + let grp_expr = group_by.column().map(|col| { + if col == "app_name" { + "COALESCE(NULLIF(app_name, ''), '')" + } else { + col + } + }); + if let Some(expr) = grp_expr { + sql.push_str(&format!(", {expr} AS grp")); + } + sql.push_str(", SUM(event_count) FROM timeline_hourly WHERE 1=1"); + + let mut bindings: Vec> = Vec::new(); + let mut idx = 1usize; + + // from/to filter on the hour bucket. Floor `from` to its hour so the partial + // boundary hour is included (matches the live query's behavior of bucketing + // any in-window row into its hour). `to` compares directly: a bucket is the + // hour floor, so `bucket <= to` keeps every hour at or before `to`. + if let Some(f) = from { + sql.push_str(&format!( + " AND bucket >= strftime('{fmt}', ?{idx})", + fmt = Bucket::Hour.strftime_format() + )); + bindings.push(Box::new(f.to_string())); + idx += 1; + } + if let Some(t) = to { + sql.push_str(&format!(" AND bucket <= ?{idx}")); + bindings.push(Box::new(t.to_string())); + idx += 1; + } + if let Some(h) = hostname { + sql.push_str(&format!(" AND hostname = ?{idx}")); + bindings.push(Box::new(h.to_string())); + idx += 1; + } + if let Some(a) = app_name { + sql.push_str(&format!(" AND app_name = ?{idx}")); + bindings.push(Box::new(a.to_string())); + idx += 1; + } + if let Some(levels) = severity_in + && !levels.is_empty() + { + let placeholders: Vec = + (0..levels.len()).map(|i| format!("?{}", idx + i)).collect(); + sql.push_str(&format!(" AND severity IN ({})", placeholders.join(", "))); + for lvl in levels { + bindings.push(Box::new(lvl.clone())); + } + } + + if grp_expr.is_some() { + sql.push_str(" GROUP BY b, grp ORDER BY b ASC, grp ASC"); + } else { + sql.push_str(" GROUP BY b ORDER BY b ASC"); + } + + let mut stmt = conn.prepare(&sql)?; + let bind_refs: Vec<&dyn rusqlite::types::ToSql> = bindings.iter().map(|b| b.as_ref()).collect(); + let has_group = grp_expr.is_some(); + let rows = stmt.query_map(rusqlite::params_from_iter(bind_refs.iter().copied()), |r| { + if has_group { + Ok(TimelinePoint { + bucket: r.get(0)?, + group: Some(r.get::<_, String>(1)?), + count: r.get(2)?, + }) + } else { + Ok(TimelinePoint { + bucket: r.get(0)?, + group: None, + count: r.get(1)?, + }) + } + })?; + Ok(rows.collect::>>()?) +} + +// ----------------------------------------------------------------------------- +// patterns: cluster near-duplicate messages by template normalization +// ----------------------------------------------------------------------------- + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct PatternEntry { + pub template: String, + pub count: i64, + pub host_count: i64, + pub sample: String, + pub first_seen: String, + pub last_seen: String, + /// Up to 5 hostnames where this template was seen. + pub hostnames: Vec, +} + +pub const PATTERN_SCAN_LIMIT_MAX: u32 = 10_000; + +#[derive(Debug, Clone)] +pub(crate) struct PatternSourceRow { + timestamp: String, + hostname: String, + message: String, +} + +fn split_csv(value: Option) -> Vec { + value + .unwrap_or_default() + .split(',') + .filter(|item| !item.is_empty()) + .map(ToString::to_string) + .collect() +} + +/// Fetch and cluster bounded log patterns without exposing storage rows. +/// +/// The adapter keeps the intermediate SQLite projection private and returns +/// only transport-neutral pattern entries plus the scanned-row count and +/// truncation flag required by application callers. +#[allow(clippy::too_many_arguments)] +pub fn fetch_patterns( + pool: &DbPool, + from: Option<&str>, + to: Option<&str>, + hostname: Option<&str>, + app_name: Option<&str>, + severity_in: Option<&[String]>, + scan_limit: u32, + top_n: u32, +) -> Result<(Vec, i64, bool)> { + let (rows, truncated) = + fetch_pattern_rows(pool, from, to, hostname, app_name, severity_in, scan_limit)?; + let (patterns, scanned) = cluster_pattern_rows(rows, top_n); + Ok((patterns, scanned, truncated)) +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn fetch_pattern_rows( + pool: &DbPool, + from: Option<&str>, + to: Option<&str>, + hostname: Option<&str>, + app_name: Option<&str>, + severity_in: Option<&[String]>, + scan_limit: u32, +) -> Result<(Vec, bool)> { + let conn = pool.get()?; + let (sql, bindings, scan_limit) = + pattern_rows_sql(from, to, hostname, app_name, severity_in, scan_limit); + + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map(rusqlite::params_from_iter(bindings.iter()), |r| { + Ok(PatternSourceRow { + timestamp: r.get::<_, String>(0)?, + hostname: r.get::<_, String>(1)?, + message: r.get::<_, String>(2)?, + }) + })?; + let mut rows = rows.collect::>>()?; + let overflow = rows.len() > scan_limit as usize; + rows.truncate(scan_limit as usize); + Ok((rows, overflow)) +} + +fn pattern_rows_sql( + from: Option<&str>, + to: Option<&str>, + hostname: Option<&str>, + app_name: Option<&str>, + severity_in: Option<&[String]>, + scan_limit: u32, +) -> (String, Vec, u32) { + let scan_limit = scan_limit.clamp(1, PATTERN_SCAN_LIMIT_MAX); + + let mut sql = String::from("SELECT timestamp, hostname, message FROM logs WHERE 1=1"); + let mut bindings: Vec = Vec::new(); + let mut idx = 1usize; + if let Some(f) = from { + sql.push_str(&format!(" AND timestamp >= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(f.to_string())); + idx += 1; + } + if let Some(t) = to { + sql.push_str(&format!(" AND timestamp <= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(t.to_string())); + idx += 1; + } + if let Some(h) = hostname { + sql.push_str(&format!(" AND hostname = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(h.to_string())); + idx += 1; + } + if let Some(a) = app_name { + sql.push_str(&format!(" AND app_name = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(a.to_string())); + idx += 1; + } + if let Some(levels) = severity_in + && !levels.is_empty() + { + let placeholders: Vec = + (0..levels.len()).map(|i| format!("?{}", idx + i)).collect(); + sql.push_str(&format!(" AND severity IN ({})", placeholders.join(", "))); + for lvl in levels { + bindings.push(rusqlite::types::Value::Text(lvl.clone())); + idx += 1; + } + } + // Cap rows scanned to bound CPU/memory — we ask for one extra to detect truncation. + sql.push_str(&format!(" ORDER BY timestamp DESC LIMIT ?{idx}")); + bindings.push(rusqlite::types::Value::Integer(i64::from(scan_limit) + 1)); + (sql, bindings, scan_limit) +} + +pub(crate) fn cluster_pattern_rows( + rows: Vec, + top_n: u32, +) -> (Vec, i64) { + struct Acc { + count: i64, + sample: String, + first_seen: String, + last_seen: String, + hosts: BTreeMap, + } + let mut by_template: BTreeMap = BTreeMap::new(); + for row in rows { + let PatternSourceRow { + timestamp, + hostname, + message, + } = row; + let template = cortex_ingest_core::normalize::normalize_template(&message); + let entry = by_template.entry(template).or_insert_with(|| Acc { + count: 0, + sample: message.clone(), + first_seen: timestamp.clone(), + last_seen: timestamp.clone(), + hosts: BTreeMap::new(), + }); + entry.count += 1; + if timestamp < entry.first_seen { + entry.first_seen = timestamp.clone(); + } + if timestamp > entry.last_seen { + entry.last_seen = timestamp; + } + *entry.hosts.entry(hostname).or_insert(0) += 1; + } + + let total_scanned = by_template.values().map(|entry| entry.count).sum(); + let mut out: Vec = by_template + .into_iter() + .map(|(template, acc)| { + let mut hosts: Vec<(String, i64)> = acc.hosts.into_iter().collect(); + hosts.sort_by_key(|(_, count)| Reverse(*count)); + let host_count = hosts.len() as i64; + let hostnames: Vec = hosts.into_iter().take(5).map(|(h, _)| h).collect(); + PatternEntry { + template, + count: acc.count, + host_count, + sample: acc.sample, + first_seen: acc.first_seen, + last_seen: acc.last_seen, + hostnames, + } + }) + .collect(); + out.sort_by_key(|entry| Reverse(entry.count)); + out.truncate(top_n as usize); + (out, total_scanned) +} + +// ----------------------------------------------------------------------------- +// context: surrounding logs for a single point of interest +// ----------------------------------------------------------------------------- + +pub struct ContextRef { + /// `Some(id)` anchors with stable (timestamp, id) tiebreaking; `None` means + /// the caller only has a timestamp (e.g. `context` invoked with + /// `hostname` + `timestamp`), in which case the query splits cleanly on + /// `< timestamp` / `> timestamp`. + pub id: Option, + pub hostname: String, + pub timestamp: String, +} + +pub fn fetch_log_by_id(pool: &DbPool, id: i64) -> Result> { + let conn = pool.get()?; + let mut stmt = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, + app_name, process_id, message, raw, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs WHERE id = ?1", + )?; + let mut rows = stmt.query(params![id])?; + if let Some(r) = rows.next()? { + Ok(Some(map_row_with_raw(r)?)) + } else { + Ok(None) + } +} + +/// Return an ascending, lossless log feed for external consumers. +/// +/// An omitted cursor establishes a high-water mark without replaying retained +/// history. Passing `Some(0)` explicitly replays from the beginning. +pub fn feed_logs( + pool: &DbPool, + after_id: Option, + hostname: Option<&str>, + limit: u32, +) -> Result<(Vec, i64, bool)> { + let conn = pool.get()?; + let after_id = match after_id { + Some(id) => id.max(0), + None => conn.query_row("SELECT COALESCE(MAX(id), 0) FROM logs", [], |row| { + row.get(0) + })?, + }; + let high_water: i64 = conn.query_row("SELECT COALESCE(MAX(id), 0) FROM logs", [], |row| { + row.get(0) + })?; + let fetch_limit = i64::from(limit.clamp(1, 1_000)) + 1; + + let sql = if hostname.is_some() { + "SELECT id, timestamp, hostname, facility, severity, + app_name, process_id, message, raw, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs WHERE id > ?1 AND hostname = ?2 + ORDER BY id ASC LIMIT ?3" + } else { + "SELECT id, timestamp, hostname, facility, severity, + app_name, process_id, message, raw, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs WHERE id > ?1 + ORDER BY id ASC LIMIT ?2" + }; + + let mut stmt = conn.prepare(sql)?; + let mut logs = if let Some(hostname) = hostname { + stmt.query_map(params![after_id, hostname, fetch_limit], map_row_with_raw)? + .collect::>>()? + } else { + stmt.query_map(params![after_id, fetch_limit], map_row_with_raw)? + .collect::>>()? + }; + + let has_more = logs.len() > limit as usize; + if has_more { + logs.truncate(limit as usize); + } + let returned_high_water = logs.last().map_or(after_id, |log| log.id); + let next_after_id = if has_more { + returned_high_water + } else { + high_water.max(returned_high_water) + }; + Ok((logs, next_after_id, has_more)) +} + +pub fn context_around( + pool: &DbPool, + reference: &ContextRef, + before: u32, + after: u32, +) -> Result<(Vec, Vec)> { + let conn = pool.get()?; + let before = before.min(500); + let after = after.min(500); + + let (mut before_rows, after_rows) = match reference.id { + Some(id) => { + // ID-anchored: stable (timestamp, id) tiebreaker — symmetrical because + // we know exactly which row at `timestamp` is the reference. + let mut before_stmt = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, + app_name, process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs + WHERE hostname = ?1 + AND (timestamp < ?2 OR (timestamp = ?2 AND id < ?3)) + ORDER BY timestamp DESC, id DESC + LIMIT ?4", + )?; + let before_rows = before_stmt + .query_map( + params![reference.hostname, reference.timestamp, id, before], + super::queries::map_row, + )? + .collect::>>()?; + + let mut after_stmt = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, + app_name, process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs + WHERE hostname = ?1 + AND (timestamp > ?2 OR (timestamp = ?2 AND id > ?3)) + ORDER BY timestamp ASC, id ASC + LIMIT ?4", + )?; + let after_rows = after_stmt + .query_map( + params![reference.hostname, reference.timestamp, id, after], + super::queries::map_row, + )? + .collect::>>()?; + (before_rows, after_rows) + } + None => { + // Timestamp-anchored: no row identity, so split strictly on the + // timestamp boundary. Rows that share the exact reference timestamp + // are excluded from both sides rather than dumped onto one — + // symmetry over completeness. + let mut before_stmt = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, + app_name, process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs + WHERE hostname = ?1 AND timestamp < ?2 + ORDER BY timestamp DESC, id DESC + LIMIT ?3", + )?; + let before_rows = before_stmt + .query_map( + params![reference.hostname, reference.timestamp, before], + super::queries::map_row, + )? + .collect::>>()?; + + let mut after_stmt = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, + app_name, process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs + WHERE hostname = ?1 AND timestamp > ?2 + ORDER BY timestamp ASC, id ASC + LIMIT ?3", + )?; + let after_rows = after_stmt + .query_map( + params![reference.hostname, reference.timestamp, after], + super::queries::map_row, + )? + .collect::>>()?; + (before_rows, after_rows) + } + }; + + // Reverse the "before" rows so the result reads chronologically. + before_rows.reverse(); + + Ok((before_rows, after_rows)) +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct LogEntryWithRaw { + pub id: i64, + pub timestamp: String, + pub hostname: String, + pub facility: Option, + pub severity: String, + pub app_name: Option, + pub process_id: Option, + pub message: String, + pub raw: String, + pub received_at: String, + pub source_ip: String, + pub ai_tool: Option, + pub ai_project: Option, + pub ai_session_id: Option, + pub ai_transcript_path: Option, + pub metadata_json: Option, +} + +// ----------------------------------------------------------------------------- +// ingest_rate: throughput over the last 1m / 5m / 15m windows +// ----------------------------------------------------------------------------- + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct IngestRateBuckets { + pub last_1m: i64, + pub last_5m: i64, + pub last_15m: i64, + pub per_sec_1m: f64, + pub per_sec_5m: f64, + pub per_sec_15m: f64, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct IngestRatePerHost { + pub hostname: String, + pub last_1m: i64, + pub last_5m: i64, + pub last_15m: i64, +} + +pub fn ingest_rate( + pool: &DbPool, + now: &str, + cut_1m: &str, + cut_5m: &str, + cut_15m: &str, +) -> Result { + let conn = pool.get()?; + let row: (i64, i64, i64) = conn.query_row( + "SELECT + SUM(CASE WHEN received_at >= ?1 THEN 1 ELSE 0 END), + SUM(CASE WHEN received_at >= ?2 THEN 1 ELSE 0 END), + SUM(CASE WHEN received_at >= ?3 THEN 1 ELSE 0 END) + FROM logs + WHERE received_at >= ?3 AND received_at <= ?4", + params![cut_1m, cut_5m, cut_15m, now], + |r| { + Ok(( + r.get::<_, Option>(0)?.unwrap_or(0), + r.get::<_, Option>(1)?.unwrap_or(0), + r.get::<_, Option>(2)?.unwrap_or(0), + )) + }, + )?; + Ok(IngestRateBuckets { + last_1m: row.0, + last_5m: row.1, + last_15m: row.2, + per_sec_1m: row.0 as f64 / 60.0, + per_sec_5m: row.1 as f64 / 300.0, + per_sec_15m: row.2 as f64 / 900.0, + }) +} + +pub fn ingest_rate_by_host( + pool: &DbPool, + now: &str, + cut_1m: &str, + cut_5m: &str, + cut_15m: &str, +) -> Result> { + let conn = pool.get()?; + let mut stmt = conn.prepare( + "SELECT hostname, + SUM(CASE WHEN received_at >= ?1 THEN 1 ELSE 0 END), + SUM(CASE WHEN received_at >= ?2 THEN 1 ELSE 0 END), + COUNT(*) + FROM logs + WHERE received_at >= ?3 AND received_at <= ?4 + GROUP BY hostname + ORDER BY COUNT(*) DESC", + )?; + let rows = stmt.query_map(params![cut_1m, cut_5m, cut_15m, now], |r| { + Ok(IngestRatePerHost { + hostname: r.get(0)?, + last_1m: r.get::<_, Option>(1)?.unwrap_or(0), + last_5m: r.get::<_, Option>(2)?.unwrap_or(0), + last_15m: r.get(3)?, + }) + })?; + Ok(rows.collect::>>()?) +} + +// ----------------------------------------------------------------------------- +// silent_hosts: hosts whose last_seen is older than a threshold +// ----------------------------------------------------------------------------- + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct SilentHostEntry { + pub hostname: String, + pub first_seen: String, + pub last_seen: String, + pub log_count: i64, + /// Approx seconds between log arrivals over the host's full history. + pub typical_interval_secs: Option, + /// Seconds since last log was received. + pub silent_for_secs: i64, +} + +pub fn silent_hosts(pool: &DbPool, cutoff: &str, now_unix: i64) -> Result> { + // Route through list_hosts so case/FQDN variants of one machine are merged + // (taking the latest last_seen) BEFORE applying the silence cutoff. Reading + // the raw `hosts` table here would flag a dormant `backuphost` identity as silent + // while the live `BACKUPHOST` keeps forwarding — the merged host is correctly + // considered alive. + let mut out: Vec = super::queries::list_hosts(pool)? + .into_iter() + .filter(|h| h.last_seen.as_str() < cutoff) + .map(|h| { + let typical_interval_secs = compute_interval(&h.first_seen, &h.last_seen, h.log_count); + let silent_for_secs = chrono::DateTime::parse_from_rfc3339(&h.last_seen) + .map(|dt| now_unix - dt.timestamp()) + .unwrap_or(0); + SilentHostEntry { + hostname: h.hostname, + first_seen: h.first_seen, + last_seen: h.last_seen, + log_count: h.log_count, + typical_interval_secs, + silent_for_secs, + } + }) + .collect(); + // Longest-silent first (oldest last_seen), preserving the prior ORDER BY ASC. + out.sort_by(|a, b| a.last_seen.cmp(&b.last_seen)); + Ok(out) +} + +fn compute_interval(first: &str, last: &str, count: i64) -> Option { + if count < 2 { + return None; + } + let f = chrono::DateTime::parse_from_rfc3339(first).ok()?; + let l = chrono::DateTime::parse_from_rfc3339(last).ok()?; + let span = (l - f).num_seconds() as f64; + if span <= 0.0 { + return None; + } + Some(span / (count - 1) as f64) +} + +// ----------------------------------------------------------------------------- +// clock_skew: per-host distribution of received_at - timestamp +// ----------------------------------------------------------------------------- + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct ClockSkewEntry { + pub hostname: String, + pub samples: i64, + pub avg_skew_secs: f64, + pub min_skew_secs: f64, + pub max_skew_secs: f64, +} + +// Per-raw-hostname skew stats. Ordering/limit and the case/FQDN variant merge +// happen in Rust (see `clock_skew`) so we must fetch every host here, not a +// SQL-LIMIT-ed prefix that could split a machine's variants across the cutoff. +const CLOCK_SKEW_SQL: &str = " + SELECT hostname, + COUNT(*), + AVG((julianday(received_at) - julianday(timestamp)) * 86400), + MIN((julianday(received_at) - julianday(timestamp)) * 86400), + MAX((julianday(received_at) - julianday(timestamp)) * 86400) + FROM logs INDEXED BY idx_logs_received_at + WHERE received_at >= ?1 + GROUP BY hostname"; + +pub fn clock_skew(pool: &DbPool, since: &str, limit: Option) -> Result> { + let conn = pool.get()?; + let mut stmt = conn.prepare(CLOCK_SKEW_SQL)?; + let rows: Vec = stmt + .query_map(params![since], |r| { + Ok(ClockSkewEntry { + hostname: r.get(0)?, + samples: r.get(1)?, + avg_skew_secs: r.get::<_, Option>(2)?.unwrap_or(0.0), + min_skew_secs: r.get::<_, Option>(3)?.unwrap_or(0.0), + max_skew_secs: r.get::<_, Option>(4)?.unwrap_or(0.0), + }) + })? + .collect::>>()?; + + // Merge case/FQDN variants of one machine (e.g. `BACKUPHOST`/`backuphost`) into a single + // skew row: sum samples, recombine the mean as a sample-weighted average, and + // widen min/max. Without this a host's clock skew is reported once per casing. + let names: Vec = rows.iter().map(|r| r.hostname.clone()).collect(); + let canon = super::queries::canonical_host_keys(&names); + let mut merged: std::collections::HashMap = + std::collections::HashMap::new(); + let mut order: Vec = Vec::new(); + for row in rows { + let key = canon + .get(&row.hostname) + .cloned() + .unwrap_or_else(|| row.hostname.clone()); + match merged.get_mut(&key) { + Some(acc) => { + let total = acc.samples + row.samples; + if total > 0 { + acc.avg_skew_secs = (acc.avg_skew_secs * acc.samples as f64 + + row.avg_skew_secs * row.samples as f64) + / total as f64; + } + acc.min_skew_secs = acc.min_skew_secs.min(row.min_skew_secs); + acc.max_skew_secs = acc.max_skew_secs.max(row.max_skew_secs); + acc.samples = total; + } + None => { + order.push(key.clone()); + merged.insert( + key.clone(), + ClockSkewEntry { + hostname: key, + ..row + }, + ); + } + } + } + let mut out: Vec = order + .into_iter() + .map(|k| merged.remove(&k).expect("key inserted above")) + .collect(); + // Largest absolute skew first (preserving the prior ORDER BY). + out.sort_by(|a, b| { + b.avg_skew_secs + .abs() + .partial_cmp(&a.avg_skew_secs.abs()) + .unwrap_or(std::cmp::Ordering::Equal) + }); + if let Some(limit) = limit { + out.truncate(limit as usize); + } + Ok(out) +} + +// ----------------------------------------------------------------------------- +// anomalies: per-host volume / error-rate vs baseline +// ----------------------------------------------------------------------------- + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct AnomalyEntry { + pub hostname: String, + pub recent_count: i64, + pub baseline_count: i64, + pub recent_per_min: f64, + pub baseline_per_min: f64, + /// recent_per_min / baseline_per_min (1.0 means unchanged). `None` when + /// baseline is zero (host is new — flagged separately). + pub ratio: Option, + /// Poisson-style z-score against baseline rate. + pub z_score: Option, + pub recent_errors: i64, + pub baseline_errors: i64, +} + +pub fn anomalies( + pool: &DbPool, + recent_from: &str, + recent_to: &str, + baseline_from: &str, + baseline_to: &str, + recent_minutes: u32, + baseline_minutes: u32, +) -> Result> { + let conn = pool.get()?; + let error_levels = "('emerg','alert','crit','err','warning')"; + + // FULL OUTER JOIN is only available in SQLite ≥ 3.39 — emulate via a + // hosts-union CTE so we still pick up hosts that exist in baseline only. + let sql = format!( + "WITH recent AS ( + SELECT hostname, + COUNT(*) AS c, + SUM(CASE WHEN severity IN {err} THEN 1 ELSE 0 END) AS e + FROM logs + WHERE timestamp >= ?1 AND timestamp <= ?2 + GROUP BY hostname + ), + baseline AS ( + SELECT hostname, + COUNT(*) AS c, + SUM(CASE WHEN severity IN {err} THEN 1 ELSE 0 END) AS e + FROM logs + WHERE timestamp >= ?3 AND timestamp <= ?4 + GROUP BY hostname + ), + all_hosts AS ( + SELECT hostname FROM recent + UNION + SELECT hostname FROM baseline + ) + SELECT a.hostname, + COALESCE(r.c, 0), COALESCE(b.c, 0), + COALESCE(r.e, 0), COALESCE(b.e, 0) + FROM all_hosts a + LEFT JOIN recent r ON r.hostname = a.hostname + LEFT JOIN baseline b ON b.hostname = a.hostname + ORDER BY a.hostname", + err = error_levels + ); + + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map( + params![recent_from, recent_to, baseline_from, baseline_to], + |r| { + let hostname: String = r.get(0)?; + let recent_count: i64 = r.get(1)?; + let baseline_count: i64 = r.get(2)?; + let recent_errors: i64 = r.get(3)?; + let baseline_errors: i64 = r.get(4)?; + Ok(( + hostname, + recent_count, + baseline_count, + recent_errors, + baseline_errors, + )) + }, + )?; + + let recent_minutes = recent_minutes.max(1) as f64; + let baseline_minutes = baseline_minutes.max(1) as f64; + let mut out = Vec::new(); + for row in rows { + let (hostname, recent_count, baseline_count, recent_errors, baseline_errors) = row?; + let recent_per_min = recent_count as f64 / recent_minutes; + let baseline_per_min = baseline_count as f64 / baseline_minutes; + let ratio = if baseline_per_min > 0.0 { + Some(recent_per_min / baseline_per_min) + } else { + None + }; + let expected = baseline_per_min * recent_minutes; + let z_score = if expected > 0.0 { + Some((recent_count as f64 - expected) / expected.sqrt()) + } else { + None + }; + out.push(AnomalyEntry { + hostname, + recent_count, + baseline_count, + recent_per_min, + baseline_per_min, + ratio, + z_score, + recent_errors, + baseline_errors, + }); + } + // Surface new-but-active hosts (`recent_count > 0` against a zero baseline) + // at the top — they have no defined `z_score`, but they are exactly the + // signal the docstring promises. Other unscored entries (e.g. recent zero + // activity, dormant hosts) sink to the bottom. + let sort_key = |e: &AnomalyEntry| -> f64 { + if e.baseline_count == 0 && e.recent_count > 0 { + f64::INFINITY + } else { + e.z_score.unwrap_or(f64::NEG_INFINITY) + } + }; + out.sort_by(|a, b| { + sort_key(b) + .partial_cmp(&sort_key(a)) + .unwrap_or(std::cmp::Ordering::Equal) + }); + Ok(out) +} + +// ----------------------------------------------------------------------------- +// compare: side-by-side diff of two time ranges +// ----------------------------------------------------------------------------- + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct RangeSummary { + pub from: String, + pub to: String, + pub total_logs: i64, + pub total_errors: i64, + pub by_severity: Vec<(String, i64)>, + pub top_hosts: Vec<(String, i64)>, + pub top_apps: Vec<(String, i64)>, +} + +pub fn summarize_range(pool: &DbPool, from: &str, to: &str) -> Result { + let conn = pool.get()?; + // ONE scan of the timestamp partition per range (previously three — + // severity, hosts, apps — so `compare` cost six unbounded scans per call, + // full-review PM4). The grouped cardinality (distinct severity × host × + // app tuples) is small relative to row count; all three breakdowns are + // derived from it in memory. Serving fully from the timeline_hourly + // rollup (O(hours)) is tracked as a follow-up — it needs partial-edge- + // hour reconciliation to stay exact. + let mut stmt = conn.prepare_cached( + "SELECT severity, hostname, app_name, COUNT(*) FROM logs + WHERE timestamp >= ?1 AND timestamp <= ?2 + GROUP BY severity, hostname, app_name", + )?; + let grouped = stmt + .query_map(params![from, to], |r| { + Ok(( + r.get::<_, String>(0)?, + r.get::<_, String>(1)?, + r.get::<_, Option>(2)?, + r.get::<_, i64>(3)?, + )) + })? + .collect::>>()?; + + const ERROR_SEVERITIES: &[&str] = &["emerg", "alert", "crit", "err", "warning"]; + let mut sev_counts: std::collections::HashMap = std::collections::HashMap::new(); + let mut host_counts: std::collections::HashMap = std::collections::HashMap::new(); + let mut app_counts: std::collections::HashMap = std::collections::HashMap::new(); + for (severity, hostname, app_name, count) in grouped { + *host_counts.entry(hostname).or_default() += count; + if let Some(app) = app_name.filter(|a| !a.is_empty()) { + *app_counts.entry(app).or_default() += count; + } + *sev_counts.entry(severity).or_default() += count; + } + + let total_logs: i64 = sev_counts.values().sum(); + let total_errors: i64 = sev_counts + .iter() + .filter(|(sev, _)| ERROR_SEVERITIES.iter().any(|e| e.eq_ignore_ascii_case(sev))) + .map(|(_, n)| n) + .sum(); + + fn sorted_desc(counts: std::collections::HashMap) -> Vec<(String, i64)> { + let mut v: Vec<(String, i64)> = counts.into_iter().collect(); + v.sort_by(|a, b| b.1.cmp(&a.1).then_with(|| a.0.cmp(&b.0))); + v + } + + let by_severity = sorted_desc(sev_counts); + let mut top_hosts = sorted_desc(host_counts); + top_hosts.truncate(10); + let mut top_apps = sorted_desc(app_counts); + top_apps.truncate(10); + + Ok(RangeSummary { + from: from.to_string(), + to: to.to_string(), + total_logs, + total_errors, + by_severity, + top_hosts, + top_apps, + }) +} + +#[cfg(test)] +#[path = "analytics_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/analytics_tests.rs b/crates/shared/cortex/storage-sqlite/src/analytics_tests.rs new file mode 100644 index 00000000..dea9a261 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/analytics_tests.rs @@ -0,0 +1,1523 @@ +use super::*; +use crate::config::StorageConfig; +use crate::{ + DbPool, LogBatchEntry, ingest_source_kind_health, init_pool, insert_logs_batch, + prune_timeline_rollup, refresh_timeline_rollup, timeline_rollup_status, +}; +use cortex_ingest_core::normalize::normalize_template; + +fn test_pool() -> (DbPool, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let pool = init_pool(&StorageConfig::for_test(db_path)).unwrap(); + (pool, dir) +} + +fn entry(ts: &str, host: &str, severity: &str, app: Option<&str>, msg: &str) -> LogBatchEntry { + entry_with_source_ip(ts, host, severity, app, msg, "127.0.0.1:514") +} + +fn entry_with_source_ip( + ts: &str, + host: &str, + severity: &str, + app: Option<&str>, + msg: &str, + source_ip: &str, +) -> LogBatchEntry { + LogBatchEntry { + timestamp: ts.to_string(), + hostname: host.to_string(), + facility: None, + severity: severity.to_string(), + app_name: app.map(String::from), + process_id: None, + message: msg.to_string(), + raw: format!("<14>{ts} {host} {}: {msg}", app.unwrap_or("test")), + source_ip: source_ip.to_string(), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + } +} + +#[test] +fn template_normalises_numbers_ips_uuids() { + let t = normalize_template( + "connection refused from 10.0.0.5:42 (id b3a1c0de-1234-5678-9abc-def012345678)", + ); + assert!(t.contains(":")); + assert!(t.contains("")); +} + +#[test] +fn template_preserves_non_ascii_codepoints() { + // Multi-byte UTF-8 sequences must round-trip rather than getting split into + // mojibake by `b as char`. + let msg = "\u{0444}\u{0430}\u{0439}\u{043b} 1234 \u{043d}\u{0435} \u{043d}\u{0430}\u{0439}\u{0434}\u{0435}\u{043d}"; + let t = normalize_template(msg); + assert!(t.contains("\u{0444}\u{0430}\u{0439}\u{043b}")); + assert!(t.contains("\u{043d}\u{0435} \u{043d}\u{0430}\u{0439}\u{0434}\u{0435}\u{043d}")); + assert!(t.contains("")); + assert!(t.is_char_boundary(t.len())); +} + +#[test] +fn list_apps_returns_distinct_apps_with_counts() { + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[ + entry("2026-01-01T00:00:01Z", "h1", "info", Some("nginx"), "hello"), + entry("2026-01-01T00:00:02Z", "h1", "info", Some("nginx"), "again"), + entry( + "2026-01-01T00:00:03Z", + "h2", + "info", + Some("sshd"), + "auth ok", + ), + ], + ) + .unwrap(); + + let apps = list_apps( + &pool, + &ListAppsParams { + hostname: None, + from: None, + to: None, + limit: 500, + offset: 0, + }, + ) + .unwrap(); + assert_eq!(apps.total, 2); + let nginx = apps.apps.iter().find(|a| a.app_name == "nginx").unwrap(); + assert_eq!(nginx.log_count, 2); + assert_eq!(nginx.host_count, 1); + + // Filter by hostname + let only_h2 = list_apps( + &pool, + &ListAppsParams { + hostname: Some("h2"), + from: None, + to: None, + limit: 500, + offset: 0, + }, + ) + .unwrap(); + assert_eq!(only_h2.apps.len(), 1); + assert_eq!(only_h2.apps[0].app_name, "sshd"); +} + +#[test] +fn ingest_source_kind_health_classifies_recent_sources() { + let (pool, _d) = test_pool(); + let mut syslog = entry( + "2026-01-01T11:58:00Z", + "h1", + "info", + Some("rsyslogd"), + "syslog", + ); + syslog.metadata_json = Some(r#"{"source_kind":"syslog-udp"}"#.to_string()); + + let mut otlp = entry("2026-01-01T11:30:00Z", "h1", "info", Some("otel"), "otlp"); + otlp.metadata_json = Some(r#"{"source_kind":"otlp"}"#.to_string()); + + let docker = entry_with_source_ip( + "2026-01-01T10:30:00Z", + "h2", + "info", + Some("container"), + "docker", + "docker://h2/container/stdout", + ); + + let mut transcript = entry_with_source_ip( + "2026-01-01T11:59:00Z", + "h3", + "info", + Some("codex"), + "transcript", + "transcript://codex", + ); + transcript.ai_transcript_path = Some("/tmp/session.jsonl".to_string()); + + insert_logs_batch(&pool, &[syslog, otlp, docker, transcript]).unwrap(); + pool.get() + .unwrap() + .execute("UPDATE logs SET received_at = timestamp", []) + .unwrap(); + + let rows = ingest_source_kind_health( + &pool, + "2026-01-01T12:00:00Z", + "2026-01-01T11:45:00Z", + "2026-01-01T11:00:00Z", + "2025-12-31T12:00:00Z", + ) + .unwrap(); + let by_kind: std::collections::HashMap<_, _> = rows + .into_iter() + .map(|row| (row.source_kind.clone(), row)) + .collect(); + + assert_eq!(by_kind["syslog-udp"].last_15m, 1); + assert_eq!(by_kind["otlp"].last_15m, 0); + assert_eq!(by_kind["otlp"].last_1h, 1); + assert_eq!(by_kind["docker-stream"].last_1h, 0); + assert_eq!(by_kind["docker-stream"].last_24h, 1); + assert_eq!(by_kind["transcript"].last_15m, 1); +} + +#[test] +fn unfiltered_list_apps_uses_inventory_stats_without_scanning_logs() { + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[ + entry("2026-01-01T00:00:01Z", "h1", "info", Some("nginx"), "hello"), + entry("2026-01-01T00:00:02Z", "h2", "info", Some("nginx"), "again"), + entry( + "2026-01-01T00:00:03Z", + "h2", + "info", + Some("sshd"), + "auth ok", + ), + ], + ) + .unwrap(); + pool.get() + .unwrap() + .execute( + "UPDATE inventory_backfill_state + SET completed_at = '2026-01-01T00:00:00Z' + WHERE name = 'app_source_inventory'", + [], + ) + .unwrap(); + + let conn = pool.get().unwrap(); + let plan = conn + .prepare( + "EXPLAIN QUERY PLAN + WITH page AS ( + SELECT app_name, log_count, first_seen, last_seen + FROM app_inventory_stats + ORDER BY last_seen DESC, app_name ASC + LIMIT 50 OFFSET 0 + ) + SELECT p.app_name, p.log_count, COUNT(h.hostname), p.first_seen, p.last_seen + FROM page p + LEFT JOIN app_host_inventory_stats h ON h.app_name = p.app_name + GROUP BY p.app_name, p.log_count, p.first_seen, p.last_seen + ORDER BY p.last_seen DESC, p.app_name ASC", + ) + .unwrap() + .query_map([], |row| row.get::<_, String>(3)) + .unwrap() + .collect::>>() + .unwrap() + .join("\n"); + assert!( + !plan.contains("logs"), + "unfiltered app inventory should not scan logs; got:\n{plan}" + ); + drop(conn); + + let apps = list_apps( + &pool, + &ListAppsParams { + hostname: None, + from: None, + to: None, + limit: 50, + offset: 0, + }, + ) + .unwrap(); + let nginx = apps.apps.iter().find(|a| a.app_name == "nginx").unwrap(); + assert_eq!(nginx.log_count, 2); + assert_eq!(nginx.host_count, 2); +} + +#[test] +fn list_apps_to_filter_excludes_future_entries() { + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[entry( + "2026-01-01T00:00:01Z", + "h1", + "info", + Some("nginx"), + "msg", + )], + ) + .unwrap(); + + // All entries are inserted with received_at = now(). A `to` in the far past + // should exclude them all; a `to` in the far future should include them. + let none = list_apps( + &pool, + &ListAppsParams { + hostname: None, + from: None, + to: Some("2000-01-01T00:00:00Z"), + limit: 500, + offset: 0, + }, + ) + .unwrap(); + assert!( + none.apps.is_empty(), + "to=2000 should exclude all entries inserted now" + ); + + let all = list_apps( + &pool, + &ListAppsParams { + hostname: None, + from: None, + to: Some("9999-01-01T00:00:00Z"), + limit: 500, + offset: 0, + }, + ) + .unwrap(); + assert!(!all.apps.is_empty(), "to=9999 should include all entries"); +} + +#[test] +fn inventory_stats_decrement_when_logs_are_deleted() { + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[ + entry_with_source_ip( + "2026-01-01T00:00:01Z", + "h1", + "info", + Some("nginx"), + "hello", + "10.0.0.1:514", + ), + entry_with_source_ip( + "2026-01-01T00:00:02Z", + "h2", + "info", + Some("nginx"), + "again", + "10.0.0.1:514", + ), + ], + ) + .unwrap(); + pool.get() + .unwrap() + .execute( + "UPDATE inventory_backfill_state + SET completed_at = '2026-01-01T00:00:00Z' + WHERE name = 'app_source_inventory'", + [], + ) + .unwrap(); + + let conn = pool.get().unwrap(); + conn.execute("DELETE FROM logs WHERE hostname = 'h1'", []) + .unwrap(); + drop(conn); + + let apps = list_apps( + &pool, + &ListAppsParams { + hostname: None, + from: None, + to: None, + limit: 50, + offset: 0, + }, + ) + .unwrap(); + let nginx = apps.apps.iter().find(|a| a.app_name == "nginx").unwrap(); + assert_eq!(nginx.log_count, 1); + assert_eq!(nginx.host_count, 1); + + let source_ips = list_source_ips( + &pool, + &ListSourceIpsParams { + limit: 50, + offset: 0, + }, + ) + .unwrap(); + let ip = source_ips + .source_ips + .iter() + .find(|entry| entry.source_ip == "10.0.0.1:514") + .unwrap(); + assert_eq!(ip.log_count, 1); + assert_eq!(ip.host_count, 1); +} + +#[test] +fn list_source_ips_truncated_when_over_limit() { + let (pool, _d) = test_pool(); + // Insert 3 entries with distinct source IPs; request limit=2 to force truncation. + insert_logs_batch( + &pool, + &[ + entry_with_source_ip( + "2026-01-01T00:00:01Z", + "h1", + "info", + None, + "a", + "10.0.0.1:514", + ), + entry_with_source_ip( + "2026-01-01T00:00:02Z", + "h1", + "info", + None, + "b", + "10.0.0.2:514", + ), + entry_with_source_ip( + "2026-01-01T00:00:03Z", + "h1", + "info", + None, + "c", + "10.0.0.3:514", + ), + ], + ) + .unwrap(); + + let result = list_source_ips( + &pool, + &ListSourceIpsParams { + limit: 2, + offset: 0, + }, + ) + .unwrap(); + assert_eq!(result.total, 3, "total should reflect all 3 distinct IPs"); + assert_eq!( + result.source_ips.len(), + 2, + "page should contain only limit=2 IPs" + ); +} + +#[test] +fn unfiltered_list_source_ips_uses_inventory_stats_without_scanning_logs() { + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[ + entry_with_source_ip("2026-01-01T00:00:01Z", "h1", "info", None, "a", "10.0.0.1"), + entry_with_source_ip("2026-01-01T00:00:02Z", "h2", "info", None, "b", "10.0.0.1"), + entry_with_source_ip("2026-01-01T00:00:03Z", "h3", "info", None, "c", "10.0.0.2"), + ], + ) + .unwrap(); + + let conn = pool.get().unwrap(); + let plan = conn + .prepare( + "EXPLAIN QUERY PLAN + WITH page AS ( + SELECT source_ip, log_count, first_seen, last_seen + FROM source_ip_inventory_stats + ORDER BY log_count DESC, source_ip ASC + LIMIT 50 OFFSET 0 + ) + SELECT p.source_ip, p.log_count, p.first_seen, p.last_seen, + h.hostname, h.log_count, h.first_seen, h.last_seen + FROM page p + LEFT JOIN source_ip_host_inventory_stats h ON h.source_ip = p.source_ip + ORDER BY p.log_count DESC, p.source_ip ASC, h.log_count DESC, h.hostname ASC", + ) + .unwrap() + .query_map([], |row| row.get::<_, String>(3)) + .unwrap() + .collect::>>() + .unwrap() + .join("\n"); + assert!( + !plan.contains("logs"), + "unfiltered source inventory should not scan logs; got:\n{plan}" + ); + drop(conn); + + let result = list_source_ips( + &pool, + &ListSourceIpsParams { + limit: 50, + offset: 0, + }, + ) + .unwrap(); + let ip = result + .source_ips + .iter() + .find(|entry| entry.source_ip == "10.0.0.1") + .unwrap(); + assert_eq!(ip.log_count, 2); + assert_eq!(ip.host_count, 2); +} + +#[test] +fn list_source_ips_chatty_ip_does_not_suppress_others() { + // One IP with many hostnames must not crowd out other distinct IPs. + let (pool, _d) = test_pool(); + let mut entries = vec![]; + // ip1 logs from 20 different hostnames + for i in 0..20 { + entries.push(entry_with_source_ip( + "2026-01-01T00:00:01Z", + &format!("host-{i}"), + "info", + None, + "msg", + "10.0.0.1:514", + )); + } + // ip2 logs once + entries.push(entry_with_source_ip( + "2026-01-01T00:00:02Z", + "h2", + "info", + None, + "msg", + "10.0.0.2:514", + )); + insert_logs_batch(&pool, &entries).unwrap(); + + let result = list_source_ips( + &pool, + &ListSourceIpsParams { + limit: 500, + offset: 0, + }, + ) + .unwrap(); + assert_eq!(result.total, 2); + assert!( + result + .source_ips + .iter() + .any(|e| e.source_ip == "10.0.0.2:514"), + "ip2 must appear even though ip1 has many hostnames" + ); +} + +#[test] +fn timeline_buckets_by_hour() { + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[ + entry("2026-01-01T00:10:00Z", "h1", "info", None, "a"), + entry("2026-01-01T00:50:00Z", "h1", "info", None, "b"), + entry("2026-01-01T01:05:00Z", "h1", "info", None, "c"), + ], + ) + .unwrap(); + // hour/day/week/month read the timeline_hourly rollup; populate it first + // (the background task does this in prod). + refresh_timeline_rollup(&pool).unwrap(); + let pts = timeline( + &pool, + Bucket::Hour, + TimelineGroupBy::None, + None, + None, + None, + None, + None, + ) + .unwrap(); + assert_eq!(pts.len(), 2); + assert_eq!(pts[0].count, 2); + assert_eq!(pts[1].count, 1); +} + +#[test] +fn patterns_clusters_by_template() { + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[ + entry( + "2026-01-01T00:00:01Z", + "h1", + "err", + None, + "disk 1234 failed", + ), + entry( + "2026-01-01T00:00:02Z", + "h1", + "err", + None, + "disk 9999 failed", + ), + entry("2026-01-01T00:00:03Z", "h2", "err", None, "disk 5 failed"), + entry("2026-01-01T00:00:04Z", "h1", "info", None, "all good"), + ], + ) + .unwrap(); + let (pats, scanned, truncated) = + fetch_patterns(&pool, None, None, None, None, None, 100, 10).unwrap(); + assert!(!truncated); + assert_eq!(scanned, 4); + let top = &pats[0]; + assert_eq!(top.count, 3); + assert_eq!(top.host_count, 2); +} + +#[test] +fn fetch_pattern_rows_limit_is_bound_and_clamped() { + let (sql, bindings, scan_limit) = pattern_rows_sql( + Some("2026-01-01T00:00:00Z"), + Some("2026-01-01T01:00:00Z"), + Some("h1"), + Some("sshd"), + Some(&["err".to_string(), "warning".to_string()]), + PATTERN_SCAN_LIMIT_MAX + 1, + ); + assert_eq!(scan_limit, PATTERN_SCAN_LIMIT_MAX); + assert!( + sql.contains("LIMIT ?"), + "fetch_pattern_rows must bind LIMIT instead of interpolating it: {sql}" + ); + assert!( + bindings.iter().any(|value| { + matches!( + value, + rusqlite::types::Value::Integer(limit) + if *limit == i64::from(PATTERN_SCAN_LIMIT_MAX + 1) + ) + }), + "fetch_pattern_rows should bind scan_limit+1 for truncation detection, got: {bindings:?}" + ); +} + +#[test] +fn fetch_log_by_id_returns_raw() { + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[entry("2026-01-01T00:00:01Z", "h1", "info", None, "hello")], + ) + .unwrap(); + let row = fetch_log_by_id(&pool, 1).unwrap().unwrap(); + assert_eq!(row.message, "hello"); + assert!(row.raw.contains("hello")); +} + +#[test] +fn context_around_returns_neighbours() { + let (pool, _d) = test_pool(); + let mut entries = Vec::new(); + for i in 0..10 { + entries.push(entry( + &format!("2026-01-01T00:00:{:02}Z", i), + "h1", + "info", + None, + &format!("msg {i}"), + )); + } + insert_logs_batch(&pool, &entries).unwrap(); + let r = ContextRef { + id: Some(5), + hostname: "h1".to_string(), + timestamp: "2026-01-01T00:00:04Z".to_string(), + }; + let (before, after) = context_around(&pool, &r, 3, 3).unwrap(); + assert_eq!(before.len(), 3); + assert_eq!(after.len(), 3); + assert!(before.last().unwrap().timestamp.as_str() < "2026-01-01T00:00:04Z"); + assert!(after.first().unwrap().timestamp.as_str() > "2026-01-01T00:00:04Z"); +} + +#[test] +fn context_timestamp_only_anchor_splits_symmetrically() { + // Two rows share the exact reference timestamp; with id=None they must not + // all land on one side. The before/after split is strict on `< ts` / `> ts`, + // so simultaneous rows are excluded from both — consistent regardless of id ordering. + let (pool, _d) = test_pool(); + let mut entries = Vec::new(); + for i in 0..5 { + entries.push(entry( + &format!("2026-01-01T00:00:{:02}Z", i), + "h1", + "info", + None, + "msg", + )); + } + // Two rows at the exact reference time. + entries.push(entry("2026-01-01T00:00:05Z", "h1", "info", None, "ref-a")); + entries.push(entry("2026-01-01T00:00:05Z", "h1", "info", None, "ref-b")); + for i in 6..10 { + entries.push(entry( + &format!("2026-01-01T00:00:{:02}Z", i), + "h1", + "info", + None, + "msg", + )); + } + insert_logs_batch(&pool, &entries).unwrap(); + + let r = ContextRef { + id: None, + hostname: "h1".to_string(), + timestamp: "2026-01-01T00:00:05Z".to_string(), + }; + let (before, after) = context_around(&pool, &r, 10, 10).unwrap(); + // 5 strictly-less timestamps, 4 strictly-greater. Neither contains a row at 05. + assert_eq!(before.len(), 5); + assert_eq!(after.len(), 4); + assert!( + before + .iter() + .all(|r| r.timestamp.as_str() < "2026-01-01T00:00:05Z") + ); + assert!( + after + .iter() + .all(|r| r.timestamp.as_str() > "2026-01-01T00:00:05Z") + ); +} + +fn ai_entry(ts: &str, tool: &str, project: &str, session_id: &str, message: &str) -> LogBatchEntry { + LogBatchEntry { + timestamp: ts.to_string(), + hostname: "host-a".to_string(), + facility: Some("local0".to_string()), + severity: "info".to_string(), + app_name: Some("ai-transcript".to_string()), + process_id: None, + message: message.to_string(), + raw: message.to_string(), + source_ip: "127.0.0.1:514".to_string(), + docker_checkpoint: None, + ai_tool: Some(tool.to_string()), + ai_project: Some(project.to_string()), + ai_session_id: Some(session_id.to_string()), + ai_transcript_path: Some(format!("{project}/{session_id}.jsonl")), + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + } +} + +#[test] +fn usage_blocks_group_into_five_hour_windows() { + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[ + ai_entry( + "2026-01-01T00:00:00Z", + "claude", + "/tmp/project", + "sess-1", + "one", + ), + ai_entry( + "2026-01-01T04:59:59Z", + "claude", + "/tmp/project", + "sess-1", + "two", + ), + ai_entry( + "2026-01-01T05:00:00Z", + "claude", + "/tmp/project", + "sess-2", + "three", + ), + ], + ) + .unwrap(); + + let result = get_ai_usage_blocks( + &pool, + &AiUsageBlocksParams { + since: Some("2026-01-01T00:00:00Z".into()), + until: Some("2026-01-01T06:00:00Z".into()), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(result.blocks.len(), 2); + assert_eq!(result.blocks[0].event_count, 1); + assert_eq!(result.blocks[1].event_count, 2); +} + +#[test] +fn usage_blocks_honors_requested_limit() { + let (pool, _d) = test_pool(); + for i in 0..3 { + insert_logs_batch( + &pool, + &[ai_entry( + "2026-01-01T00:00:00Z", + "claude", + &format!("/tmp/project-{i}"), + &format!("sess-{i}"), + "usage block", + )], + ) + .unwrap(); + } + + let result = get_ai_usage_blocks( + &pool, + &AiUsageBlocksParams { + since: Some("2026-01-01T00:00:00Z".into()), + until: Some("2026-01-01T01:00:00Z".into()), + limit: Some(2), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.blocks.len(), 2); + assert_eq!(result.total_blocks, 2); + assert!(result.truncated); +} + +#[test] +fn usage_blocks_total_blocks_equals_len_when_truncated() { + // When truncated, total_blocks == blocks.len() (the limit); truncated flag + // is the authoritative indicator that more groups exist. + let (pool, _d) = test_pool(); + let mut entries = Vec::new(); + for i in 0..1002 { + entries.push(ai_entry( + "2026-01-01T00:00:00Z", + "claude", + &format!("/tmp/project-{i}"), + &format!("sess-{i}"), + "usage block", + )); + } + insert_logs_batch(&pool, &entries).unwrap(); + + let result = get_ai_usage_blocks( + &pool, + &AiUsageBlocksParams { + since: Some("2026-01-01T00:00:00Z".into()), + until: Some("2026-07-31T00:00:00Z".into()), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.blocks.len(), 1000); + assert_eq!(result.total_blocks, 1000); + assert!(result.truncated); +} + +#[test] +fn project_context_returns_recent_entries() { + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[ + ai_entry( + "2026-01-01T00:00:00Z", + "claude", + "/tmp/project", + "sess-1", + "one", + ), + ai_entry( + "2026-01-01T00:01:00Z", + "claude", + "/tmp/project", + "sess-2", + "two", + ), + ], + ) + .unwrap(); + let result = get_ai_project_context( + &pool, + &AiProjectContextParams { + project: "/tmp/project".into(), + limit: Some(1), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(result.project, "/tmp/project"); + assert_eq!(result.event_count, 2); + assert_eq!(result.recent_entries.len(), 1); +} + +#[test] +fn project_context_snippets_are_bounded_to_256_chars() { + let (pool, _d) = test_pool(); + let long_message = "a".repeat(300); + insert_logs_batch( + &pool, + &[ai_entry( + "2026-01-01T00:00:00Z", + "claude", + "/tmp/project", + "sess-1", + &long_message, + )], + ) + .unwrap(); + + let result = get_ai_project_context( + &pool, + &AiProjectContextParams { + project: "/tmp/project".into(), + limit: Some(1), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.recent_entries[0].message.chars().count(), 256); +} + +#[test] +fn clock_skew_plan_uses_received_at_range_index() { + let (pool, _d) = test_pool(); + let conn = pool.get().unwrap(); + let sql = format!("EXPLAIN QUERY PLAN {CLOCK_SKEW_SQL}"); + let mut stmt = conn.prepare(&sql).unwrap(); + let rows = stmt + .query_map(rusqlite::params!["2026-01-01T00:00:00Z"], |row| { + row.get::<_, String>(3) + }) + .unwrap() + .collect::>>() + .unwrap(); + let plan = rows.join("\n"); + + assert!( + plan.contains("idx_logs_received_at"), + "clock_skew must range-scan received_at; got:\n{plan}" + ); + assert!( + !plan.contains("SCAN logs USING INDEX idx_logs_hostname"), + "clock_skew must not scan the hostname index for grouped recent windows; got:\n{plan}" + ); +} + +#[test] +fn clock_skew_limits_hosts_in_skew_order() { + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[ + entry("2026-01-01T00:00:00Z", "h-low", "info", None, "low"), + entry("2026-01-01T00:00:00Z", "h-high", "info", None, "high"), + entry("2026-01-01T00:00:00Z", "h-mid", "info", None, "mid"), + ], + ) + .unwrap(); + + let conn = pool.get().unwrap(); + for (host, received_at) in [ + ("h-low", "2026-01-01T00:00:10Z"), + ("h-high", "2026-01-01T00:10:00Z"), + ("h-mid", "2026-01-01T00:01:00Z"), + ] { + conn.execute( + "UPDATE logs SET received_at = ?1 WHERE hostname = ?2", + rusqlite::params![received_at, host], + ) + .unwrap(); + } + drop(conn); + + let result = clock_skew(&pool, "2026-01-01T00:00:00Z", Some(2)).unwrap(); + + assert_eq!(result.len(), 2); + assert_eq!(result[0].hostname, "h-high"); + assert_eq!(result[1].hostname, "h-mid"); +} + +#[test] +fn summarize_range_counts_errors() { + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[ + entry("2026-01-01T00:00:01Z", "h1", "err", Some("a"), "x"), + entry("2026-01-01T00:00:02Z", "h1", "info", Some("a"), "y"), + entry("2026-01-01T00:00:03Z", "h2", "warning", Some("b"), "z"), + ], + ) + .unwrap(); + let summary = summarize_range(&pool, "2026-01-01T00:00:00Z", "2026-01-01T00:00:10Z").unwrap(); + assert_eq!(summary.total_logs, 3); + assert_eq!(summary.total_errors, 2); + assert_eq!(summary.top_apps.len(), 2); +} + +#[test] +fn list_source_ips_aggregates_hostnames() { + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[ + entry_with_source_ip( + "2026-01-01T00:00:01Z", + "h1", + "info", + None, + "x", + "10.0.0.1:514", + ), + entry_with_source_ip( + "2026-01-01T00:00:02Z", + "h2", + "info", + None, + "x", + "10.0.0.1:514", + ), + entry_with_source_ip( + "2026-01-01T00:00:03Z", + "h2", + "info", + None, + "x", + "10.0.0.1:514", + ), + entry_with_source_ip( + "2026-01-01T00:00:04Z", + "h3", + "info", + None, + "x", + "10.0.0.2:514", + ), + ], + ) + .unwrap(); + let result = list_source_ips( + &pool, + &ListSourceIpsParams { + limit: 500, + offset: 0, + }, + ) + .unwrap(); + assert_eq!(result.total, 2); + let first = result + .source_ips + .iter() + .find(|e| e.source_ip == "10.0.0.1:514") + .unwrap(); + assert_eq!(first.host_count, 2); + assert_eq!(first.log_count, 3); +} + +#[test] +fn bucket_week_formats_correctly() { + // Week bucket uses ISO week number format: "YYYY-WNN" + assert_eq!(Bucket::Week.strftime_format(), "%Y-W%W"); + assert_eq!(Bucket::parse("week"), Some(Bucket::Week)); + assert_eq!(Bucket::parse("w"), Some(Bucket::Week)); +} + +#[test] +fn bucket_month_formats_correctly() { + // Month bucket uses year-month format: "YYYY-MM" + assert_eq!(Bucket::Month.strftime_format(), "%Y-%m"); + assert_eq!(Bucket::parse("month"), Some(Bucket::Month)); +} + +#[test] +fn bucket_default_lookback_days_scales_with_bucket_size() { + assert!(Bucket::Minute.default_lookback_days() < Bucket::Hour.default_lookback_days()); + assert!(Bucket::Hour.default_lookback_days() < Bucket::Day.default_lookback_days()); + assert!(Bucket::Day.default_lookback_days() < Bucket::Week.default_lookback_days()); + assert!(Bucket::Week.default_lookback_days() < Bucket::Month.default_lookback_days()); + assert_eq!(Bucket::Week.default_lookback_days(), 180); + assert_eq!(Bucket::Month.default_lookback_days(), 730); +} + +#[test] +fn timeline_buckets_by_week() { + let (pool, _d) = test_pool(); + // Two logs in the same week, one in a different week + insert_logs_batch( + &pool, + &[ + entry("2026-01-05T00:00:00Z", "h1", "info", None, "a"), // week 1 + entry("2026-01-06T00:00:00Z", "h1", "info", None, "b"), // same week 1 + entry("2026-01-12T00:00:00Z", "h1", "info", None, "c"), // week 2 + ], + ) + .unwrap(); + refresh_timeline_rollup(&pool).unwrap(); + let pts = timeline( + &pool, + Bucket::Week, + TimelineGroupBy::None, + None, + None, + None, + None, + None, + ) + .unwrap(); + assert_eq!(pts.len(), 2); + assert_eq!(pts[0].count, 2); + assert_eq!(pts[1].count, 1); + // Bucket labels should contain "W" + assert!( + pts[0].bucket.contains('W'), + "week bucket label must contain 'W': {}", + pts[0].bucket + ); +} + +#[test] +fn timeline_buckets_by_month() { + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[ + entry("2026-01-01T00:00:00Z", "h1", "info", None, "a"), + entry("2026-01-15T00:00:00Z", "h1", "info", None, "b"), + entry("2026-02-01T00:00:00Z", "h1", "info", None, "c"), + ], + ) + .unwrap(); + refresh_timeline_rollup(&pool).unwrap(); + let pts = timeline( + &pool, + Bucket::Month, + TimelineGroupBy::None, + None, + None, + None, + None, + None, + ) + .unwrap(); + assert_eq!(pts.len(), 2); + assert_eq!(pts[0].count, 2); + assert_eq!(pts[1].count, 1); + // Bucket labels should look like "YYYY-MM" + assert_eq!(pts[0].bucket, "2026-01"); + assert_eq!(pts[1].bucket, "2026-02"); +} + +// ----------------------------------------------------------------------------- +// timeline_hourly rollup (bead syslog-mcp-kcvq) +// ----------------------------------------------------------------------------- + +/// Hand-compute the live timeline counts directly off `logs`, bypassing the +/// rollup, so a test can assert rollup == live for the unbounded case. +fn live_hour_counts(pool: &DbPool) -> Vec<(String, i64)> { + let conn = pool.get().unwrap(); + let mut stmt = conn + .prepare( + "SELECT strftime('%Y-%m-%dT%H:00:00Z', timestamp) AS b, COUNT(*) + FROM logs GROUP BY b ORDER BY b ASC", + ) + .unwrap(); + let rows = stmt + .query_map([], |r| Ok((r.get::<_, String>(0)?, r.get::<_, i64>(1)?))) + .unwrap(); + rows.collect::>>().unwrap() +} + +#[test] +fn timeline_rollup_matches_live_for_hour_unbounded() { + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[ + entry("2026-03-01T00:10:00Z", "h1", "info", Some("nginx"), "a"), + entry("2026-03-01T00:40:00Z", "h2", "err", Some("sshd"), "b"), + entry("2026-03-01T01:05:00Z", "h1", "info", None, "c"), + entry("2026-03-01T01:55:00Z", "h1", "warning", Some("nginx"), "d"), + ], + ) + .unwrap(); + refresh_timeline_rollup(&pool).unwrap(); + // Unbounded range + full refresh => rollup is an EXACT match for live. + // (A mid-hour `from`/`to` would legitimately differ in the boundary hour + // because the rollup can only filter at hour granularity — that imprecision + // is documented and accepted; see timeline_from_rollup.) + let rollup = timeline( + &pool, + Bucket::Hour, + TimelineGroupBy::None, + None, + None, + None, + None, + None, + ) + .unwrap(); + let live = live_hour_counts(&pool); + assert_eq!(rollup.len(), live.len()); + for (pt, (b, c)) in rollup.iter().zip(live.iter()) { + assert_eq!(&pt.bucket, b); + assert_eq!(pt.count, *c); + } +} + +#[test] +fn timeline_rollup_incremental_add_does_not_double_count() { + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[ + entry("2026-03-01T00:10:00Z", "h1", "info", None, "a"), + entry("2026-03-01T00:20:00Z", "h1", "info", None, "b"), + ], + ) + .unwrap(); + let folded = refresh_timeline_rollup(&pool).unwrap(); + assert_eq!(folded, 2, "first refresh folds both rows"); + // A second refresh with nothing new must be a no-op (watermark current). + let folded2 = refresh_timeline_rollup(&pool).unwrap(); + assert_eq!(folded2, 0, "no-op refresh folds nothing"); + let pts = timeline( + &pool, + Bucket::Hour, + TimelineGroupBy::None, + None, + None, + None, + None, + None, + ) + .unwrap(); + assert_eq!(pts.len(), 1); + assert_eq!(pts[0].count, 2, "no double-count after redundant refresh"); + + // Insert MORE into the SAME hour; refresh must ADD, not recount. + insert_logs_batch( + &pool, + &[entry("2026-03-01T00:30:00Z", "h1", "info", None, "c")], + ) + .unwrap(); + let folded3 = refresh_timeline_rollup(&pool).unwrap(); + assert_eq!(folded3, 1, "only the new row is folded"); + let pts = timeline( + &pool, + Bucket::Hour, + TimelineGroupBy::None, + None, + None, + None, + None, + None, + ) + .unwrap(); + assert_eq!( + pts[0].count, 3, + "incremental add yields 3, not double-counted" + ); +} + +#[test] +fn timeline_rollup_late_arriving_old_timestamp_lands_in_old_bucket() { + let (pool, _d) = test_pool(); + // Ingest a recent hour first, refresh. + insert_logs_batch( + &pool, + &[entry("2026-03-01T05:00:00Z", "h1", "info", None, "recent")], + ) + .unwrap(); + refresh_timeline_rollup(&pool).unwrap(); + // Now a NEW (higher-id) row arrives carrying an OLD timestamp. + insert_logs_batch( + &pool, + &[entry("2026-03-01T02:00:00Z", "h1", "info", None, "late")], + ) + .unwrap(); + let folded = refresh_timeline_rollup(&pool).unwrap(); + assert_eq!(folded, 1); + let pts = timeline( + &pool, + Bucket::Hour, + TimelineGroupBy::None, + None, + None, + None, + None, + None, + ) + .unwrap(); + // The late row must land in its OWN old (02:00) bucket, not the recent one. + assert_eq!(pts.len(), 2); + assert_eq!(pts[0].bucket, "2026-03-01T02:00:00Z"); + assert_eq!(pts[0].count, 1); + assert_eq!(pts[1].bucket, "2026-03-01T05:00:00Z"); + assert_eq!(pts[1].count, 1); +} + +#[test] +fn timeline_rollup_null_app_groups_as_none() { + // BLOCKER regression guard: app_name is stored COALESCE(app_name,'') NOT NULL + // in the rollup; group_by=app_name must project '' back to '' AND the + // null-app rows must NOT double-count across refreshes (NULL-distinct PK bug). + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[ + entry("2026-03-01T00:05:00Z", "h1", "info", None, "no-app-1"), + entry("2026-03-01T00:10:00Z", "h1", "info", None, "no-app-2"), + entry("2026-03-01T00:15:00Z", "h1", "info", Some("nginx"), "app-1"), + ], + ) + .unwrap(); + refresh_timeline_rollup(&pool).unwrap(); + // Refresh again to exercise the ON CONFLICT path for the null-app grain — + // if NULLs were stored as NULL, this would duplicate rows and inflate counts. + insert_logs_batch( + &pool, + &[entry( + "2026-03-01T00:20:00Z", + "h1", + "info", + None, + "no-app-3", + )], + ) + .unwrap(); + refresh_timeline_rollup(&pool).unwrap(); + let pts = timeline( + &pool, + Bucket::Hour, + TimelineGroupBy::AppName, + None, + None, + None, + None, + None, + ) + .unwrap(); + let none_total: i64 = pts + .iter() + .filter(|p| p.group.as_deref() == Some("")) + .map(|p| p.count) + .sum(); + let nginx_total: i64 = pts + .iter() + .filter(|p| p.group.as_deref() == Some("nginx")) + .map(|p| p.count) + .sum(); + assert_eq!( + none_total, 3, + "null-app rows group as , no double-count" + ); + assert_eq!(nginx_total, 1); +} + +#[test] +fn prune_timeline_rollup_drops_buckets_older_than_oldest_log() { + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[ + entry("2026-03-01T00:00:00Z", "h1", "info", None, "oldest"), + entry("2026-03-02T00:00:00Z", "h1", "info", None, "mid"), + entry("2026-03-03T00:00:00Z", "h1", "info", None, "newest"), + ], + ) + .unwrap(); + refresh_timeline_rollup(&pool).unwrap(); + // Simulate a retention purge of the oldest log. + { + let conn = pool.get().unwrap(); + conn.execute( + "DELETE FROM logs WHERE timestamp = '2026-03-01T00:00:00Z'", + [], + ) + .unwrap(); + } + // Before prune, the old bucket still ghosts in the rollup. + let deleted = prune_timeline_rollup(&pool).unwrap(); + assert_eq!(deleted, 1, "the single ghost bucket is pruned"); + let pts = timeline( + &pool, + Bucket::Day, + TimelineGroupBy::None, + None, + None, + None, + None, + None, + ) + .unwrap(); + assert_eq!(pts.len(), 2, "only buckets >= oldest remaining log survive"); + assert_eq!(pts[0].bucket, "2026-03-02T00:00:00Z"); +} + +#[test] +fn timeline_rollup_status_reports_watermark() { + let (pool, _d) = test_pool(); + let before = timeline_rollup_status(&pool).unwrap(); + assert_eq!(before.source_max_id, 0); + assert!(before.refreshed_at.is_none()); + insert_logs_batch( + &pool, + &[entry("2026-03-01T00:00:00Z", "h1", "info", None, "a")], + ) + .unwrap(); + refresh_timeline_rollup(&pool).unwrap(); + let after = timeline_rollup_status(&pool).unwrap(); + assert!(after.source_max_id > 0); + assert!(after.refreshed_at.is_some()); +} + +#[test] +fn silent_hosts_merges_case_variants_before_cutoff() { + // Regression: silent_hosts read the raw, case-sensitive `hosts` table, so a + // dormant `backuphost` identity was flagged as silent even though the live `BACKUPHOST` + // kept forwarding. Routing through list_hosts() merges case/FQDN variants + // (latest last_seen wins) first, so the machine is correctly considered alive. + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[ + entry("2026-06-19T20:00:00Z", "BACKUPHOST", "info", None, "live"), + entry("2026-06-11T06:00:00Z", "backuphost", "info", None, "old"), + entry("2026-06-11T06:00:00Z", "WINHOST", "info", None, "old"), + ], + ) + .unwrap(); + // hosts.last_seen is stamped with insert-time `now`; pin it explicitly. + let conn = pool.get().unwrap(); + for (host, last_seen) in [ + ("BACKUPHOST", "2026-06-19T20:00:00.000Z"), + ("backuphost", "2026-06-11T06:00:00.000Z"), + ("WINHOST", "2026-06-11T06:00:00.000Z"), + ] { + conn.execute( + "UPDATE hosts SET last_seen = ?1 WHERE hostname = ?2", + rusqlite::params![last_seen, host], + ) + .unwrap(); + } + drop(conn); + + let now_unix = chrono::DateTime::parse_from_rfc3339("2026-06-19T21:00:00Z") + .unwrap() + .timestamp(); + let silent = silent_hosts(&pool, "2026-06-15T00:00:00.000Z", now_unix).unwrap(); + let names: Vec = silent.iter().map(|h| h.hostname.clone()).collect(); + + assert!( + !names.iter().any(|n| n == "backuphost"), + "merged backuphost is live (BACKUPHOST forwarding) and must not be flagged silent: {names:?}" + ); + assert!( + names.iter().any(|n| n == "winhost"), + "genuinely-dormant WINHOST (lowercased) must still be flagged: {names:?}" + ); +} + +#[test] +fn clock_skew_merges_case_variants() { + // Regression: clock_skew GROUP BY hostname was case-sensitive, so `BACKUPHOST` and + // `backuphost` reported as two separate skew rows. They must merge into one host + // with summed samples and a sample-weighted average skew. + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[ + entry("2026-01-01T00:00:00Z", "BACKUPHOST", "info", None, "a"), + entry("2026-01-01T00:00:00Z", "BACKUPHOST", "info", None, "b"), + entry("2026-01-01T00:00:00Z", "backuphost", "info", None, "c"), + ], + ) + .unwrap(); + let conn = pool.get().unwrap(); + // BACKUPHOST rows skew +10s, the backuphost row skews +40s. + for (msg, received_at) in [ + ("a", "2026-01-01T00:00:10Z"), + ("b", "2026-01-01T00:00:10Z"), + ("c", "2026-01-01T00:00:40Z"), + ] { + conn.execute( + "UPDATE logs SET received_at = ?1 WHERE message = ?2", + rusqlite::params![received_at, msg], + ) + .unwrap(); + } + drop(conn); + + let result = clock_skew(&pool, "2026-01-01T00:00:00Z", None).unwrap(); + assert_eq!( + result.len(), + 1, + "BACKUPHOST/backuphost must collapse to one host" + ); + assert_eq!(result[0].hostname, "backuphost"); + assert_eq!(result[0].samples, 3); + // Sample-weighted: (10 + 10 + 40) / 3 = 20. + assert!( + (result[0].avg_skew_secs - 20.0).abs() < 0.5, + "weighted avg skew should be ~20s, got {}", + result[0].avg_skew_secs + ); + assert!((result[0].max_skew_secs - 40.0).abs() < 0.5); +} + +#[test] +fn feed_cursor_is_never_below_the_greatest_returned_id() { + let (pool, _d) = test_pool(); + insert_logs_batch( + &pool, + &[entry("2026-01-01T00:00:00Z", "h", "info", None, "one")], + ) + .unwrap(); + let (first, cursor, more) = feed_logs(&pool, Some(0), None, 100).unwrap(); + assert!(!more); + assert_eq!(first.len(), 1); + assert!(cursor >= first.iter().map(|row| row.id).max().unwrap()); + + insert_logs_batch( + &pool, + &[entry("2026-01-01T00:00:01Z", "h", "info", None, "two")], + ) + .unwrap(); + let (second, next_cursor, _) = feed_logs(&pool, Some(cursor), None, 100).unwrap(); + assert_eq!(second.len(), 1); + assert!(next_cursor >= second[0].id); + let (duplicate, _, _) = feed_logs(&pool, Some(next_cursor), None, 100).unwrap(); + assert!(duplicate.is_empty()); +} diff --git a/crates/shared/cortex/storage-sqlite/src/config.rs b/crates/shared/cortex/storage-sqlite/src/config.rs new file mode 100644 index 00000000..0c9936d4 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/config.rs @@ -0,0 +1,99 @@ +//! SQLite-specific storage configuration. +//! +//! These fields are extracted from Cortex's product-level configuration so the +//! persistence adapter can be configured without depending on the runtime. + +use anyhow::Context as _; +use serde::{Deserialize, Serialize}; +use std::path::PathBuf; + +/// Configuration required by the Cortex SQLite adapter. +#[derive(Debug, Clone, Serialize, Deserialize)] +#[serde(default)] +pub struct StorageConfig { + pub db_path: PathBuf, + pub pool_size: u32, + pub sqlite_page_cache_mb: u64, + pub sqlite_mmap_mb: u64, + pub heavy_read_concurrency: usize, + pub wal_checkpoint_mb: u64, + pub retention_days: u32, + pub wal_mode: bool, + pub max_db_size_mb: u64, + pub recovery_db_size_mb: u64, + pub min_free_disk_mb: u64, + pub recovery_free_disk_mb: u64, + pub cleanup_interval_secs: u64, + pub cleanup_chunk_size: usize, + pub err_floor_window_hours: u64, + pub err_floor_per_source_cap: usize, +} + +impl Default for StorageConfig { + fn default() -> Self { + Self { + db_path: PathBuf::from("/data/cortex.db"), + pool_size: 8, + sqlite_page_cache_mb: 128, + sqlite_mmap_mb: 256, + heavy_read_concurrency: 1, + wal_checkpoint_mb: 256, + retention_days: 90, + wal_mode: true, + max_db_size_mb: 1024, + recovery_db_size_mb: 900, + min_free_disk_mb: 0, + recovery_free_disk_mb: 0, + cleanup_interval_secs: 60, + cleanup_chunk_size: 2_000, + err_floor_window_hours: 24, + err_floor_per_source_cap: 10_000, + } + } +} + +impl StorageConfig { + pub fn sqlite_page_cache_kib_per_connection(&self) -> anyhow::Result { + let pool_size = u64::from(self.pool_size.max(1)); + let total_kib = self + .sqlite_page_cache_mb + .checked_mul(1024) + .context("storage.sqlite_page_cache_mb is too large")?; + let per_conn = (total_kib / pool_size).max(1); + i64::try_from(per_conn) + .context( + "storage.sqlite_page_cache_mb is too large; derived cache_size must fit in i64", + ) + .map(|value| -value) + } + + pub fn sqlite_mmap_bytes_i64(&self) -> anyhow::Result { + i64::try_from(self.sqlite_mmap_bytes()) + .context("storage.sqlite_mmap_mb is too large; derived mmap_size must fit in i64") + } + + #[must_use] + pub fn sqlite_mmap_bytes(&self) -> u64 { + self.sqlite_mmap_mb.saturating_mul(1024 * 1024) + } + + #[must_use] + pub fn wal_checkpoint_threshold_bytes(&self) -> u64 { + self.wal_checkpoint_mb.saturating_mul(1024 * 1024) + } + + #[cfg(test)] + pub(crate) fn for_test(db_path: PathBuf) -> Self { + Self { + db_path, + pool_size: 1, + wal_mode: false, + cleanup_chunk_size: 1, + ..Self::default() + } + } +} + +#[cfg(test)] +#[path = "config_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/config_tests.rs b/crates/shared/cortex/storage-sqlite/src/config_tests.rs new file mode 100644 index 00000000..ffa05d41 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/config_tests.rs @@ -0,0 +1,24 @@ +use super::*; + +#[test] +fn storage_defaults_and_derived_sqlite_values_match_donor_contract() { + let config = StorageConfig::default(); + assert_eq!(config.pool_size, 8); + assert_eq!(config.retention_days, 90); + assert_eq!( + config.sqlite_page_cache_kib_per_connection().unwrap(), + -16_384 + ); + assert_eq!(config.sqlite_mmap_bytes(), 256 * 1024 * 1024); + assert_eq!(config.wal_checkpoint_threshold_bytes(), 256 * 1024 * 1024); +} + +#[test] +fn page_cache_conversion_rejects_values_that_do_not_fit_sqlite_i64() { + let config = StorageConfig { + sqlite_page_cache_mb: u64::MAX, + pool_size: 1, + ..StorageConfig::default() + }; + assert!(config.sqlite_page_cache_kib_per_connection().is_err()); +} diff --git a/crates/shared/cortex/storage-sqlite/src/entity_resolution.rs b/crates/shared/cortex/storage-sqlite/src/entity_resolution.rs new file mode 100644 index 00000000..86c081ea --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/entity_resolution.rs @@ -0,0 +1,22 @@ +//! Canonical entity resolution: key grammar, observations, and deterministic +//! resolver decisions for the investigation graph. +//! +//! This module owns the hard-break canonical service identity contract: +//! `logical_service:plex` for logical identity and +//! `service_instance:nashost/plex` for host-scoped deployment topology. +//! Legacy nested shapes (`nashost:plex`, `nashost:plex:plex`, `plex/plex/plex`) +//! are classified for rejection, never normalized. + +pub mod adapters; +pub mod observation; +pub mod resolver; +pub mod vocab; + +pub use adapters::*; +pub use observation::*; +pub use resolver::*; +pub use vocab::*; + +#[cfg(test)] +#[path = "entity_resolution_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/entity_resolution/adapters.rs b/crates/shared/cortex/storage-sqlite/src/entity_resolution/adapters.rs new file mode 100644 index 00000000..455ddb76 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/entity_resolution/adapters.rs @@ -0,0 +1,232 @@ +//! Pure adapters that convert source rows (agent Docker identity, raw log +//! app labels, verified inventory services) into bounded resolver +//! observations. Adapters never touch the database. + +use cortex_ingest_core::AGENT_DOCKER_SOURCE_KIND; +use cortex_inventory::{InventoryService, TrustLevel}; + +use super::observation::*; +use super::vocab::{logical_service_key, service_instance_key}; + +/// Convert structured agent-attested Docker identity into observations. +/// The compose service label (falling back to the container name) is the +/// logical service identity; the agent host scopes the service instance. +pub fn observations_from_agent_docker_identity( + identity: &AgentDockerIdentity, +) -> Vec { + let Some(host_key) = logical_service_key(&identity.agent_host) else { + return Vec::new(); + }; + // Evidence-path attribution follows the actual source of the service + // name: the compose service label when present, else the container name. + let (service_name, service_evidence_path) = match identity.compose_service.as_deref() { + Some(compose_service) => (compose_service, "agent_docker.compose_service"), + None => ( + identity.container_name.as_str(), + "agent_docker.container_name", + ), + }; + let Some(logical_key) = logical_service_key(service_name) else { + return Vec::new(); + }; + let Some(instance_key) = service_instance_key(&host_key, &logical_key) else { + return Vec::new(); + }; + vec![ + ResolverObservation { + kind: ObservationKind::Host, + observed_key: host_key.clone(), + display_label: safe_display_value(&identity.agent_host), + host_key: Some(host_key.clone()), + logical_service_key: None, + service_instance_key: None, + source_kind: AGENT_DOCKER_SOURCE_KIND.to_string(), + source_id: identity.container_id.clone(), + evidence_path: "agent_docker.host".to_string(), + observed_at: identity.observed_at.clone(), + trust: ResolverTrust::Verified, + structured: true, + }, + ResolverObservation { + kind: ObservationKind::LogicalService, + observed_key: logical_key.clone(), + display_label: safe_display_value(service_name), + host_key: None, + logical_service_key: Some(logical_key.clone()), + service_instance_key: None, + source_kind: AGENT_DOCKER_SOURCE_KIND.to_string(), + source_id: identity.container_id.clone(), + evidence_path: service_evidence_path.to_string(), + observed_at: identity.observed_at.clone(), + trust: ResolverTrust::Verified, + structured: true, + }, + ResolverObservation { + kind: ObservationKind::ServiceInstance, + observed_key: instance_key.clone(), + display_label: instance_key.clone(), + host_key: Some(host_key), + logical_service_key: Some(logical_key), + service_instance_key: Some(instance_key), + source_kind: AGENT_DOCKER_SOURCE_KIND.to_string(), + source_id: identity.container_id.clone(), + // The instance key is host + service name (compose service label + // or container-name fallback); `compose_project` is never read. + evidence_path: "agent_docker.host_service".to_string(), + observed_at: identity.observed_at.clone(), + trust: ResolverTrust::Verified, + structured: true, + }, + ] +} + +/// Convert a raw observed log app label into a single weak observation. +/// Raw labels never produce `LogicalService` / `ServiceInstance` +/// observations on their own — they must be matched to structured evidence +/// by the resolver, or they stay raw. +// Test-only contract coverage: pins the plan-locked "raw labels never +// self-upgrade" resolver rule (entity_resolution_tests.rs). +pub fn observations_from_raw_app_label( + app_name: &str, + host: &str, + source_kind: &str, + source_id: &str, + observed_at: &str, +) -> Vec { + let observed_key = app_name.trim().to_ascii_lowercase(); + vec![ResolverObservation { + kind: ObservationKind::RawAppLabel, + observed_key, + display_label: safe_display_value(app_name), + host_key: super::vocab::logical_service_key(host), + logical_service_key: None, + service_instance_key: None, + source_kind: source_kind.to_string(), + source_id: source_id.to_string(), + evidence_path: "logs.app_name".to_string(), + observed_at: observed_at.to_string(), + trust: ResolverTrust::Claimed, + structured: false, + }] +} + +/// Convert a verified/observed inventory service into observations: the +/// logical service always (when the name canonicalizes), plus the host, +/// service instance, and domain/mount (storage) context when the inventory +/// row carries a host. A hostless service still asserts logical identity — +/// deployment topology is simply absent, never guessed into an `unknown/` +/// instance. +pub fn observations_from_inventory_service(service: &InventoryService) -> Vec { + let Some(logical_key) = logical_service_key(&service.name) else { + return Vec::new(); + }; + let trust = inventory_trust(&service.trust_level); + let source_kind = "app_inventory".to_string(); + let source_id = service.id.clone(); + let observed_at = service.provenance.collected_at.clone(); + let mut observations = vec![ResolverObservation { + kind: ObservationKind::LogicalService, + observed_key: logical_key.clone(), + display_label: safe_display_value(&service.name), + host_key: None, + logical_service_key: Some(logical_key.clone()), + service_instance_key: None, + source_kind: source_kind.clone(), + source_id: source_id.clone(), + evidence_path: "inventory.services.name".to_string(), + observed_at: observed_at.clone(), + trust, + structured: true, + }]; + + let host = service.host.as_deref(); + let host_key = host.and_then(logical_service_key); + let instance_key = host_key + .as_deref() + .and_then(|host_key| service_instance_key(host_key, &logical_key)); + let (Some(host), Some(host_key), Some(instance_key)) = (host, host_key, instance_key) else { + return observations; + }; + + observations.push(ResolverObservation { + kind: ObservationKind::Host, + observed_key: host_key.clone(), + display_label: safe_display_value(host), + host_key: Some(host_key.clone()), + logical_service_key: None, + service_instance_key: None, + source_kind: source_kind.clone(), + source_id: source_id.clone(), + evidence_path: "inventory.services.host".to_string(), + observed_at: observed_at.clone(), + trust, + structured: true, + }); + observations.push(ResolverObservation { + kind: ObservationKind::ServiceInstance, + observed_key: instance_key.clone(), + display_label: instance_key.clone(), + host_key: Some(host_key.clone()), + logical_service_key: Some(logical_key.clone()), + service_instance_key: Some(instance_key.clone()), + source_kind: source_kind.clone(), + source_id: source_id.clone(), + evidence_path: "inventory.services".to_string(), + observed_at: observed_at.clone(), + trust, + structured: true, + }); + for domain in &service.domains { + let domain_key = domain.trim().to_ascii_lowercase(); + if domain_key.is_empty() { + continue; + } + observations.push(ResolverObservation { + kind: ObservationKind::Domain, + observed_key: domain_key, + display_label: safe_display_value(domain), + host_key: Some(host_key.clone()), + logical_service_key: Some(logical_key.clone()), + service_instance_key: Some(instance_key.clone()), + source_kind: source_kind.clone(), + source_id: source_id.clone(), + evidence_path: "inventory.services.domains".to_string(), + observed_at: observed_at.clone(), + trust, + structured: true, + }); + } + for mount in &service.mounts { + let target = mount.target.trim(); + if target.is_empty() { + continue; + } + observations.push(ResolverObservation { + kind: ObservationKind::Storage, + observed_key: format!("{host_key}:{target}"), + display_label: safe_display_value(target), + host_key: Some(host_key.clone()), + logical_service_key: Some(logical_key.clone()), + service_instance_key: Some(instance_key.clone()), + source_kind: source_kind.clone(), + source_id: source_id.clone(), + evidence_path: "inventory.services.mounts".to_string(), + observed_at: observed_at.clone(), + trust, + structured: true, + }); + } + observations +} + +fn inventory_trust(trust_level: &TrustLevel) -> ResolverTrust { + match trust_level { + TrustLevel::Verified | TrustLevel::Observed => ResolverTrust::Verified, + TrustLevel::Claimed => ResolverTrust::Claimed, + TrustLevel::Inferred => ResolverTrust::Inferred, + } +} + +#[cfg(test)] +#[path = "adapters_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/entity_resolution/adapters_tests.rs b/crates/shared/cortex/storage-sqlite/src/entity_resolution/adapters_tests.rs new file mode 100644 index 00000000..3a5996da --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/entity_resolution/adapters_tests.rs @@ -0,0 +1,57 @@ +use super::*; +use cortex_inventory::{MountRef, Provenance}; + +fn service(host: Option<&str>) -> InventoryService { + InventoryService { + id: "service:nas:plex".into(), + name: "Plex".into(), + kind: "container".into(), + trust_level: TrustLevel::Observed, + provenance: Provenance::new("docker:nas", "app_inventory", "2026-08-18T00:00:00Z".into()), + host: host.map(str::to_owned), + image: None, + status: Some("running".into()), + domains: vec!["plex.example.test".into()], + ports: Vec::new(), + mounts: vec![MountRef { + source: None, + target: "/media".into(), + read_only: true, + }], + env_keys: Vec::new(), + labels: Default::default(), + details: Default::default(), + } +} + +#[test] +fn hostless_inventory_service_asserts_only_logical_identity() { + let observations = observations_from_inventory_service(&service(None)); + assert_eq!(observations.len(), 1); + assert_eq!(observations[0].kind, ObservationKind::LogicalService); +} + +#[test] +fn hosted_inventory_service_emits_topology_context() { + let observations = observations_from_inventory_service(&service(Some("NAS"))); + assert!( + observations + .iter() + .any(|o| o.kind == ObservationKind::ServiceInstance && o.observed_key == "nas/plex") + ); + assert!( + observations + .iter() + .any(|o| o.kind == ObservationKind::Domain) + ); + assert!( + observations + .iter() + .any(|o| o.kind == ObservationKind::Storage) + ); + assert!( + observations + .iter() + .all(|o| o.trust == ResolverTrust::Verified) + ); +} diff --git a/crates/shared/cortex/storage-sqlite/src/entity_resolution/observation.rs b/crates/shared/cortex/storage-sqlite/src/entity_resolution/observation.rs new file mode 100644 index 00000000..e2b2cd64 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/entity_resolution/observation.rs @@ -0,0 +1,104 @@ +//! Bounded typed resolver observations. +//! +//! Observations are chunk-local, in-memory inputs to the deterministic +//! resolver. They are never persisted per-log-row; projection code converts +//! source rows into observations, resolves them, and stores only the +//! resulting graph entities/relationships/evidence. + +/// Epistemic trust of an observation's source. Ordered strongest-first so +/// `min()` over evidence selects the strongest supporting evidence: +/// independent corroboration cannot be weakened by additional weak +/// observations. +#[derive(Debug, Clone, Copy, PartialEq, Eq, PartialOrd, Ord)] +pub enum ResolverTrust { + Verified, + Claimed, + Inferred, +} + +/// What kind of thing an observation describes. +// Plan-locked vocabulary (2026-07-13 canonical-entity-resolution): several +// kinds are reserved for future adapters and not constructed yet. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ObservationKind { + Host, + LogicalService, + ServiceInstance, + Container, + ComposeProject, + Domain, + ReverseProxy, + Storage, + ConfigArtifact, + RawAppLabel, + AiProject, + AiSession, + Command, + User, + Device, +} + +/// One bounded, typed observation extracted from a source row. Display +/// values must already be safe (see [`safe_display_value`]). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolverObservation { + pub kind: ObservationKind, + pub observed_key: String, + pub display_label: String, + pub host_key: Option, + pub logical_service_key: Option, + pub service_instance_key: Option, + pub source_kind: String, + pub source_id: String, + pub evidence_path: String, + pub observed_at: String, + pub trust: ResolverTrust, + pub structured: bool, +} + +/// Structured agent-attested Docker identity for one log line, extracted +/// from `metadata_json.agent_docker`. This is the supported Docker identity +/// source; central-pull `docker://` rows are not resolver proof. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct AgentDockerIdentity { + pub agent_host: String, + pub container_id: String, + pub container_name: String, + pub compose_project: Option, + pub compose_service: Option, + pub image: Option, + pub stream: String, + pub observed_at: String, +} + +/// Redact display values that look sensitive (credentialed URLs, home paths, +/// token/secret material, metadata payload paths) and bound the rest to 128 +/// printable characters. +pub fn safe_display_value(value: &str) -> String { + let lower = value.to_ascii_lowercase(); + // `:` + `@` catches scheme-less credentials (`user:pass@host`) as well + // as credentialed URLs; over-redaction is acceptable here. + let sensitive = lower.contains(':') && lower.contains('@') + || lower.contains("token") + || lower.contains("password") + || lower.contains("secret") + || lower.contains("api_key") + || lower.contains("apikey") + || lower.contains("/home/") + || lower.contains("/users/") + || lower.contains("metadata_json") + || lower.contains("cache_path") + || lower.contains("source_path"); + if sensitive { + return "[redacted]".to_string(); + } + value + .chars() + .filter(|ch| !ch.is_control()) + .take(128) + .collect() +} + +#[cfg(test)] +#[path = "observation_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/entity_resolution/observation_tests.rs b/crates/shared/cortex/storage-sqlite/src/entity_resolution/observation_tests.rs new file mode 100644 index 00000000..c7710c23 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/entity_resolution/observation_tests.rs @@ -0,0 +1,25 @@ +use super::*; + +#[test] +fn display_values_are_bounded_and_sensitive_values_are_redacted() { + assert_eq!( + safe_display_value("https://user:pass@example.test"), + "[redacted]" + ); + assert_eq!(safe_display_value("/home/alice/token.txt"), "[redacted]"); + let long = "x".repeat(200); + assert_eq!(safe_display_value(&long).len(), 128); + assert_eq!( + safe_display_value( + "a +b c" + ), + "abc" + ); +} + +#[test] +fn resolver_trust_orders_strongest_first() { + assert!(ResolverTrust::Verified < ResolverTrust::Claimed); + assert!(ResolverTrust::Claimed < ResolverTrust::Inferred); +} diff --git a/crates/shared/cortex/storage-sqlite/src/entity_resolution/resolver.rs b/crates/shared/cortex/storage-sqlite/src/entity_resolution/resolver.rs new file mode 100644 index 00000000..0b90321d --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/entity_resolution/resolver.rs @@ -0,0 +1,194 @@ +//! Deterministic resolver: converts bounded observations into ranked, +//! evidence-backed entity decisions and lookup diagnostics. +//! +//! Resolution is deterministic-first: no LLM calls, no fuzzy or substring +//! matching. Raw app labels never upgrade themselves into logical-service +//! identity; only structured observations (agent Docker metadata, verified +//! inventory) produce `logical_service` / `service_instance` decisions. + +use std::collections::BTreeMap; + +use super::observation::{ObservationKind, ResolverObservation, ResolverTrust}; +use super::vocab::{ENTITY_TYPE_LOGICAL_SERVICE, ENTITY_TYPE_SERVICE_INSTANCE}; + +/// Evidence rows kept per decision/diagnostic sample. +pub const MAX_RESOLVER_EVIDENCE_SAMPLE: usize = 5; + +/// Outcome class of a resolution or lookup. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum ResolverStatus { + Resolved, + Ambiguous, + RejectedLegacyShape, + Degraded, +} + +impl ResolverStatus { + pub fn as_str(&self) -> &'static str { + match self { + ResolverStatus::Resolved => "resolved", + ResolverStatus::Ambiguous => "ambiguous", + ResolverStatus::RejectedLegacyShape => "rejected_legacy_shape", + ResolverStatus::Degraded => "degraded", + } + } +} + +/// One piece of evidence backing a resolver decision. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolverEvidence { + pub rule_id: &'static str, + pub source_kind: String, + pub source_id: String, + pub evidence_path: String, + pub trust: ResolverTrust, + pub safe_excerpt: Option, +} + +/// A resolved canonical entity with its supporting evidence sample. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolvedEntityDecision { + pub entity_type: &'static str, + pub canonical_key: String, + pub display_label: String, + pub status: ResolverStatus, + pub trust: ResolverTrust, + pub evidence: Vec, +} + +/// Diagnostic result for a lookup input (topic, graph key, alias). +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ResolverDiagnostic { + pub status: ResolverStatus, + pub input: String, + pub reason: String, + pub candidates: Vec, + pub evidence_sample: Vec, + pub total_evidence_count: usize, +} + +/// Deterministically resolve observations into entity decisions. +/// +/// Only structured service identity observations produce decisions: +/// `ServiceInstance` observations yield both the instance and its logical +/// service; `LogicalService` observations yield the logical service. +/// `RawAppLabel` observations never produce decisions (no self-upgrade). +/// Single pass over the observations with keyed aggregation. +pub fn resolve_observations(observations: &[ResolverObservation]) -> Vec { + let mut by_entity: BTreeMap<(&'static str, String), Vec> = BTreeMap::new(); + for obs in observations { + match obs.kind { + ObservationKind::LogicalService => { + if let Some(key) = obs.logical_service_key.clone() { + by_entity + .entry((ENTITY_TYPE_LOGICAL_SERVICE, key)) + .or_default() + .push(evidence(obs, "logical_service_observation")); + } else { + skipped_missing_key(obs, "logical_service_key"); + } + } + ObservationKind::ServiceInstance => { + if let Some(key) = obs.service_instance_key.clone() { + by_entity + .entry((ENTITY_TYPE_SERVICE_INSTANCE, key)) + .or_default() + .push(evidence(obs, "service_instance_observation")); + } else { + skipped_missing_key(obs, "service_instance_key"); + } + if let Some(key) = obs.logical_service_key.clone() { + by_entity + .entry((ENTITY_TYPE_LOGICAL_SERVICE, key)) + .or_default() + .push(evidence(obs, "service_instance_logical_service")); + } else { + skipped_missing_key(obs, "logical_service_key"); + } + } + // Raw app labels are weak claims: never a decision by themselves. + ObservationKind::RawAppLabel => {} + _ => {} + } + } + by_entity + .into_iter() + .map(|((entity_type, canonical_key), evidence)| { + let trust = evidence + .iter() + .map(|e| e.trust) + .min() + .unwrap_or(ResolverTrust::Inferred); + ResolvedEntityDecision { + entity_type, + display_label: canonical_key.clone(), + canonical_key, + status: ResolverStatus::Resolved, + trust, + evidence: evidence + .into_iter() + .take(MAX_RESOLVER_EVIDENCE_SAMPLE) + .collect(), + } + }) + .collect() +} + +/// Classify a lookup input before any graph lookup. Legacy nested service +/// shapes (`nashost:plex`, `nashost:plex:plex`, `plex/plex/plex`) are rejected +/// outright; anything else is degraded pending candidate resolution by the +/// caller (which owns database access). +pub fn diagnose_lookup_input(input: &str) -> ResolverDiagnostic { + if super::vocab::classify_legacy_shape(input).is_some() { + return ResolverDiagnostic { + status: ResolverStatus::RejectedLegacyShape, + input: input.to_string(), + reason: "rejected_legacy_shape".to_string(), + candidates: Vec::new(), + evidence_sample: Vec::new(), + total_evidence_count: 0, + }; + } + ResolverDiagnostic { + status: ResolverStatus::Degraded, + input: input.to_string(), + reason: "no_resolver_candidates".to_string(), + candidates: Vec::new(), + evidence_sample: Vec::new(), + total_evidence_count: 0, + } +} + +/// Adapters always populate the keys their observation kind requires; a miss +/// here means an adapter bug, so it is loud in debug builds and traced (not +/// silently dropped) in release builds. +fn skipped_missing_key(obs: &ResolverObservation, missing: &'static str) { + debug_assert!( + false, + "{:?} observation missing required {missing} (observed_key={:?}, source={}:{})", + obs.kind, obs.observed_key, obs.source_kind, obs.source_id + ); + tracing::debug!( + kind = ?obs.kind, + observed_key = %obs.observed_key, + source_kind = %obs.source_kind, + source_id = %obs.source_id, + missing, + "resolver skipped observation missing its required key" + ); +} + +fn evidence(obs: &ResolverObservation, rule_id: &'static str) -> ResolverEvidence { + ResolverEvidence { + rule_id, + source_kind: obs.source_kind.clone(), + source_id: obs.source_id.clone(), + evidence_path: obs.evidence_path.clone(), + trust: obs.trust, + safe_excerpt: Some(obs.display_label.clone()), + } +} + +#[cfg(test)] +#[path = "resolver_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/entity_resolution/resolver_tests.rs b/crates/shared/cortex/storage-sqlite/src/entity_resolution/resolver_tests.rs new file mode 100644 index 00000000..48c0bf38 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/entity_resolution/resolver_tests.rs @@ -0,0 +1,43 @@ +use super::*; + +fn observation(kind: ObservationKind) -> ResolverObservation { + ResolverObservation { + kind, + observed_key: "plex".into(), + display_label: "Plex".into(), + host_key: Some("nas".into()), + logical_service_key: Some("plex".into()), + service_instance_key: Some("nas/plex".into()), + source_kind: "app_inventory".into(), + source_id: "service:nas:plex".into(), + evidence_path: "inventory.services".into(), + observed_at: "2026-08-18T00:00:00Z".into(), + trust: ResolverTrust::Verified, + structured: true, + } +} + +#[test] +fn raw_labels_never_self_upgrade_but_structured_instances_do() { + assert!(resolve_observations(&[observation(ObservationKind::RawAppLabel)]).is_empty()); + let decisions = resolve_observations(&[observation(ObservationKind::ServiceInstance)]); + assert_eq!(decisions.len(), 2); + assert!( + decisions + .iter() + .any(|d| d.entity_type == ENTITY_TYPE_LOGICAL_SERVICE && d.canonical_key == "plex") + ); + assert!(decisions.iter().any(|d| d.entity_type == ENTITY_TYPE_SERVICE_INSTANCE && d.canonical_key == "nas/plex")); +} + +#[test] +fn lookup_diagnostics_reject_legacy_shapes() { + assert_eq!( + diagnose_lookup_input("nas:plex").status, + ResolverStatus::RejectedLegacyShape + ); + assert_eq!( + diagnose_lookup_input("plex").status, + ResolverStatus::Degraded + ); +} diff --git a/crates/shared/cortex/storage-sqlite/src/entity_resolution/vocab.rs b/crates/shared/cortex/storage-sqlite/src/entity_resolution/vocab.rs new file mode 100644 index 00000000..d15c9c2a --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/entity_resolution/vocab.rs @@ -0,0 +1,159 @@ +//! Canonical entity/relationship/reason vocabulary and key grammar for the +//! resolver-backed graph projection contract. + +pub const ENTITY_TYPE_LOGICAL_SERVICE: &str = "logical_service"; +pub const ENTITY_TYPE_SERVICE_INSTANCE: &str = "service_instance"; +pub const REL_INSTANCE_OF: &str = "instance_of"; +pub const REASON_RESOLVER_INSTANCE_OF: &str = "resolver_instance_of"; +pub const REASON_RESOLVER_SERVICE_INSTANCE: &str = "resolver_service_instance"; +pub const REASON_RESOLVER_RAW_APP_LABEL: &str = "resolver_raw_app_label"; +pub const GRAPH_PROJECTION_CONTRACT_KEY: &str = "graph_projection_contract"; +pub const GRAPH_PROJECTION_CONTRACT_V2: &str = "entity_resolution_v2"; + +/// Inclusion reasons annotating why a correlated log row was pulled in, and +/// the fallback kind marking the explicit degraded host-context path. +pub const INCLUSION_SERVICE_INSTANCE: &str = "service_instance"; +pub const INCLUSION_GRAPH_RELATED: &str = "graph_related"; +pub const INCLUSION_HOST_CONTEXT: &str = "host_context"; +pub const FALLBACK_EXPLICIT_DEGRADED_HOST_CONTEXT: &str = "explicit_degraded_host_context"; + +/// Legacy (pre entity-resolution) service identity shapes. These are +/// classified so callers can reject them; they are never normalized into +/// canonical keys. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum LegacyShape { + HostService, + HostProjectService, + SlashTriplet, +} + +/// Canonical logical-service key: lowercased, trimmed, non-key characters +/// mapped to `-`. Kept characters are ASCII alphanumerics plus `-`, `_`, +/// and `.` (dots preserve raw hostnames like `nashost.lan`). Returns `None` +/// when nothing canonical remains. +pub fn logical_service_key(name: &str) -> Option { + canonical_component(name) +} + +/// Canonical service-instance key `host/service`. Host and service are each +/// canonicalized independently; `None` when either side is empty. +pub fn service_instance_key(host: &str, service: &str) -> Option { + Some(format!( + "{}/{}", + canonical_component(host)?, + canonical_component(service)? + )) +} + +/// Split a canonical `host/service` key. Rejects empty components and any +/// extra `/` segments (which would be a legacy slash-triplet shape). +/// +/// This validates *shape*, not canonicality: components are not checked +/// against the canonical character set, so do not use this as an input +/// validator for untrusted keys. +pub fn split_service_instance_key(key: &str) -> Option<(&str, &str)> { + let (host, service) = key.split_once('/')?; + if host.is_empty() || service.is_empty() || service.contains('/') { + return None; + } + Some((host, service)) +} + +/// Split a canonical container key `host:container_id` and return just the +/// host segment. Rejects an empty host or an empty container-id segment +/// (including keys with no colon at all), mirroring the shape validation +/// `split_service_instance_key` applies to service-instance keys. +/// +/// This validates *shape*, not canonicality: the returned host is not +/// checked against the canonical character set, so do not use this as an +/// input validator for untrusted keys. +pub fn container_key_host(key: &str) -> Option<&str> { + let (host, container) = key.split_once(':')?; + if host.is_empty() || container.is_empty() { + return None; + } + Some(host) +} + +/// Classify legacy service identity shapes (`nashost:plex`, +/// `nashost:plex:plex`, `plex/plex/plex`). Canonical inputs return `None`, as +/// do free-text inputs that merely contain colons or slashes without looking +/// like legacy keys: anything with ASCII whitespace, colon shapes whose +/// segments are not all name-like (`10.0.0.5:443`, `12:30`) or contain a +/// slash (URLs like `http://example.com`, URIs like `agent-command://foo`), +/// and absolute paths (`/mnt/user/media`). +pub fn classify_legacy_shape(value: &str) -> Option { + let trimmed = value.trim(); + if trimmed.chars().any(|ch| ch.is_ascii_whitespace()) { + return None; + } + let colon_count = trimmed.matches(':').count(); + if colon_count >= 1 { + // A colon segment containing `/` means the input is a URL/URI, not a + // legacy `host:service` key. Returning `None` here (instead of + // falling through) also keeps `://` strings out of the slash-triplet + // branch: `https://a.b/c` has two slashes but is never a legacy shape. + let name_like_segments = trimmed.split(':').all(|segment| { + !segment.contains('/') && segment.chars().any(|ch| ch.is_ascii_alphabetic()) + }); + if !name_like_segments { + return None; + } + if colon_count == 1 { + return Some(LegacyShape::HostService); + } + return Some(LegacyShape::HostProjectService); + } + // NOTE(over-match tradeoff): this matches ANY 2+-slash, non-absolute-path + // string, not just the specific `{compose_project}/{compose_service}/ + // {container_name}` shape the old `agent::docker::container_app_name` + // used to emit (that agent path now emits a flat, slash-free APP-NAME — + // see `agent::docker::container_app_name`'s doc comment). A legitimate + // 2+-slash app label from another source would still be misclassified + // as legacy and dropped from graph `app`-entity projection. Investigated + // as part of syslog-mcp-5k1zb: the only other app-label source in this + // codebase, OTLP ingest (`otlp::entries::build_entries`), sets `app_name` + // exclusively from the resource-level `service.name` attribute — never + // from OTel instrumentation-scope names (e.g. + // `go.opentelemetry.io/collector/receiver`), which are stored only in + // `metadata_json.resource_attributes`/`log_attributes`, not `app_name`. + // So this risk is currently theoretical for OTLP. The legacy central-pull + // Docker ingest compat path (`docker_ingest::models::ContainerMeta:: + // app_name`, disabled by default, kept for compatibility fixtures/explicit + // remote Docker Engine endpoints) was also flattened to match the primary + // agent path (no longer emits a slash-triplet — see that function's doc + // comment). No currently-active source in this codebase emits a 2+-slash + // app label; the `SlashTriplet` classification below only matters for + // historical/already-stored rows and any future producer. If a future + // app-label source legitimately needs 2+ slashes, + // narrow this to the specific triplet shape (three non-empty, + // canonical-component-like segments) instead of widening the exemption + // list. + let slash_count = trimmed.matches('/').count(); + if slash_count >= 2 && !trimmed.starts_with('/') { + return Some(LegacyShape::SlashTriplet); + } + None +} + +fn canonical_component(value: &str) -> Option { + let out = value + .trim() + .to_ascii_lowercase() + .chars() + .map(|ch| { + if ch.is_ascii_alphanumeric() || ch == '-' || ch == '_' || ch == '.' { + ch + } else { + '-' + } + }) + .collect::() + .trim_matches(['-', '.']) + .to_string(); + (!out.is_empty()).then_some(out) +} + +#[cfg(test)] +#[path = "vocab_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/entity_resolution/vocab_tests.rs b/crates/shared/cortex/storage-sqlite/src/entity_resolution/vocab_tests.rs new file mode 100644 index 00000000..ffb2098a --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/entity_resolution/vocab_tests.rs @@ -0,0 +1,38 @@ +use super::*; + +#[test] +fn canonical_service_keys_and_splits_are_stable() { + assert_eq!( + logical_service_key(" Plex Media ").as_deref(), + Some("plex-media") + ); + assert_eq!( + service_instance_key("NAS Host", "Plex").as_deref(), + Some("nas-host/plex") + ); + assert_eq!( + split_service_instance_key("nas/plex"), + Some(("nas", "plex")) + ); + assert_eq!(split_service_instance_key("nas/proj/plex"), None); + assert_eq!(container_key_host("nas:abcdef"), Some("nas")); + assert_eq!(container_key_host("nas"), None); +} + +#[test] +fn legacy_shape_classifier_rejects_urls_and_paths() { + assert_eq!( + classify_legacy_shape("nas:plex"), + Some(LegacyShape::HostService) + ); + assert_eq!( + classify_legacy_shape("nas:proj:plex"), + Some(LegacyShape::HostProjectService) + ); + assert_eq!( + classify_legacy_shape("plex/plex/plex"), + Some(LegacyShape::SlashTriplet) + ); + assert_eq!(classify_legacy_shape("https://example.test/path"), None); + assert_eq!(classify_legacy_shape("/mnt/user/media"), None); +} diff --git a/crates/shared/cortex/storage-sqlite/src/entity_resolution_tests.rs b/crates/shared/cortex/storage-sqlite/src/entity_resolution_tests.rs new file mode 100644 index 00000000..70c51577 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/entity_resolution_tests.rs @@ -0,0 +1,339 @@ +use super::vocab::*; + +#[test] +fn canonical_service_keys_separate_logic_from_topology() { + assert_eq!(logical_service_key(" Plex "), Some("plex".to_string())); + assert_eq!( + service_instance_key("Nashost", " Plex "), + Some("nashost/plex".to_string()) + ); + assert_eq!( + split_service_instance_key("nashost/plex"), + Some(("nashost", "plex")) + ); +} + +#[test] +fn container_key_host_extracts_leading_host_segment() { + assert_eq!(container_key_host("nashost:abc123"), Some("nashost")); + // Extra colons (e.g. a malformed docker_host) still yield the leading + // segment as host, matching the container-key construction site which + // only ever emits a single colon between host and container id. + assert_eq!(container_key_host("nashost:abc:123"), Some("nashost")); + assert_eq!(container_key_host("no-colon-key"), None); + assert_eq!(container_key_host(":abc123"), None); + assert_eq!(container_key_host("nashost:"), None); + assert_eq!(container_key_host(""), None); +} + +#[test] +fn old_nested_service_shapes_are_classified_not_normalized() { + assert_eq!( + classify_legacy_shape("nashost:plex"), + Some(LegacyShape::HostService) + ); + assert_eq!( + classify_legacy_shape("nashost:plex:plex"), + Some(LegacyShape::HostProjectService) + ); + assert_eq!( + classify_legacy_shape("plex/plex/plex"), + Some(LegacyShape::SlashTriplet) + ); + assert_eq!(classify_legacy_shape("plex"), None); + assert_eq!(classify_legacy_shape("nashost/plex"), None); +} + +#[test] +fn legacy_shape_classifier_ignores_free_text_and_non_name_segments() { + let cases: &[(&str, Option)] = &[ + // Whitespace anywhere → free text, never a legacy key. + ("what is nashost:plex doing", None), + ("plex on nashost: status", None), + ("a/b/c d", None), + // Colon shapes where a segment lacks any ASCII alphabetic char. + ("10.0.0.5:443", None), + ("12:30", None), + ("nashost:8080", None), + (":plex", None), + ("nashost:", None), + // Absolute paths are not slash triplets. + ("/mnt/user/media", None), + ("/var/lib/docker", None), + // URLs/URIs are never legacy shapes: a colon segment containing `/` + // rejects in the colon branch and never falls through to the + // slash-triplet branch (`https://a.b/c` has two slashes). + ("http://example.com", None), + ("https://a.b/c", None), + ("agent-command://foo", None), + // Plan-asserted legacy shapes must keep classifying. + ("nashost:plex", Some(LegacyShape::HostService)), + ("nashost:plex:plex", Some(LegacyShape::HostProjectService)), + ("plex/plex/plex", Some(LegacyShape::SlashTriplet)), + ("a/b/c", Some(LegacyShape::SlashTriplet)), + ]; + for (input, expected) in cases { + assert_eq!( + classify_legacy_shape(input), + *expected, + "classify_legacy_shape({input:?})" + ); + } +} + +#[test] +fn key_grammar_edge_cases_pin_non_ascii_empty_and_long_inputs() { + // Non-ASCII-only input: every char maps to `-`, which trims to nothing. + assert_eq!(logical_service_key("\u{65e5}\u{672c}\u{8a9e}"), None); + // Mixed input: ASCII survives, non-ASCII maps to `-` and trims away. + assert_eq!(logical_service_key("caf\u{00e9}"), Some("caf".to_string())); + // Empty / whitespace-only input canonicalizes to nothing. + assert_eq!(logical_service_key(""), None); + assert_eq!(logical_service_key(" "), None); + // Pin current behavior for very long input: there is NO length bound in + // the key grammar — a >512-char name canonicalizes at full length. + let long = "a".repeat(600); + assert_eq!( + logical_service_key(&long).as_deref().map(str::len), + Some(600) + ); +} + +#[test] +fn canonical_keys_preserve_dots_in_hostnames() { + let cases: &[(&str, Option<&str>)] = &[ + ("nashost.lan", Some("nashost.lan")), + ("Nashost.LAN", Some("nashost.lan")), + (".plex.", Some("plex")), + ("-.plex.-", Some("plex")), + ("...", None), + ("plex media server", Some("plex-media-server")), + ]; + for (input, expected) in cases { + assert_eq!( + logical_service_key(input).as_deref(), + *expected, + "logical_service_key({input:?})" + ); + } + assert_eq!( + service_instance_key("nashost.lan", "plex"), + Some("nashost.lan/plex".to_string()) + ); +} + +use super::adapters::*; +use super::observation::*; + +#[test] +fn agent_docker_identity_extracts_structured_service_instance() { + let identity = AgentDockerIdentity { + agent_host: "Nashost".to_string(), + container_id: "abcdef1234567890".to_string(), + container_name: "plex".to_string(), + compose_project: Some("plex".to_string()), + compose_service: Some("plex".to_string()), + image: Some("lscr.io/linuxserver/plex:latest".to_string()), + stream: "stdout".to_string(), + observed_at: "2026-01-01T00:00:00Z".to_string(), + }; + let observations = observations_from_agent_docker_identity(&identity); + assert!(observations.iter().any(|o| { + o.kind == ObservationKind::ServiceInstance + && o.service_instance_key.as_deref() == Some("nashost/plex") + && o.logical_service_key.as_deref() == Some("plex") + && o.trust == ResolverTrust::Verified + && o.structured + })); +} + +#[test] +fn agent_docker_evidence_path_follows_actual_service_name_source() { + let with_compose = AgentDockerIdentity { + agent_host: "nashost".to_string(), + container_id: "abcdef1234567890".to_string(), + container_name: "plex-container".to_string(), + compose_project: Some("plex".to_string()), + compose_service: Some("plex".to_string()), + image: None, + stream: "stdout".to_string(), + observed_at: "2026-01-01T00:00:00Z".to_string(), + }; + let observations = observations_from_agent_docker_identity(&with_compose); + assert!(observations.iter().any(|o| { + o.kind == ObservationKind::LogicalService + && o.evidence_path == "agent_docker.compose_service" + })); + assert!(observations.iter().any(|o| { + o.kind == ObservationKind::ServiceInstance && o.evidence_path == "agent_docker.host_service" + })); + + // No compose service label: the logical name fell back to the container + // name, and the evidence path must say so. + let without_compose = AgentDockerIdentity { + compose_project: None, + compose_service: None, + ..with_compose + }; + let observations = observations_from_agent_docker_identity(&without_compose); + assert!(observations.iter().any(|o| { + o.kind == ObservationKind::LogicalService + && o.observed_key == "plex-container" + && o.evidence_path == "agent_docker.container_name" + })); + assert!(observations.iter().any(|o| { + o.kind == ObservationKind::ServiceInstance && o.evidence_path == "agent_docker.host_service" + })); +} + +#[test] +fn raw_app_label_does_not_create_logical_service_observation_by_itself() { + let observations = observations_from_raw_app_label( + "plex/plex/plex", + "nashost", + "log", + "42", + "2026-01-01T00:00:00Z", + ); + assert!( + observations + .iter() + .any(|o| o.kind == ObservationKind::RawAppLabel) + ); + assert!( + !observations + .iter() + .any(|o| o.kind == ObservationKind::LogicalService) + ); +} + +#[test] +fn safe_observation_display_redacts_sensitive_values() { + assert_eq!( + safe_display_value("https://user:pass@example.test/path"), + "[redacted]" + ); + assert_eq!( + safe_display_value("/home/jmagar/.cortex/token.txt"), + "[redacted]" + ); + // Scheme-less credentials must also redact (`:` + `@`, no `://`). + assert_eq!( + safe_display_value("user:pass@example.test/path"), + "[redacted]" + ); + assert_eq!(safe_display_value("plex"), "plex"); +} + +use super::resolver::*; + +#[test] +fn resolver_converges_duplicate_hosts_under_one_logical_service() { + let nashost = ResolverObservation { + kind: ObservationKind::ServiceInstance, + observed_key: "nashost/plex".to_string(), + display_label: "nashost/plex".to_string(), + host_key: Some("nashost".to_string()), + logical_service_key: Some("plex".to_string()), + service_instance_key: Some("nashost/plex".to_string()), + source_kind: "app_inventory".to_string(), + source_id: "inventory:nashost".to_string(), + evidence_path: "inventory.services.plex".to_string(), + observed_at: "2026-01-01T00:00:00Z".to_string(), + trust: ResolverTrust::Verified, + structured: true, + }; + let backuphost = ResolverObservation { + service_instance_key: Some("backuphost/plex".to_string()), + host_key: Some("backuphost".to_string()), + source_id: "inventory:backuphost".to_string(), + observed_key: "backuphost/plex".to_string(), + display_label: "backuphost/plex".to_string(), + ..nashost.clone() + }; + let decisions = resolve_observations(&[nashost, backuphost]); + assert!( + decisions + .iter() + .any(|d| { d.entity_type == ENTITY_TYPE_LOGICAL_SERVICE && d.canonical_key == "plex" }) + ); + assert!(decisions.iter().any(|d| { + d.entity_type == ENTITY_TYPE_SERVICE_INSTANCE && d.canonical_key == "nashost/plex" + })); + assert!(decisions.iter().any(|d| { + d.entity_type == ENTITY_TYPE_SERVICE_INSTANCE && d.canonical_key == "backuphost/plex" + })); +} + +#[test] +fn mixed_trust_evidence_pins_decision_trust_to_strongest() { + let verified = ResolverObservation { + kind: ObservationKind::LogicalService, + observed_key: "plex".to_string(), + display_label: "plex".to_string(), + host_key: None, + logical_service_key: Some("plex".to_string()), + service_instance_key: None, + source_kind: "app_inventory".to_string(), + source_id: "inventory:nashost".to_string(), + evidence_path: "inventory.services.name".to_string(), + observed_at: "2026-01-01T00:00:00Z".to_string(), + trust: ResolverTrust::Verified, + structured: true, + }; + let inferred = ResolverObservation { + source_id: "inventory:guess".to_string(), + trust: ResolverTrust::Inferred, + ..verified.clone() + }; + let decisions = resolve_observations(&[verified, inferred]); + let decision = decisions + .iter() + .find(|d| d.entity_type == ENTITY_TYPE_LOGICAL_SERVICE && d.canonical_key == "plex") + .expect("logical service decision"); + // min() over the strongest-first Ord picks Verified: weak corroborating + // observations must not weaken independently verified identity. + assert_eq!(decision.trust, ResolverTrust::Verified); +} + +#[test] +fn resolver_rejects_old_key_shapes_before_lookup() { + for input in ["nashost:plex", "nashost:plex:plex", "plex/plex/plex"] { + let diagnostic = diagnose_lookup_input(input); + assert_eq!(diagnostic.status, ResolverStatus::RejectedLegacyShape); + assert_eq!(diagnostic.reason, "rejected_legacy_shape"); + assert!(diagnostic.candidates.is_empty()); + } +} + +#[test] +fn weak_raw_labels_do_not_upgrade_themselves() { + let observations = + observations_from_raw_app_label("complex", "nashost", "log", "99", "2026-01-01T00:00:00Z"); + let decisions = resolve_observations(&observations); + assert!(!decisions.iter().any(|d| d.canonical_key == "plex")); + assert!( + !decisions + .iter() + .any(|d| d.entity_type == ENTITY_TYPE_LOGICAL_SERVICE) + ); +} + +#[test] +fn structured_agent_docker_metadata_resolves_without_central_docker_uri() { + let identity = AgentDockerIdentity { + agent_host: "nashost".to_string(), + container_id: "abcdef1234567890".to_string(), + container_name: "plex".to_string(), + compose_project: Some("plex".to_string()), + compose_service: Some("plex".to_string()), + image: Some("lscr.io/linuxserver/plex:latest".to_string()), + stream: "stdout".to_string(), + observed_at: "2026-01-01T00:00:00Z".to_string(), + }; + let observations = observations_from_agent_docker_identity(&identity); + let decisions = resolve_observations(&observations); + assert!(decisions.iter().any(|d| { + d.entity_type == ENTITY_TYPE_SERVICE_INSTANCE && d.canonical_key == "nashost/plex" + })); +} diff --git a/crates/shared/cortex/storage-sqlite/src/error_signatures.rs b/crates/shared/cortex/storage-sqlite/src/error_signatures.rs new file mode 100644 index 00000000..f5f93e49 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/error_signatures.rs @@ -0,0 +1,300 @@ +//! Database operations for the error signature detection subsystem. +//! +//! All functions take a `&r2d2::Pool` (i.e. `&DbPool`) +//! and are intended to be called from inside `tokio::task::spawn_blocking`. +//! They use rusqlite transactions, NOT sqlx. + +use anyhow::Result; +use cortex_domain::ErrorSignatureEntry; +use rusqlite::params; + +use super::pool::DbPool; + +// --------------------------------------------------------------------------- +// Cursor + +/// Return the last scanned log ID from `error_scan_cursor`. +pub fn cursor_get(pool: &DbPool) -> Result { + let conn = pool.get()?; + let id: i64 = conn.query_row( + "SELECT last_scanned_log_id FROM error_scan_cursor WHERE id = 1", + [], + |row| row.get(0), + )?; + Ok(id) +} + +/// Advance the cursor to `new_last_id` and record the scan completion time. +pub fn cursor_advance(conn: &rusqlite::Connection, new_last_id: i64) -> Result<()> { + conn.execute( + "UPDATE error_scan_cursor + SET last_scanned_log_id = ?1, + last_scan_completed_at = strftime('%Y-%m-%dT%H:%M:%fZ','now') + WHERE id = 1", + params![new_last_id], + )?; + Ok(()) +} + +// --------------------------------------------------------------------------- +// Upsert signature + +/// Parameters for `upsert_signature`. +pub struct UpsertSignatureParams<'a> { + pub hash: &'a str, + pub normalizer_version: i64, + pub template: &'a str, + pub sample_message: &'a str, + pub sample_hostname: &'a str, + pub sample_app_name: Option<&'a str>, + pub severity: &'a str, + pub first_seen_at: &'a str, + pub last_seen_at: &'a str, + pub delta: i64, +} + +/// Upsert a signature into `error_signatures`. +/// +/// On INSERT (first time we see this hash+version): write all sample fields. +/// On UPDATE (already exists): advance `last_seen_at` and add `delta` to +/// `total_count`. Sample fields are NEVER overwritten. +pub fn upsert_signature(conn: &rusqlite::Connection, p: UpsertSignatureParams<'_>) -> Result<()> { + conn.execute( + "INSERT INTO error_signatures + (signature_hash, normalizer_version, template, sample_message, + sample_hostname, sample_app_name, severity, + first_seen_at, last_seen_at, total_count) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10) + ON CONFLICT(signature_hash, normalizer_version) DO UPDATE SET + last_seen_at = CASE WHEN excluded.last_seen_at > last_seen_at + THEN excluded.last_seen_at ELSE last_seen_at END, + total_count = total_count + excluded.total_count", + params![ + p.hash, + p.normalizer_version, + p.template, + p.sample_message, + p.sample_hostname, + p.sample_app_name, + p.severity, + p.first_seen_at, + p.last_seen_at, + p.delta, + ], + )?; + Ok(()) +} + +// --------------------------------------------------------------------------- +// Window + +/// Insert a window record. Overlapping windows for the same (hash, ver, +/// start, end) are merged via `ON CONFLICT … DO UPDATE`. +pub fn insert_window( + conn: &rusqlite::Connection, + signature_hash: &str, + normalizer_version: i64, + window_start: &str, + window_end: &str, + count: i64, +) -> Result<()> { + conn.execute( + "INSERT INTO error_signature_windows + (signature_hash, normalizer_version, window_start, window_end, count_in_window) + VALUES (?1, ?2, ?3, ?4, ?5) + ON CONFLICT(signature_hash, normalizer_version, window_start, window_end) + DO UPDATE SET count_in_window = count_in_window + excluded.count_in_window", + params![ + signature_hash, + normalizer_version, + window_start, + window_end, + count + ], + )?; + Ok(()) +} + +// --------------------------------------------------------------------------- +// Ack / unack + +/// Record an ack or unack audit event. +pub fn record_ack_event( + conn: &rusqlite::Connection, + signature_hash: &str, + normalizer_version: i64, + event_type: &str, // "ack" | "unack" + actor: &str, + notes: Option<&str>, +) -> Result<()> { + conn.execute( + "INSERT INTO error_signature_ack_events + (signature_hash, normalizer_version, event_type, actor, notes) + VALUES (?1, ?2, ?3, ?4, ?5)", + params![signature_hash, normalizer_version, event_type, actor, notes], + )?; + Ok(()) +} + +/// Update the ack projection column on `error_signatures`. +/// Call this after `record_ack_event` inside the same transaction. +pub fn update_ack_projection( + conn: &rusqlite::Connection, + signature_hash: &str, + normalizer_version: i64, + acknowledged_at: Option<&str>, // Some → ack, None → clear (unack) + acknowledged_by: Option<&str>, +) -> Result<()> { + conn.execute( + "UPDATE error_signatures + SET acknowledged_at = ?3, acknowledged_by = ?4 + WHERE signature_hash = ?1 AND normalizer_version = ?2", + params![ + signature_hash, + normalizer_version, + acknowledged_at, + acknowledged_by, + ], + )?; + Ok(()) +} + +// --------------------------------------------------------------------------- +// Read queries + +/// Return a page of unacknowledged (or all, if `include_acknowledged`) +/// signatures ordered by `last_seen_at DESC`. +pub fn read_unaddressed_page( + pool: &DbPool, + limit: i64, + offset: i64, + include_acknowledged: bool, +) -> Result> { + let conn = pool.get()?; + let cutoff_1h = chrono::Utc::now() + .checked_sub_signed(chrono::TimeDelta::hours(1)) + .map(|dt| dt.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string()) + .unwrap_or_default(); + + let filter_clause = if include_acknowledged { + "" + } else { + "AND s.acknowledged_at IS NULL" + }; + + let sql = format!( + "SELECT + s.signature_hash, + s.template, + s.sample_message, + s.sample_hostname, + s.sample_app_name, + s.severity, + s.first_seen_at, + s.last_seen_at, + s.total_count, + COALESCE(w.total_1h, 0) AS count_last_1h, + s.acknowledged_at + FROM error_signatures s + LEFT JOIN ( + SELECT signature_hash, normalizer_version, SUM(count_in_window) AS total_1h + FROM error_signature_windows + WHERE window_end >= ?1 + GROUP BY signature_hash, normalizer_version + ) w USING (signature_hash, normalizer_version) + WHERE 1=1 {filter_clause} + ORDER BY s.last_seen_at DESC + LIMIT ?2 OFFSET ?3" + ); + + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map(params![cutoff_1h, limit, offset.max(0)], |row| { + Ok(ErrorSignatureEntry { + signature_hash: row.get(0)?, + template: row.get(1)?, + sample_message: row.get(2)?, + sample_hostname: row.get(3)?, + sample_app_name: row.get(4)?, + severity: row.get(5)?, + first_seen_at: row.get(6)?, + last_seen_at: row.get(7)?, + total_count: row.get(8)?, + count_last_1h: row.get(9)?, + acknowledged_at: row.get(10)?, + }) + })?; + + rows.collect::, _>>() + .map_err(Into::into) +} + +/// Look up a single signature by hash and normalizer version. Returns `None` if not found. +/// +/// The table PK is `(signature_hash, normalizer_version)`, so both parameters are +/// required to uniquely identify a row. +pub fn read_signature_by_hash( + pool: &DbPool, + signature_hash: &str, + normalizer_version: i64, +) -> Result> { + let conn = pool.get()?; + let cutoff_1h = chrono::Utc::now() + .checked_sub_signed(chrono::TimeDelta::hours(1)) + .map(|dt| dt.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string()) + .unwrap_or_default(); + + // The `USING (...)` join form is load-bearing for performance (bead q2e8): + // it lets SQLite push the outer `s.signature_hash = ?1` equality down into + // the materialized aggregate subquery, so only this hash's windows are summed + // (PK index seek, sub-ms). Rewriting it to an explicit `ON w.x = s.x` defeats + // that pushdown and degrades to a full GROUP BY over error_signature_windows + // (~100x slower at scale). Keep the `USING` form. + let mut stmt = conn.prepare( + "SELECT + s.signature_hash, + s.template, + s.sample_message, + s.sample_hostname, + s.sample_app_name, + s.severity, + s.first_seen_at, + s.last_seen_at, + s.total_count, + COALESCE(w.total_1h, 0) AS count_last_1h, + s.acknowledged_at + FROM error_signatures s + LEFT JOIN ( + SELECT signature_hash, normalizer_version, SUM(count_in_window) AS total_1h + FROM error_signature_windows + WHERE window_end >= ?3 + GROUP BY signature_hash, normalizer_version + ) w USING (signature_hash, normalizer_version) + WHERE s.signature_hash = ?1 AND s.normalizer_version = ?2 + LIMIT 1", + )?; + + let mut rows = stmt.query_map( + params![signature_hash, normalizer_version, cutoff_1h], + |row| { + Ok(ErrorSignatureEntry { + signature_hash: row.get(0)?, + template: row.get(1)?, + sample_message: row.get(2)?, + sample_hostname: row.get(3)?, + sample_app_name: row.get(4)?, + severity: row.get(5)?, + first_seen_at: row.get(6)?, + last_seen_at: row.get(7)?, + total_count: row.get(8)?, + count_last_1h: row.get(9)?, + acknowledged_at: row.get(10)?, + }) + }, + )?; + + rows.next().transpose().map_err(Into::into) +} + +#[cfg(test)] +#[path = "error_signatures_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/error_signatures_tests.rs b/crates/shared/cortex/storage-sqlite/src/error_signatures_tests.rs new file mode 100644 index 00000000..1349f9b6 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/error_signatures_tests.rs @@ -0,0 +1,403 @@ +use super::*; +use crate::config::StorageConfig; +use tempfile::TempDir; + +fn test_pool() -> (DbPool, TempDir) { + let dir = TempDir::new().unwrap(); + let storage = StorageConfig { + db_path: dir.path().join("test.db"), + pool_size: 1, + wal_mode: false, + ..Default::default() + }; + let pool = crate::init_pool(&storage).unwrap(); + (pool, dir) +} + +fn insert_sig(conn: &rusqlite::Connection, hash: &str, version: i64, last_seen_at: &str) { + upsert_signature( + conn, + UpsertSignatureParams { + hash, + normalizer_version: version, + template: &format!("template {hash}"), + sample_message: &format!("sample {hash}"), + sample_hostname: "host1", + sample_app_name: Some("sshd"), + severity: "err", + first_seen_at: "2026-06-13T00:00:00.000Z", + last_seen_at, + delta: 1, + }, + ) + .unwrap(); +} + +fn recent_timestamp(minutes_ago: i64) -> String { + chrono::Utc::now() + .checked_sub_signed(chrono::TimeDelta::minutes(minutes_ago)) + .unwrap() + .format("%Y-%m-%dT%H:%M:%S%.3fZ") + .to_string() +} + +#[test] +fn cursor_get_and_advance_round_trip() { + let (pool, _dir) = test_pool(); + assert_eq!(cursor_get(&pool).unwrap(), 0); + + { + let conn = pool.get().unwrap(); + cursor_advance(&conn, 42).unwrap(); + } + + assert_eq!(cursor_get(&pool).unwrap(), 42); + let conn = pool.get().unwrap(); + let completed_at: Option = conn + .query_row( + "SELECT last_scan_completed_at FROM error_scan_cursor WHERE id = 1", + [], + |r| r.get(0), + ) + .unwrap(); + assert!(completed_at.is_some()); +} + +#[test] +fn insert_window_merges_conflicting_counts_and_keeps_dimensions_separate() { + let (pool, _dir) = test_pool(); + let conn = pool.get().unwrap(); + + insert_window( + &conn, + "deadbeef", + 1, + "2026-06-13T00:00:00.000Z", + "2026-06-13T01:00:00.000Z", + 2, + ) + .unwrap(); + insert_window( + &conn, + "deadbeef", + 1, + "2026-06-13T00:00:00.000Z", + "2026-06-13T01:00:00.000Z", + 3, + ) + .unwrap(); + insert_window( + &conn, + "deadbeef", + 2, + "2026-06-13T00:00:00.000Z", + "2026-06-13T01:00:00.000Z", + 11, + ) + .unwrap(); + + let count_v1: i64 = conn + .query_row( + "SELECT count_in_window FROM error_signature_windows + WHERE signature_hash = 'deadbeef' AND normalizer_version = 1", + [], + |r| r.get(0), + ) + .unwrap(); + let count_v2: i64 = conn + .query_row( + "SELECT count_in_window FROM error_signature_windows + WHERE signature_hash = 'deadbeef' AND normalizer_version = 2", + [], + |r| r.get(0), + ) + .unwrap(); + + assert_eq!(count_v1, 5); + assert_eq!(count_v2, 11); +} + +#[test] +fn read_unaddressed_filters_acknowledged_and_sums_recent_windows() { + let (pool, _dir) = test_pool(); + let newer = recent_timestamp(5); + let older = recent_timestamp(10); + let stale = recent_timestamp(120); + + { + let conn = pool.get().unwrap(); + insert_sig(&conn, "unacked", 1, &newer); + insert_window(&conn, "unacked", 1, &older, &newer, 4).unwrap(); + insert_window(&conn, "unacked", 1, &stale, &stale, 99).unwrap(); + + insert_sig(&conn, "acked", 1, &older); + insert_window(&conn, "acked", 1, &older, &newer, 7).unwrap(); + update_ack_projection( + &conn, + "acked", + 1, + Some("2026-06-13T00:30:00.000Z"), + Some("admin"), + ) + .unwrap(); + } + + let unaddressed = read_unaddressed_page(&pool, 10, 0, false).unwrap(); + assert_eq!(unaddressed.len(), 1); + assert_eq!(unaddressed[0].signature_hash, "unacked"); + assert_eq!(unaddressed[0].count_last_1h, 4); + assert!(unaddressed[0].acknowledged_at.is_none()); + + let with_acknowledged = read_unaddressed_page(&pool, 10, 0, true).unwrap(); + assert_eq!( + with_acknowledged + .iter() + .map(|row| row.signature_hash.as_str()) + .collect::>(), + vec!["unacked", "acked"] + ); + let acked = with_acknowledged + .iter() + .find(|row| row.signature_hash == "acked") + .unwrap(); + assert_eq!(acked.count_last_1h, 7); + assert!(acked.acknowledged_at.is_some()); +} + +#[test] +fn read_signature_by_hash_returns_none_for_missing_or_wrong_version() { + let (pool, _dir) = test_pool(); + let recent = recent_timestamp(5); + { + let conn = pool.get().unwrap(); + insert_sig(&conn, "look-me-up", 3, &recent); + insert_window(&conn, "look-me-up", 3, &recent, &recent, 6).unwrap(); + } + + assert!( + read_signature_by_hash(&pool, "look-me-up", 2) + .unwrap() + .is_none() + ); + assert!( + read_signature_by_hash(&pool, "missing", 3) + .unwrap() + .is_none() + ); + + let row = read_signature_by_hash(&pool, "look-me-up", 3) + .unwrap() + .unwrap(); + assert_eq!(row.signature_hash, "look-me-up"); + assert_eq!(row.count_last_1h, 6); +} + +#[test] +fn update_ack_projection_unknown_hash_touches_no_rows() { + let (pool, _dir) = test_pool(); + { + let conn = pool.get().unwrap(); + insert_sig(&conn, "real", 1, "2026-06-13T00:00:00.000Z"); + + update_ack_projection( + &conn, + "forged", + 1, + Some("2026-06-13T00:30:00.000Z"), + Some("attacker"), + ) + .unwrap(); + + assert_eq!(conn.changes(), 0); + } + assert!( + read_signature_by_hash(&pool, "forged", 1) + .unwrap() + .is_none() + ); + let real = read_signature_by_hash(&pool, "real", 1).unwrap().unwrap(); + assert!(real.acknowledged_at.is_none()); +} + +#[test] +fn upsert_idempotency() { + let (pool, _dir) = test_pool(); + let conn = pool.get().unwrap(); + + // First insert + upsert_signature( + &conn, + UpsertSignatureParams { + hash: "aabbcc", + normalizer_version: 1, + template: "template text", + sample_message: "sample msg", + sample_hostname: "host1", + sample_app_name: Some("sshd"), + severity: "err", + first_seen_at: "2024-01-01T00:00:00.000Z", + last_seen_at: "2024-01-01T00:00:00.000Z", + delta: 5, + }, + ) + .unwrap(); + + // Second insert (same hash+version) should increment count and update last_seen_at + upsert_signature( + &conn, + UpsertSignatureParams { + hash: "aabbcc", + normalizer_version: 1, + template: "template text", + sample_message: "sample msg", + sample_hostname: "host2", + sample_app_name: Some("sshd"), + severity: "err", + first_seen_at: "2024-01-01T00:05:00.000Z", + last_seen_at: "2024-01-01T00:05:00.000Z", + delta: 3, + }, + ) + .unwrap(); + + let total: i64 = conn + .query_row( + "SELECT total_count FROM error_signatures WHERE signature_hash = 'aabbcc'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(total, 8, "total_count should be 5+3=8"); + + // sample_hostname should be the FIRST one (not overwritten) + let hostname: String = conn + .query_row( + "SELECT sample_hostname FROM error_signatures WHERE signature_hash = 'aabbcc'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!( + hostname, "host1", + "sample_hostname should not be overwritten on update" + ); +} + +#[test] +fn test_ack_event_appends_to_audit_chain() { + let (pool, _dir) = test_pool(); + let conn = pool.get().unwrap(); + + // First insert a signature to satisfy the ack_events foreign-key-style constraints + upsert_signature( + &conn, + UpsertSignatureParams { + hash: "deadbeef", + normalizer_version: 1, + template: "template", + sample_message: "sample", + sample_hostname: "host1", + sample_app_name: None, + severity: "err", + first_seen_at: "2024-01-01T00:00:00.000Z", + last_seen_at: "2024-01-01T00:00:00.000Z", + delta: 1, + }, + ) + .unwrap(); + + // Record ack then unack + record_ack_event(&conn, "deadbeef", 1, "ack", "admin", None).unwrap(); + record_ack_event(&conn, "deadbeef", 1, "unack", "admin", Some("reopening")).unwrap(); + + // Both events should be present (audit chain — no row deleted) + let count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM error_signature_ack_events + WHERE signature_hash = 'deadbeef'", + [], + |r| r.get::<_, i64>(0), + ) + .unwrap(); + assert_eq!(count, 2, "audit chain should have 2 events, not 1"); + + // Both event types present + let ack_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM error_signature_ack_events + WHERE signature_hash = 'deadbeef' AND event_type = 'ack'", + [], + |r| r.get::<_, i64>(0), + ) + .unwrap(); + let unack_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM error_signature_ack_events + WHERE signature_hash = 'deadbeef' AND event_type = 'unack'", + [], + |r| r.get::<_, i64>(0), + ) + .unwrap(); + assert_eq!(ack_count, 1, "one 'ack' event"); + assert_eq!(unack_count, 1, "one 'unack' event"); +} + +#[test] +fn test_update_ack_projection_sets_and_clears() { + let (pool, _dir) = test_pool(); + let conn = pool.get().unwrap(); + + upsert_signature( + &conn, + UpsertSignatureParams { + hash: "cafebabe", + normalizer_version: 1, + template: "template", + sample_message: "sample", + sample_hostname: "host1", + sample_app_name: None, + severity: "err", + first_seen_at: "2024-01-01T00:00:00.000Z", + last_seen_at: "2024-01-01T00:00:00.000Z", + delta: 1, + }, + ) + .unwrap(); + + // Acknowledge: set acknowledged_at and acknowledged_by + update_ack_projection( + &conn, + "cafebabe", + 1, + Some("2024-06-01T12:00:00.000Z"), + Some("admin"), + ) + .unwrap(); + + let acked_at: Option = conn + .query_row( + "SELECT acknowledged_at FROM error_signatures WHERE signature_hash = 'cafebabe'", + [], + |r| r.get::<_, Option>(0), + ) + .unwrap(); + assert!( + acked_at.is_some(), + "acknowledged_at should be set after ack" + ); + + // Unacknowledge: clear both columns + update_ack_projection(&conn, "cafebabe", 1, None, None).unwrap(); + + let acked_at_after: Option = conn + .query_row( + "SELECT acknowledged_at FROM error_signatures WHERE signature_hash = 'cafebabe'", + [], + |r| r.get::<_, Option>(0), + ) + .unwrap(); + assert!( + acked_at_after.is_none(), + "acknowledged_at should be NULL after unack" + ); +} diff --git a/crates/shared/cortex/storage-sqlite/src/graph.rs b/crates/shared/cortex/storage-sqlite/src/graph.rs new file mode 100644 index 00000000..228f6e29 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/graph.rs @@ -0,0 +1,3151 @@ +//! Derived investigation graph schema vocabulary. +//! +//! The graph is a rebuildable projection over authoritative source tables +//! (`logs`, heartbeats, AI session rollups, source inventory, signatures). Keep +//! vocabulary constants here so schema, extraction, service, adapters, and docs +//! do not drift into hand-written string variants. + +use std::time::Instant; + +use anyhow::{Context, Result}; +use parking_lot::Mutex; +use rusqlite::{OptionalExtension, params}; +use serde::{Deserialize, Serialize}; +use serde_json::Value; + +pub use cortex_domain::{ + GraphEntity as GraphEntityRow, GraphEntityCandidate as GraphEntityCandidateRow, +}; + +use super::graph_resolver_projection; +use super::pool::{DbPool, write_lock}; + +const GRAPH_REBUILD_CHUNK_SIZE: i64 = 10_000; +static GRAPH_REBUILD_LOCK: Mutex<()> = Mutex::new(()); +#[cfg(test)] +pub(crate) static GRAPH_TEST_LOCK: Mutex<()> = Mutex::new(()); + +pub const ENTITY_TYPE_HOST: &str = "host"; +pub const ENTITY_TYPE_CONTAINER: &str = "container"; +/// Retired legacy entity type: still present in migration CHECK constraints +/// (and gated for rejection), but deliberately absent from [`ENTITY_TYPES`] +/// so it no longer validates as a lookup type. +pub const ENTITY_TYPE_SERVICE: &str = "service"; +pub const ENTITY_TYPE_APP: &str = "app"; +pub const ENTITY_TYPE_SOURCE_IP: &str = "source_ip"; +pub const ENTITY_TYPE_AI_PROJECT: &str = "ai_project"; +pub const ENTITY_TYPE_AI_SESSION: &str = "ai_session"; +pub const ENTITY_TYPE_ERROR_SIGNATURE: &str = "error_signature"; +pub const ENTITY_TYPE_COMPOSE_PROJECT: &str = "compose_project"; +pub const ENTITY_TYPE_REVERSE_PROXY: &str = "reverse_proxy"; +pub const ENTITY_TYPE_DOMAIN: &str = "domain"; +pub const ENTITY_TYPE_NETWORK: &str = "network"; +pub const ENTITY_TYPE_STORAGE: &str = "storage"; +pub const ENTITY_TYPE_CONFIG_ARTIFACT: &str = "config_artifact"; +/// A git commit event observed in an agent-command or shell-history row. +pub const ENTITY_TYPE_GIT_COMMIT: &str = "git_commit"; +/// A human/identity principal (operator, authenticated username). +pub const ENTITY_TYPE_USER: &str = "user"; +/// A client endpoint (DNS client IP, MAC) distinct from a server `host`. +pub const ENTITY_TYPE_DEVICE: &str = "device"; +/// Canonical logical service identity (`plex`), resolver-owned. +pub const ENTITY_TYPE_LOGICAL_SERVICE: &str = + crate::entity_resolution::vocab::ENTITY_TYPE_LOGICAL_SERVICE; +/// Host-scoped runtime deployment of a logical service (`nashost/plex`). +pub const ENTITY_TYPE_SERVICE_INSTANCE: &str = + crate::entity_resolution::vocab::ENTITY_TYPE_SERVICE_INSTANCE; + +pub const ENTITY_TYPES: &[&str] = &[ + ENTITY_TYPE_HOST, + ENTITY_TYPE_CONTAINER, + ENTITY_TYPE_APP, + ENTITY_TYPE_SOURCE_IP, + ENTITY_TYPE_AI_PROJECT, + ENTITY_TYPE_AI_SESSION, + ENTITY_TYPE_ERROR_SIGNATURE, + ENTITY_TYPE_COMPOSE_PROJECT, + ENTITY_TYPE_REVERSE_PROXY, + ENTITY_TYPE_DOMAIN, + ENTITY_TYPE_NETWORK, + ENTITY_TYPE_STORAGE, + ENTITY_TYPE_CONFIG_ARTIFACT, + ENTITY_TYPE_GIT_COMMIT, + ENTITY_TYPE_USER, + ENTITY_TYPE_DEVICE, + ENTITY_TYPE_LOGICAL_SERVICE, + ENTITY_TYPE_SERVICE_INSTANCE, +]; + +pub const REL_OBSERVED_AS: &str = "observed_as"; +pub const REL_RUNS_ON: &str = "runs_on"; +pub const REL_EMITTED_BY: &str = "emitted_by"; +pub const REL_WORKED_ON: &str = "worked_on"; +pub const REL_MATCHES_SIGNATURE: &str = "matches_signature"; +pub const REL_DEFINES_SERVICE: &str = "defines_service"; +pub const REL_ROUTES_TO: &str = "routes_to"; +pub const REL_EXPOSES_DOMAIN: &str = "exposes_domain"; +pub const REL_ATTACHED_TO: &str = "attached_to"; +pub const REL_MOUNTS: &str = "mounts"; +pub const REL_BACKED_BY: &str = "backed_by"; +pub const REL_HAS_ARTIFACT: &str = "has_artifact"; +/// A user authenticated against a service/host (Authelia auth events). +pub const REL_AUTHENTICATED_AS: &str = "authenticated_as"; +/// A user or device accessed a domain/service/host (AdGuard DNS, shell use). +pub const REL_ACCESSED: &str = "accessed"; +/// A device communicates with a peer (UniFi flow data). Vocabulary-reserved. +pub const REL_COMMUNICATES_WITH: &str = "communicates_with"; +/// A service instance is a deployment of a logical service (resolver-owned). +pub const REL_INSTANCE_OF: &str = crate::entity_resolution::vocab::REL_INSTANCE_OF; + +pub const RELATIONSHIP_TYPES: &[&str] = &[ + REL_OBSERVED_AS, + REL_RUNS_ON, + REL_EMITTED_BY, + REL_WORKED_ON, + REL_MATCHES_SIGNATURE, + REL_DEFINES_SERVICE, + REL_ROUTES_TO, + REL_EXPOSES_DOMAIN, + REL_ATTACHED_TO, + REL_MOUNTS, + REL_BACKED_BY, + REL_HAS_ARTIFACT, + REL_AUTHENTICATED_AS, + REL_ACCESSED, + REL_COMMUNICATES_WITH, + REL_INSTANCE_OF, +]; + +pub const TRUST_VERIFIED: &str = "verified"; +pub const TRUST_CLAIMED: &str = "claimed"; +pub const TRUST_INFERRED: &str = "inferred"; +/// `correlated` is a *derivation method* (temporal co-occurrence), not an +/// epistemic status. Reserved for future query-time correlation edges; its +/// effective confidence is capped by `cortex_domain::graph_confidence::TRUST_CORRELATED_CEILING`. +pub const TRUST_CORRELATED: &str = "correlated"; +/// A relationship that was believed true but has been explicitly disproved or +/// retracted. Refuted edges are excluded from every traversal/query result and +/// must not be resurrected by rebuild. Set by manual override only. +pub const TRUST_REFUTED: &str = "refuted"; + +pub const TRUST_LEVELS: &[&str] = &[ + TRUST_VERIFIED, + TRUST_CLAIMED, + TRUST_INFERRED, + TRUST_CORRELATED, + TRUST_REFUTED, +]; + +/// Map a flat v1 reason code to its hierarchical v2 namespace +/// (`::`, OTel-attribute style). This registry gives +/// the flat vocabulary a queryable hierarchy — prefix matching (`source:docker:*`) +/// and family-level weighting — without changing the stored v1 string values. +/// The v2 strings are the planned migration target (see the contract). +pub fn reason_code_namespace(reason_code: &str) -> &'static str { + match reason_code { + REASON_SYSLOG_CLAIMED_HOSTNAME => "source:syslog:claimed_hostname", + REASON_LOG_APP_NAME => "source:log:app_name", + REASON_DOCKER_CONTAINER_ID => "source:docker:container_id", + REASON_DOCKER_SERVICE_LABEL => "source:docker:service_label", + REASON_DOCKER_NETWORK => "source:docker:network", + REASON_COMPOSE_CONFIG => "source:compose:config", + REASON_REVERSE_PROXY_CONFIG => "source:nginx:reverse_proxy_config", + REASON_INVENTORY_NODE => "source:inventory:node", + REASON_INVENTORY_SERVICE => "source:inventory:service", + REASON_STORAGE_PROBE => "source:storage:probe", + REASON_CONFIG_ARTIFACT => "source:compose:config_artifact", + REASON_HEARTBEAT_HOST_STATE => "source:heartbeat:host_state", + REASON_AGENT_COMMAND_SESSION => "source:agent:command_session", + REASON_AGENT_COMMAND_CWD_INFER => "source:agent:command_cwd_infer", + REASON_AGENT_COMMAND_GIT_COMMIT => "source:agent:git_commit", + REASON_SHELL_HISTORY_GIT_COMMIT => "source:shell:git_commit", + REASON_ADGUARD_CLIENT_QUERY => "source:adguard:client_query", + REASON_SHELL_HISTORY_USER => "source:shell:user", + REASON_AUTHELIA_AUTH => "source:authelia:auth", + REASON_AI_SESSION_PROJECT => "derivation:ai:session_project", + REASON_ERROR_SIGNATURE_MATCH => "derivation:error:signature_match", + REASON_RESOLVER_INSTANCE_OF => "derivation:resolver:instance_of", + REASON_RESOLVER_SERVICE_INSTANCE => "derivation:resolver:service_instance", + REASON_RESOLVER_RAW_APP_LABEL => "derivation:resolver:raw_app_label", + _ => "unknown:unknown:unknown", + } +} + +/// The hierarchical family of a reason code (the leading `source` / `derivation` +/// segment of its v2 namespace), for family-level weighting and filtering. +pub fn reason_code_family(reason_code: &str) -> &'static str { + reason_code_namespace(reason_code) + .split(':') + .next() + .unwrap_or("unknown") +} + +pub const SOURCE_KIND_LOG: &str = "log"; +pub const SOURCE_KIND_HEARTBEAT: &str = "heartbeat"; +pub const SOURCE_KIND_AI_SESSION_ROLLUP: &str = "ai_session_rollup"; +pub const SOURCE_KIND_SOURCE_INVENTORY: &str = "source_inventory"; +pub const SOURCE_KIND_APP_INVENTORY: &str = "app_inventory"; +pub const SOURCE_KIND_ERROR_SIGNATURE: &str = "error_signature"; + +pub const EVIDENCE_SOURCE_KINDS: &[&str] = &[ + SOURCE_KIND_LOG, + SOURCE_KIND_HEARTBEAT, + SOURCE_KIND_AI_SESSION_ROLLUP, + SOURCE_KIND_SOURCE_INVENTORY, + SOURCE_KIND_APP_INVENTORY, + SOURCE_KIND_ERROR_SIGNATURE, +]; + +pub const REASON_SYSLOG_CLAIMED_HOSTNAME: &str = "syslog_claimed_hostname"; +pub const REASON_LOG_APP_NAME: &str = "log_app_name"; +pub const REASON_DOCKER_CONTAINER_ID: &str = "docker_container_id"; +pub const REASON_DOCKER_SERVICE_LABEL: &str = "docker_service_label"; +pub const REASON_AI_SESSION_PROJECT: &str = "ai_session_project"; +pub const REASON_HEARTBEAT_HOST_STATE: &str = "heartbeat_host_state"; +pub const REASON_ERROR_SIGNATURE_MATCH: &str = "error_signature_match"; +pub const REASON_INVENTORY_NODE: &str = "inventory_node"; +pub const REASON_INVENTORY_SERVICE: &str = "inventory_service"; +pub const REASON_COMPOSE_CONFIG: &str = "compose_config"; +pub const REASON_REVERSE_PROXY_CONFIG: &str = "reverse_proxy_config"; +pub const REASON_DOCKER_NETWORK: &str = "docker_network"; +pub const REASON_STORAGE_PROBE: &str = "storage_probe"; +pub const REASON_CONFIG_ARTIFACT: &str = "config_artifact"; +/// Agent-command log row links its host context to the AI session that ran it. +/// `session_id` is a hard FK on the spool record, so this edge is verified. +pub const REASON_AGENT_COMMAND_SESSION: &str = "agent_command_session"; +/// Agent-command `cwd` infers the AI project worked on, used when the row +/// carries no clean project name (only the raw working directory). +pub const REASON_AGENT_COMMAND_CWD_INFER: &str = "agent_command_cwd_infer"; +/// An agent-command row whose command is a `git commit`/`git push` — links the +/// AI session and project to a `git_commit` entity. +pub const REASON_AGENT_COMMAND_GIT_COMMIT: &str = "agent_command_git_commit"; +/// A shell-history row whose command is a `git commit`/`git push` — links the +/// host to a `git_commit` entity. +pub const REASON_SHELL_HISTORY_GIT_COMMIT: &str = "shell_history_git_commit"; +/// AdGuard DNS query — links a client device to the queried domain. +pub const REASON_ADGUARD_CLIENT_QUERY: &str = "adguard_client_query"; +/// Shell-history identity — links the operating user to the host. +pub const REASON_SHELL_HISTORY_USER: &str = "shell_history_user"; +/// Authelia auth event — links a user to the service/host they authenticated to. +pub const REASON_AUTHELIA_AUTH: &str = "authelia_auth"; +/// Resolver linked a `service_instance` to its `logical_service`. +pub const REASON_RESOLVER_INSTANCE_OF: &str = + crate::entity_resolution::vocab::REASON_RESOLVER_INSTANCE_OF; +/// Resolver projected a `service_instance` from structured evidence. +/// **Vocabulary-reserved:** registered in the schema/reason registry but not +/// emitted by any projection path today — only `resolver_instance_of` is. +pub const REASON_RESOLVER_SERVICE_INSTANCE: &str = + crate::entity_resolution::vocab::REASON_RESOLVER_SERVICE_INSTANCE; +/// Resolver linked a raw observed app label to a host (never a self-upgrade +/// to logical-service identity). **Vocabulary-reserved:** registered in the +/// schema/reason registry but not emitted by any projection path today — +/// only `resolver_instance_of` is. +pub const REASON_RESOLVER_RAW_APP_LABEL: &str = + crate::entity_resolution::vocab::REASON_RESOLVER_RAW_APP_LABEL; + +pub const REASON_CODES: &[&str] = &[ + REASON_SYSLOG_CLAIMED_HOSTNAME, + REASON_LOG_APP_NAME, + REASON_DOCKER_CONTAINER_ID, + REASON_DOCKER_SERVICE_LABEL, + REASON_AI_SESSION_PROJECT, + REASON_HEARTBEAT_HOST_STATE, + REASON_ERROR_SIGNATURE_MATCH, + REASON_INVENTORY_NODE, + REASON_INVENTORY_SERVICE, + REASON_COMPOSE_CONFIG, + REASON_REVERSE_PROXY_CONFIG, + REASON_DOCKER_NETWORK, + REASON_STORAGE_PROBE, + REASON_CONFIG_ARTIFACT, + REASON_AGENT_COMMAND_SESSION, + REASON_AGENT_COMMAND_CWD_INFER, + REASON_AGENT_COMMAND_GIT_COMMIT, + REASON_SHELL_HISTORY_GIT_COMMIT, + REASON_ADGUARD_CLIENT_QUERY, + REASON_SHELL_HISTORY_USER, + REASON_AUTHELIA_AUTH, + REASON_RESOLVER_INSTANCE_OF, + REASON_RESOLVER_SERVICE_INSTANCE, + REASON_RESOLVER_RAW_APP_LABEL, +]; + +pub const PROJECTION_STATUS_NEVER_BUILT: &str = "never_built"; +pub const PROJECTION_STATUS_BUILDING: &str = "building"; +pub const PROJECTION_STATUS_READY: &str = "ready"; +pub const PROJECTION_STATUS_STALE: &str = "stale"; +pub const PROJECTION_STATUS_FAILED: &str = "failed"; + +pub const PROJECTION_STATUSES: &[&str] = &[ + PROJECTION_STATUS_NEVER_BUILT, + PROJECTION_STATUS_BUILDING, + PROJECTION_STATUS_READY, + PROJECTION_STATUS_STALE, + PROJECTION_STATUS_FAILED, +]; + +pub fn is_known_entity_type(value: &str) -> bool { + ENTITY_TYPES.contains(&value) +} + +pub fn is_known_relationship_type(value: &str) -> bool { + RELATIONSHIP_TYPES.contains(&value) +} + +pub fn is_known_reason_code(value: &str) -> bool { + REASON_CODES.contains(&value) +} + +pub fn is_known_trust_level(value: &str) -> bool { + TRUST_LEVELS.contains(&value) +} + +pub fn is_known_evidence_source_kind(value: &str) -> bool { + EVIDENCE_SOURCE_KINDS.contains(&value) +} + +pub fn canonical_graph_key(value: &str) -> Option { + normalized(value) +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct GraphProjectionStatus { + pub projection_status: String, + pub last_started_at: Option, + pub last_completed_at: Option, + pub source_watermark: String, + pub source_row_count: i64, + pub entity_count: i64, + pub relationship_count: i64, + pub evidence_count: i64, + pub is_degraded: bool, + pub last_error: Option, + pub last_runtime_ms: i64, + pub last_chunk_count: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub struct GraphRebuildStats { + pub source_row_count: i64, + pub entity_count: i64, + pub relationship_count: i64, + pub evidence_count: i64, + pub source_watermark: String, + pub runtime_ms: i64, + pub chunk_count: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq, Eq)] +pub enum GraphRebuildOutcome { + Rebuilt(GraphRebuildStats), + AlreadyRunning, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct GraphRelationshipRow { + pub id: i64, + pub relationship_key: String, + pub src_entity_id: i64, + pub dst_entity_id: i64, + pub relationship_type: String, + pub reason_code: String, + pub trust_level: String, + pub confidence: f64, + pub evidence_count: i64, + pub first_seen_at: Option, + pub last_seen_at: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct GraphEvidenceRow { + pub id: i64, + pub relationship_id: i64, + pub evidence_key: String, + pub source_kind: String, + pub source_id: String, + pub source_log_id: Option, + pub source_heartbeat_id: Option, + pub source_signature_hash: Option, + pub observed_at: String, + pub reason_code: String, + pub reason_text: Option, + pub confidence_delta: f64, + pub trust_level: String, + pub safe_excerpt: Option, + pub metadata_path: Option, + pub evidence_count: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct GraphAroundRows { + pub relationships: Vec, + pub entities: Vec, + pub evidence: Vec, + pub truncated: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct GraphSourceLogSummaryRow { + pub id: i64, + pub timestamp: String, + pub received_at: String, + pub hostname: String, + pub severity: String, + pub app_name: Option, + pub process_id: Option, + pub source_ip: String, + pub message: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct GraphEvidenceLookupRows { + pub evidence: GraphEvidenceRow, + pub relationship: GraphRelationshipRow, + pub src_entity: GraphEntityRow, + pub dst_entity: GraphEntityRow, + pub source_log_summary: Option, +} + +#[derive(Debug)] +pub(crate) struct LogGraphRow { + pub(crate) id: i64, + pub(crate) timestamp: String, + pub(crate) hostname: String, + pub(crate) app_name: Option, + pub(crate) source_ip: String, + pub(crate) ai_tool: Option, + pub(crate) ai_project: Option, + pub(crate) ai_session_id: Option, + pub(crate) metadata_json: Option, + /// The log message — for agent-command and shell-history rows this is the + /// (scrubbed) command surface, used to detect `git commit`/`git push`. + pub(crate) message: String, +} + +pub fn graph_projection_status(pool: &DbPool) -> Result { + let conn = pool.get()?; + conn.query_row( + "SELECT projection_status, last_started_at, last_completed_at, + source_watermark, source_row_count, entity_count, + relationship_count, evidence_count, is_degraded, last_error, + COALESCE(last_runtime_ms, 0), COALESCE(last_chunk_count, 0) + FROM graph_projection_meta WHERE id = 1", + [], + |row| { + Ok(GraphProjectionStatus { + projection_status: row.get(0)?, + last_started_at: row.get(1)?, + last_completed_at: row.get(2)?, + source_watermark: row.get(3)?, + source_row_count: row.get(4)?, + entity_count: row.get(5)?, + relationship_count: row.get(6)?, + evidence_count: row.get(7)?, + is_degraded: row.get::<_, i64>(8)? != 0, + last_error: row.get(9)?, + last_runtime_ms: row.get(10)?, + last_chunk_count: row.get(11)?, + }) + }, + ) + .context("read graph projection status") +} + +pub fn find_graph_entity_by_key( + pool: &DbPool, + entity_type: &str, + canonical_key: &str, +) -> Result> { + let conn = pool.get()?; + let key = canonical_graph_key(canonical_key).unwrap_or_else(|| canonical_key.to_string()); + conn.query_row( + "SELECT id, entity_type, canonical_key, display_label, source_kind, + source_id, trust_level, first_seen_at, last_seen_at + FROM graph_entities + WHERE entity_type = ?1 AND canonical_key = ?2", + params![entity_type, key], + graph_entity_from_row, + ) + .optional() + .map_err(Into::into) +} + +pub fn find_graph_entity_by_id(pool: &DbPool, entity_id: i64) -> Result> { + let conn = pool.get()?; + conn.query_row( + "SELECT id, entity_type, canonical_key, display_label, source_kind, + source_id, trust_level, first_seen_at, last_seen_at + FROM graph_entities + WHERE id = ?1", + [entity_id], + graph_entity_from_row, + ) + .optional() + .map_err(Into::into) +} + +pub fn find_graph_entities_by_alias( + pool: &DbPool, + alias_type: &str, + alias_key: &str, + limit: u32, +) -> Result> { + let conn = pool.get()?; + let key = canonical_graph_key(alias_key).unwrap_or_else(|| alias_key.to_string()); + let limit = limit.clamp(1, 500); + let mut stmt = conn.prepare( + "SELECT DISTINCT e.id, e.entity_type, e.canonical_key, e.display_label, e.source_kind, + e.source_id, e.trust_level, e.first_seen_at, e.last_seen_at, + a.alias_type, a.alias_key + FROM graph_entity_aliases a + JOIN graph_entities e ON e.id = a.entity_id + WHERE a.alias_type = ?1 AND a.alias_key = ?2 + ORDER BY e.last_seen_at DESC, e.id ASC + LIMIT ?3", + )?; + let rows = stmt + .query_map(params![alias_type, key, limit], |row| { + Ok(GraphEntityCandidateRow { + entity: GraphEntityRow { + id: row.get(0)?, + entity_type: row.get(1)?, + canonical_key: row.get(2)?, + display_label: row.get(3)?, + source_kind: row.get(4)?, + source_id: row.get(5)?, + trust_level: row.get(6)?, + first_seen_at: row.get(7)?, + last_seen_at: row.get(8)?, + }, + match_reason: "alias".to_string(), + alias_type: row.get(9)?, + alias_key: row.get(10)?, + }) + })? + .collect::>>()?; + Ok(rows) +} + +/// Fairly select up to `limit` relationships across neighbor entity types so a +/// single high-churn type (e.g. `error_signature`, whose rows are re-touched on +/// every error scan and therefore dominate a recency sort) can't crowd out the +/// rest of a host's neighborhood (apps, source_ips, …). +/// +/// `candidates` must already be ordered by recency (freshest first); each is +/// paired with its neighbor entity type. Selection is a round-robin across types +/// in first-appearance order, taking the freshest remaining item of each type +/// per round, until `limit` is reached. Returns the chosen relationships in a +/// stable recency-keyed order and whether anything was left out. +fn fair_share_relationships( + candidates: Vec<(String, GraphRelationshipRow)>, + limit: usize, + candidates_capped: bool, +) -> (Vec, bool) { + let total = candidates.len(); + // Bucket by neighbor type, preserving the incoming recency order and the + // order in which each type first appears. + let mut type_order: Vec = Vec::new(); + let mut buckets: std::collections::HashMap< + String, + std::collections::VecDeque, + > = std::collections::HashMap::new(); + for (neighbor_type, rel) in candidates { + if !buckets.contains_key(&neighbor_type) { + type_order.push(neighbor_type.clone()); + } + buckets.entry(neighbor_type).or_default().push_back(rel); + } + + let mut selected: Vec = Vec::with_capacity(limit.min(total)); + while selected.len() < limit { + let mut progressed = false; + for ty in &type_order { + if selected.len() >= limit { + break; + } + if let Some(rel) = buckets.get_mut(ty).and_then(|b| b.pop_front()) { + selected.push(rel); + progressed = true; + } + } + if !progressed { + break; // every bucket drained + } + } + + let truncated = candidates_capped || selected.len() < total; + // Re-sort the fair selection back into recency order for a stable response. + selected.sort_by(|a, b| b.last_seen_at.cmp(&a.last_seen_at).then(b.id.cmp(&a.id))); + (selected, truncated) +} + +/// Resolve `compose_project` entities by a bare project name. Compose-project +/// canonical keys are host-scoped (`:`, e.g. `devhost:axon`), so a +/// plain `key="axon"` never matches via [`find_graph_entity_by_key`]. This +/// matches the project portion (the segment after the last `:`) — or the full +/// key — and returns every host that runs that project as candidates, so the +/// caller can resolve a unique hit or surface the ambiguity. +pub fn find_compose_projects_by_project_name( + pool: &DbPool, + project_key: &str, + limit: u32, +) -> Result> { + let conn = pool.get()?; + let key = canonical_graph_key(project_key).unwrap_or_else(|| project_key.to_string()); + let limit = limit.clamp(1, 500) as usize; + let mut stmt = conn.prepare( + "SELECT id, entity_type, canonical_key, display_label, source_kind, + source_id, trust_level, first_seen_at, last_seen_at + FROM graph_entities + WHERE entity_type = ?1 + ORDER BY last_seen_at DESC, id ASC", + )?; + let rows = stmt + .query_map(params![ENTITY_TYPE_COMPOSE_PROJECT], graph_entity_from_row)? + .collect::>>()?; + let mut out = Vec::new(); + for entity in rows { + let matches = entity.canonical_key == key + || entity + .canonical_key + .rsplit_once(':') + .map(|(_, project)| project == key) + .unwrap_or(false); + if matches { + out.push(GraphEntityCandidateRow { + entity, + match_reason: "compose_project_name".to_string(), + alias_type: Some(ENTITY_TYPE_COMPOSE_PROJECT.to_string()), + alias_key: Some(key.clone()), + }); + if out.len() >= limit { + break; + } + } + } + Ok(out) +} + +pub fn graph_around_entity( + pool: &DbPool, + entity_id: i64, + limit: u32, + evidence_sample_limit: u32, +) -> Result { + let conn = pool.get()?; + let limit = limit.clamp(1, 500); + let evidence_sample_limit = evidence_sample_limit.clamp(0, 10); + // Pull a generous recency-ordered candidate pool (joined to the neighbor's + // entity type), then fair-share across types so apps/source_ips aren't + // buried under high-churn error_signature edges. Fetch one past the cap to + // detect whether even the candidate pool was truncated. + let candidate_cap = ((limit as usize).saturating_mul(8)).clamp(64, 4000); + let fetch_limit = (candidate_cap as u32).saturating_add(1); + + let mut stmt = conn.prepare( + "SELECT r.id, r.relationship_key, r.src_entity_id, r.dst_entity_id, r.relationship_type, + r.reason_code, r.trust_level, r.confidence, r.evidence_count, + r.first_seen_at, r.last_seen_at, ne.entity_type AS neighbor_type + FROM graph_relationships r + JOIN graph_entities ne + ON ne.id = CASE WHEN r.src_entity_id = ?1 THEN r.dst_entity_id ELSE r.src_entity_id END + WHERE (r.src_entity_id = ?1 OR r.dst_entity_id = ?1) + AND r.trust_level != 'refuted' + ORDER BY r.last_seen_at DESC, r.id DESC + LIMIT ?2", + )?; + let candidates = stmt + .query_map(params![entity_id, fetch_limit], |row| { + Ok((row.get::<_, String>(11)?, graph_relationship_from_row(row)?)) + })? + .collect::>>()?; + let candidates_capped = candidates.len() > candidate_cap; + let mut candidates = candidates; + candidates.truncate(candidate_cap); + let (relationships, truncated) = + fair_share_relationships(candidates, limit as usize, candidates_capped); + + let mut entity_ids = Vec::with_capacity(relationships.len() * 2 + 1); + entity_ids.push(entity_id); + for rel in &relationships { + entity_ids.push(rel.src_entity_id); + entity_ids.push(rel.dst_entity_id); + } + entity_ids.sort_unstable(); + entity_ids.dedup(); + let entities = graph_entities_by_ids(&conn, &entity_ids)?; + + let relationship_ids: Vec = relationships.iter().map(|rel| rel.id).collect(); + let evidence = + graph_evidence_for_relationships(&conn, &relationship_ids, evidence_sample_limit)?; + + Ok(GraphAroundRows { + relationships, + entities, + evidence, + truncated, + }) +} + +/// Absolute ceiling on graph-traversal depth. SQLite recursive CTEs stay in the +/// millisecond range at homelab scale up to depth 6 (research: degradation +/// begins past depth 6 / 100K entities). Callers' `max_depth` is clamped here. +pub const GRAPH_WALK_MAX_DEPTH: u8 = 6; + +/// One entity reached by a graph walk. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct GraphWalkEntity { + pub entity_type: String, + pub canonical_key: String, +} + +/// Bounded entity cap for the general-purpose n-hop walk ([`graph_walk_n_hops`]): +/// both the recursive CTE's row budget and the final result `LIMIT`. Mirrors +/// `GRAPH_SERVICE_TOPIC_ENTITY_CAP`'s +/// pattern but is set higher, because `graph_walk_n_hops` traverses every +/// relationship type (it has no +/// `GRAPH_SERVICE_TOPIC_RELATIONSHIPS`-style +/// restriction), so it needs more headroom to still return a useful topology +/// for its broader caller set (`search_logs_from_graph_related_entities`, +/// generic `topic_correlate` seeds). +pub const GRAPH_WALK_N_HOPS_ENTITY_CAP: usize = 500; + +/// Walk the investigation graph outward from a set of seed entities (matched by +/// `canonical_key`) and return every distinct entity reachable within +/// `max_depth` hops, including the seeds themselves (depth 0). +/// +/// Uses a `WITH RECURSIVE` CTE with `UNION` (not `UNION ALL`) so cycles in the +/// topology are de-duplicated before each iteration rather than looping. The +/// recursive join leads on `graph_relationships(src_entity_id)` / +/// `(dst_entity_id)` — both indexed — so each hop is index-served. `max_depth` +/// is clamped to `[1, GRAPH_WALK_MAX_DEPTH]`; an empty seed set returns empty. +/// +/// Unlike the private `graph_walk_service_topic` projection helper, +/// this walk traverses every relationship +/// type, so a densely connected homelab graph can otherwise expand without +/// bound. Both the recursive CTE and the final result are bounded at +/// [`GRAPH_WALK_N_HOPS_ENTITY_CAP`] entities (a single overall `LIMIT`, not a +/// per-level cap — SQLite's recursive CTE `LIMIT` has no per-level form). +/// Graphs at or under the cap see no behavior change. +/// +/// This is the reusable traversal primitive behind graph-anchored log fan-out +/// (`search_logs_from_graph_related_entities`) and topic correlation. +pub fn graph_walk_n_hops( + conn: &rusqlite::Connection, + start_keys: &[String], + max_depth: u8, +) -> Result> { + if start_keys.is_empty() { + return Ok(Vec::new()); + } + let depth = i64::from(max_depth.clamp(1, GRAPH_WALK_MAX_DEPTH)); + + let placeholders = vec!["?"; start_keys.len()].join(", "); + let sql = format!( + "WITH RECURSIVE graph_walk(entity_id, depth) AS ( + SELECT id, 0 FROM graph_entities WHERE canonical_key IN ({placeholders}) + UNION + SELECT CASE WHEN r.src_entity_id = gw.entity_id + THEN r.dst_entity_id ELSE r.src_entity_id END, + gw.depth + 1 + FROM graph_relationships r + JOIN graph_walk gw + ON r.src_entity_id = gw.entity_id OR r.dst_entity_id = gw.entity_id + WHERE gw.depth < ? AND r.trust_level != 'refuted' + LIMIT ? + ) + SELECT DISTINCT e.entity_type, e.canonical_key + FROM graph_entities e + JOIN graph_walk gw ON e.id = gw.entity_id + LIMIT ?" + ); + + let mut bindings: Vec = start_keys + .iter() + .map(|k| rusqlite::types::Value::Text(k.clone())) + .collect(); + bindings.push(rusqlite::types::Value::Integer(depth)); + bindings.push(rusqlite::types::Value::Integer( + GRAPH_WALK_N_HOPS_ENTITY_CAP as i64, + )); + bindings.push(rusqlite::types::Value::Integer( + GRAPH_WALK_N_HOPS_ENTITY_CAP as i64, + )); + + let mut stmt = conn.prepare(&sql)?; + let entities = stmt + .query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + Ok(GraphWalkEntity { + entity_type: row.get(0)?, + canonical_key: row.get(1)?, + }) + })? + .collect::>>()?; + Ok(entities) +} + +pub fn graph_evidence_by_id( + pool: &DbPool, + evidence_id: i64, +) -> Result> { + let conn = pool.get()?; + let Some((evidence, relationship)) = conn + .query_row( + "SELECT + e.id, e.relationship_id, e.evidence_key, e.source_kind, e.source_id, + e.source_log_id, e.source_heartbeat_id, e.source_signature_hash, + e.observed_at, e.reason_code, e.reason_text, e.confidence_delta, + e.trust_level, e.safe_excerpt, e.metadata_path, e.evidence_count, + r.id, r.relationship_key, r.src_entity_id, r.dst_entity_id, + r.relationship_type, r.reason_code, r.trust_level, r.confidence, + r.evidence_count, r.first_seen_at, r.last_seen_at + FROM graph_relationship_evidence e + JOIN graph_relationships r ON r.id = e.relationship_id + WHERE e.id = ?1", + [evidence_id], + |row| { + Ok(( + graph_evidence_from_row(row)?, + GraphRelationshipRow { + id: row.get(16)?, + relationship_key: row.get(17)?, + src_entity_id: row.get(18)?, + dst_entity_id: row.get(19)?, + relationship_type: row.get(20)?, + reason_code: row.get(21)?, + trust_level: row.get(22)?, + confidence: row.get(23)?, + evidence_count: row.get(24)?, + first_seen_at: row.get(25)?, + last_seen_at: row.get(26)?, + }, + )) + }, + ) + .optional()? + else { + return Ok(None); + }; + + let src_entity = conn.query_row( + "SELECT id, entity_type, canonical_key, display_label, source_kind, + source_id, trust_level, first_seen_at, last_seen_at + FROM graph_entities + WHERE id = ?1", + [relationship.src_entity_id], + graph_entity_from_row, + )?; + let dst_entity = conn.query_row( + "SELECT id, entity_type, canonical_key, display_label, source_kind, + source_id, trust_level, first_seen_at, last_seen_at + FROM graph_entities + WHERE id = ?1", + [relationship.dst_entity_id], + graph_entity_from_row, + )?; + let source_log_summary = match evidence.source_log_id { + Some(source_log_id) => conn + .query_row( + "SELECT id, timestamp, received_at, hostname, severity, app_name, + process_id, source_ip, message + FROM logs + WHERE id = ?1", + [source_log_id], + |row| { + Ok(GraphSourceLogSummaryRow { + id: row.get(0)?, + timestamp: row.get(1)?, + received_at: row.get(2)?, + hostname: row.get(3)?, + severity: row.get(4)?, + app_name: row.get(5)?, + process_id: row.get(6)?, + source_ip: row.get(7)?, + message: row.get(8)?, + }) + }, + ) + .optional()?, + None => None, + }; + + Ok(Some(GraphEvidenceLookupRows { + evidence, + relationship, + src_entity, + dst_entity, + source_log_summary, + })) +} + +fn graph_entities_by_ids(conn: &rusqlite::Connection, ids: &[i64]) -> Result> { + if ids.is_empty() { + return Ok(Vec::new()); + } + let placeholders = std::iter::repeat_n("?", ids.len()) + .collect::>() + .join(","); + let sql = format!( + "SELECT id, entity_type, canonical_key, display_label, source_kind, + source_id, trust_level, first_seen_at, last_seen_at + FROM graph_entities + WHERE id IN ({placeholders}) + ORDER BY entity_type ASC, display_label ASC" + ); + let params = ids.iter().copied().map(rusqlite::types::Value::Integer); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt + .query_map(rusqlite::params_from_iter(params), graph_entity_from_row)? + .collect::>>()?; + Ok(rows) +} + +fn graph_evidence_for_relationships( + conn: &rusqlite::Connection, + relationship_ids: &[i64], + evidence_sample_limit: u32, +) -> Result> { + if relationship_ids.is_empty() || evidence_sample_limit == 0 { + return Ok(Vec::new()); + } + let placeholders = std::iter::repeat_n("?", relationship_ids.len()) + .collect::>() + .join(","); + let sql = format!( + "SELECT id, relationship_id, evidence_key, source_kind, source_id, + source_log_id, source_heartbeat_id, source_signature_hash, + observed_at, reason_code, reason_text, confidence_delta, + trust_level, safe_excerpt, metadata_path, evidence_count + FROM ( + SELECT e.*, + ROW_NUMBER() OVER ( + PARTITION BY relationship_id + ORDER BY observed_at DESC, id DESC + ) AS rn + FROM graph_relationship_evidence e + WHERE relationship_id IN ({placeholders}) + ) + WHERE rn <= ? + ORDER BY relationship_id ASC, observed_at DESC" + ); + let mut values: Vec = relationship_ids + .iter() + .copied() + .map(rusqlite::types::Value::Integer) + .collect(); + values.push(rusqlite::types::Value::Integer( + evidence_sample_limit as i64, + )); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt + .query_map(rusqlite::params_from_iter(values), graph_evidence_from_row)? + .collect::>>()?; + Ok(rows) +} + +fn graph_entity_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(GraphEntityRow { + id: row.get(0)?, + entity_type: row.get(1)?, + canonical_key: row.get(2)?, + display_label: row.get(3)?, + source_kind: row.get(4)?, + source_id: row.get(5)?, + trust_level: row.get(6)?, + first_seen_at: row.get(7)?, + last_seen_at: row.get(8)?, + }) +} + +fn graph_relationship_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(GraphRelationshipRow { + id: row.get(0)?, + relationship_key: row.get(1)?, + src_entity_id: row.get(2)?, + dst_entity_id: row.get(3)?, + relationship_type: row.get(4)?, + reason_code: row.get(5)?, + trust_level: row.get(6)?, + confidence: row.get(7)?, + evidence_count: row.get(8)?, + first_seen_at: row.get(9)?, + last_seen_at: row.get(10)?, + }) +} + +fn graph_evidence_from_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(GraphEvidenceRow { + id: row.get(0)?, + relationship_id: row.get(1)?, + evidence_key: row.get(2)?, + source_kind: row.get(3)?, + source_id: row.get(4)?, + source_log_id: row.get(5)?, + source_heartbeat_id: row.get(6)?, + source_signature_hash: row.get(7)?, + observed_at: row.get(8)?, + reason_code: row.get(9)?, + reason_text: row.get(10)?, + confidence_delta: row.get(11)?, + trust_level: row.get(12)?, + safe_excerpt: row.get(13)?, + metadata_path: row.get(14)?, + evidence_count: row.get(15)?, + }) +} + +pub fn refresh_graph_projection(pool: &DbPool) -> Result { + let Some(_rebuild_guard) = GRAPH_REBUILD_LOCK.try_lock() else { + return Ok(GraphRebuildOutcome::AlreadyRunning); + }; + full_rebuild_locked(pool) +} + +/// Full rebuild body, run while holding [`GRAPH_REBUILD_LOCK`]. Rescans every +/// source row and atomically swaps the projection. Callers MUST hold the lock. +fn full_rebuild_locked(pool: &DbPool) -> Result { + mark_graph_projection_building(pool)?; + let started = Instant::now(); + match refresh_graph_projection_inner(pool, started) { + Ok(stats) => Ok(GraphRebuildOutcome::Rebuilt(stats)), + Err(err) => { + let _ = mark_graph_projection_failed(pool, &err); + Err(err) + } + } +} + +/// Incremental refresh: project only logs newer than the recorded watermark into +/// the live graph tables, then re-project the bounded heartbeat/error-signature +/// snapshots. Reuses the existing staging extractors but merges the delta into +/// the live tables by natural key (remapping staging row ids to final ids and +/// recomputing each `relationship_key` from final entity ids) instead of the +/// full DELETE-all swap. Falls back to a full rebuild when no usable prior +/// projection exists. Safe to run while the server ingests: the long log scan +/// builds into per-connection TEMP staging without the write lock; only the +/// final merge transaction briefly takes [`write_lock`]. +pub fn refresh_graph_projection_incremental(pool: &DbPool) -> Result { + let Some(_rebuild_guard) = GRAPH_REBUILD_LOCK.try_lock() else { + return Ok(GraphRebuildOutcome::AlreadyRunning); + }; + + // Contract-drift probe (downgrade → re-upgrade): a pre-resolver binary + // may have projected legacy `service` rows after migration 41 already + // ran (the CHECK constraint still allows the string for compat). Cheap + // EXISTS; when found, purge the legacy topology and force a full rebuild + // instead of merging deltas on top of a mixed-contract graph. + { + let mut conn = pool.get()?; + let legacy_rows: bool = conn.query_row( + "SELECT EXISTS(SELECT 1 FROM graph_entities WHERE entity_type = 'service')", + [], + |row| row.get(0), + )?; + if legacy_rows { + tracing::warn!( + contract = crate::entity_resolution::vocab::GRAPH_PROJECTION_CONTRACT_V2, + "legacy service topology detected in graph projection; \ + cleaning and forcing a full rebuild" + ); + graph_resolver_projection::cleanup_legacy_service_topology(&mut conn)?; + drop(conn); + return full_rebuild_locked(pool); + } + } + + let status = graph_projection_status(pool)?; + let after_log_id = if status.projection_status == "ready" && !status.is_degraded { + parse_log_watermark(&status.source_watermark) + } else { + None + }; + let Some(after_log_id) = after_log_id else { + // No usable prior projection (never built, mid-build, degraded, or an + // unparseable watermark) — fall back to a clean full rebuild. + return full_rebuild_locked(pool); + }; + + let started = Instant::now(); + match project_graph_delta(pool, after_log_id, started) { + Ok(stats) => Ok(GraphRebuildOutcome::Rebuilt(stats)), + Err(err) => { + let _ = mark_graph_projection_failed(pool, &err); + Err(err) + } + } +} + +/// Parse the `logs:` cursor out of a `graph_source_watermark` string of the +/// form `logs:N;heartbeats:M;signatures:K`. Returns None when absent/unparseable +/// so the caller can fall back to a full rebuild. +fn parse_log_watermark(watermark: &str) -> Option { + watermark + .split(';') + .find_map(|part| part.trim().strip_prefix("logs:")) + .and_then(|value| value.trim().parse::().ok()) +} + +fn project_graph_delta( + pool: &DbPool, + after_log_id: i64, + started: Instant, +) -> Result { + let mut conn = pool.get()?; + create_graph_staging_tables(&conn)?; + + // Build delta staging from logs newer than the watermark. Short per-chunk + // transactions against TEMP staging — no global write lock held here. + let mut delta_log_rows = 0_i64; + let mut chunk_count = 0_i64; + let max_log_id: i64 = + conn.query_row("SELECT COALESCE(MAX(id), 0) FROM logs", [], |r| r.get(0))?; + let mut cursor = after_log_id; + while cursor < max_log_id { + let rows = fetch_log_graph_rows(&conn, cursor, GRAPH_REBUILD_CHUNK_SIZE)?; + if rows.is_empty() { + break; + } + chunk_count += 1; + { + let tx = conn.transaction()?; + let mut entity_memo = EntityMemo::default(); + for row in &rows { + cursor = cursor.max(row.id); + delta_log_rows += 1; + extract_log_row(&tx, row, &mut entity_memo)?; + } + tx.commit()?; + } + } + + // Heartbeat + error-signature projections are bounded snapshots (capped at + // 14 days / signature count), so re-project them in full every pass. Their + // evidence keys are stable, so the merge upsert is idempotent. + extract_heartbeat_latest(&conn)?; + extract_error_signatures(&conn)?; + + let source_watermark = graph_source_watermark(&conn)?; + let runtime_ms = started.elapsed().as_millis().min(i64::MAX as u128) as i64; + let stats = merge_graph_delta(&mut conn, &source_watermark, runtime_ms, chunk_count)?; + let _ = conn.execute("DROP TABLE IF EXISTS _graph_entities_staging", []); + let _ = conn.execute("DROP TABLE IF EXISTS _graph_aliases_staging", []); + let _ = conn.execute("DROP TABLE IF EXISTS _graph_relationships_staging", []); + let _ = conn.execute("DROP TABLE IF EXISTS _graph_evidence_staging", []); + tracing::info!( + delta_log_rows, + chunk_count, + entities = stats.entity_count, + relationships = stats.relationship_count, + evidence = stats.evidence_count, + runtime_ms, + "graph incremental projection merged delta into live tables" + ); + Ok(stats) +} + +/// Merge the delta staging tables into the live graph tables by natural key. +/// +/// Runs as a single transaction under [`write_lock`]. Staging row ids are local +/// to this delta, so they are remapped to live ids and each `relationship_key` +/// is recomputed from final entity ids — keeping keys consistent with what the +/// last full rebuild wrote (which copied staging ids verbatim, so live id == +/// the staging id encoded in existing keys). +fn merge_graph_delta( + conn: &mut rusqlite::Connection, + source_watermark: &str, + runtime_ms: i64, + chunk_count: i64, +) -> Result { + let _guard = write_lock(); + let tx = conn.transaction()?; + + // 1. Entities: upsert by (entity_type, canonical_key), widening seen window. + tx.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, source_kind, source_id, + trust_level, first_seen_at, last_seen_at) + SELECT entity_type, canonical_key, display_label, source_kind, source_id, + trust_level, first_seen_at, last_seen_at + FROM _graph_entities_staging + WHERE true + ON CONFLICT(entity_type, canonical_key) DO UPDATE SET + display_label = CASE + WHEN graph_entities.display_label = '' THEN excluded.display_label + ELSE graph_entities.display_label END, + first_seen_at = CASE + WHEN graph_entities.first_seen_at IS NULL THEN excluded.first_seen_at + WHEN excluded.first_seen_at IS NULL THEN graph_entities.first_seen_at + WHEN excluded.first_seen_at < graph_entities.first_seen_at THEN excluded.first_seen_at + ELSE graph_entities.first_seen_at END, + last_seen_at = CASE + WHEN graph_entities.last_seen_at IS NULL THEN excluded.last_seen_at + WHEN excluded.last_seen_at IS NULL THEN graph_entities.last_seen_at + WHEN excluded.last_seen_at > graph_entities.last_seen_at THEN excluded.last_seen_at + ELSE graph_entities.last_seen_at END, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')", + [], + )?; + + // 2. Map staging entity ids -> live entity ids by natural key. + tx.execute("DROP TABLE IF EXISTS _graph_entity_idmap", [])?; + tx.execute( + "CREATE TEMP TABLE _graph_entity_idmap AS + SELECT s.id AS staging_id, f.id AS final_id + FROM _graph_entities_staging s + JOIN graph_entities f + ON f.entity_type = s.entity_type AND f.canonical_key = s.canonical_key", + [], + )?; + tx.execute( + "CREATE INDEX _ix_graph_entity_idmap ON _graph_entity_idmap(staging_id)", + [], + )?; + + // 3. Aliases: remap entity_id, upsert by natural key. + tx.execute( + "INSERT INTO graph_entity_aliases + (entity_id, alias_type, alias_key, alias_value, source_kind, + trust_level, first_seen_at, last_seen_at) + SELECT m.final_id, a.alias_type, a.alias_key, a.alias_value, a.source_kind, + a.trust_level, a.first_seen_at, a.last_seen_at + FROM _graph_aliases_staging a + JOIN _graph_entity_idmap m ON m.staging_id = a.entity_id + WHERE true + ON CONFLICT(entity_id, alias_type, alias_key, source_kind) DO UPDATE SET + last_seen_at = CASE + WHEN graph_entity_aliases.last_seen_at IS NULL THEN excluded.last_seen_at + WHEN excluded.last_seen_at IS NULL THEN graph_entity_aliases.last_seen_at + WHEN excluded.last_seen_at > graph_entity_aliases.last_seen_at THEN excluded.last_seen_at + ELSE graph_entity_aliases.last_seen_at END, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')", + [], + )?; + + // 4. Relationships: remap src/dst ids, recompute relationship_key from live + // ids, upsert. evidence_count is recomputed in step 7. + tx.execute( + "INSERT INTO graph_relationships + (relationship_key, src_entity_id, dst_entity_id, relationship_type, + reason_code, trust_level, confidence, evidence_count, + first_seen_at, last_seen_at) + SELECT ms.final_id || ':' || r.relationship_type || ':' || md.final_id, + ms.final_id, md.final_id, r.relationship_type, r.reason_code, + r.trust_level, r.confidence, 0, r.first_seen_at, r.last_seen_at + FROM _graph_relationships_staging r + JOIN _graph_entity_idmap ms ON ms.staging_id = r.src_entity_id + JOIN _graph_entity_idmap md ON md.staging_id = r.dst_entity_id + WHERE true + ON CONFLICT(relationship_key) DO UPDATE SET + confidence = MAX(graph_relationships.confidence, excluded.confidence), + first_seen_at = CASE + WHEN graph_relationships.first_seen_at IS NULL THEN excluded.first_seen_at + WHEN excluded.first_seen_at IS NULL THEN graph_relationships.first_seen_at + WHEN excluded.first_seen_at < graph_relationships.first_seen_at THEN excluded.first_seen_at + ELSE graph_relationships.first_seen_at END, + last_seen_at = CASE + WHEN graph_relationships.last_seen_at IS NULL THEN excluded.last_seen_at + WHEN excluded.last_seen_at IS NULL THEN graph_relationships.last_seen_at + WHEN excluded.last_seen_at > graph_relationships.last_seen_at THEN excluded.last_seen_at + ELSE graph_relationships.last_seen_at END, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')", + [], + )?; + + // 5. Map staging relationship ids -> live relationship ids. + tx.execute("DROP TABLE IF EXISTS _graph_rel_idmap", [])?; + tx.execute( + "CREATE TEMP TABLE _graph_rel_idmap AS + SELECT r.id AS staging_id, f.id AS final_id + FROM _graph_relationships_staging r + JOIN _graph_entity_idmap ms ON ms.staging_id = r.src_entity_id + JOIN _graph_entity_idmap md ON md.staging_id = r.dst_entity_id + JOIN graph_relationships f + ON f.relationship_key = ms.final_id || ':' || r.relationship_type || ':' || md.final_id", + [], + )?; + tx.execute( + "CREATE INDEX _ix_graph_rel_idmap ON _graph_rel_idmap(staging_id)", + [], + )?; + + // 6. Evidence: remap relationship_id, upsert by (relationship_id, key). Each + // log evidence key is unique per log row (never re-seen thanks to the + // watermark) and snapshot keys are stable, so replacing evidence_count is + // idempotent. + tx.execute( + "INSERT INTO graph_relationship_evidence + (relationship_id, evidence_key, source_kind, source_id, source_log_id, + source_heartbeat_id, source_signature_hash, observed_at, reason_code, + reason_text, confidence_delta, trust_level, safe_excerpt, metadata_path, + evidence_count) + SELECT rm.final_id, e.evidence_key, e.source_kind, e.source_id, e.source_log_id, + e.source_heartbeat_id, e.source_signature_hash, e.observed_at, e.reason_code, + e.reason_text, e.confidence_delta, e.trust_level, e.safe_excerpt, e.metadata_path, + e.evidence_count + FROM _graph_evidence_staging e + JOIN _graph_rel_idmap rm ON rm.staging_id = e.relationship_id + WHERE true + ON CONFLICT(relationship_id, evidence_key) DO UPDATE SET + evidence_count = excluded.evidence_count, + observed_at = CASE + WHEN excluded.observed_at > graph_relationship_evidence.observed_at THEN excluded.observed_at + ELSE graph_relationship_evidence.observed_at END", + [], + )?; + + // 7. Recompute evidence_count for relationships touched this pass. + tx.execute( + "UPDATE graph_relationships + SET evidence_count = ( + SELECT COALESCE(SUM(evidence_count), 0) + FROM graph_relationship_evidence + WHERE relationship_id = graph_relationships.id + ), + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + WHERE id IN (SELECT final_id FROM _graph_rel_idmap)", + [], + )?; + + // 8. Refresh projection metadata. source_row_count tracks the cumulative + // source footprint so `graph status` stays representative across deltas. + let entity_count: i64 = + tx.query_row("SELECT COUNT(*) FROM graph_entities", [], |r| r.get(0))?; + let relationship_count: i64 = + tx.query_row("SELECT COUNT(*) FROM graph_relationships", [], |r| r.get(0))?; + let evidence_count: i64 = tx.query_row( + "SELECT COUNT(*) FROM graph_relationship_evidence", + [], + |r| r.get(0), + )?; + let source_row_count: i64 = tx.query_row( + "SELECT (SELECT COUNT(*) FROM logs) + + (SELECT COUNT(*) FROM host_heartbeats_latest) + + (SELECT COUNT(*) FROM error_signatures)", + [], + |r| r.get(0), + )?; + tx.execute( + "UPDATE graph_projection_meta + SET projection_status = 'ready', + last_completed_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), + source_watermark = ?1, + source_row_count = ?2, + entity_count = ?3, + relationship_count = ?4, + evidence_count = ?5, + is_degraded = 0, + last_error = NULL, + last_runtime_ms = ?6, + last_chunk_count = ?7, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + WHERE id = 1", + params![ + source_watermark, + source_row_count, + entity_count, + relationship_count, + evidence_count, + runtime_ms, + chunk_count + ], + )?; + tx.execute("DROP TABLE IF EXISTS _graph_entity_idmap", [])?; + tx.execute("DROP TABLE IF EXISTS _graph_rel_idmap", [])?; + tx.commit()?; + + Ok(GraphRebuildStats { + source_row_count, + entity_count, + relationship_count, + evidence_count, + source_watermark: source_watermark.to_string(), + runtime_ms, + chunk_count, + }) +} + +fn refresh_graph_projection_inner(pool: &DbPool, started: Instant) -> Result { + let mut conn = pool.get()?; + create_graph_staging_tables(&conn)?; + + let mut source_row_count = 0_i64; + let mut chunk_count = 0_i64; + let max_log_id: i64 = + conn.query_row("SELECT COALESCE(MAX(id), 0) FROM logs", [], |r| r.get(0))?; + let mut after_id = 0_i64; + while after_id < max_log_id { + let rows = fetch_log_graph_rows(&conn, after_id, GRAPH_REBUILD_CHUNK_SIZE)?; + if rows.is_empty() { + break; + } + chunk_count += 1; + { + let tx = conn.transaction()?; + let mut entity_memo = EntityMemo::default(); + for row in &rows { + after_id = after_id.max(row.id); + source_row_count += 1; + extract_log_row(&tx, row, &mut entity_memo)?; + } + tx.commit()?; + } + mark_graph_projection_progress(&conn, source_row_count, chunk_count)?; + } + + source_row_count += extract_heartbeat_latest(&conn)?; + source_row_count += extract_error_signatures(&conn)?; + + let source_watermark = graph_source_watermark(&conn)?; + let runtime_ms = started.elapsed().as_millis().min(i64::MAX as u128) as i64; + let stats = swap_graph_projection( + &mut conn, + source_row_count, + &source_watermark, + runtime_ms, + chunk_count, + )?; + let _ = conn.execute("DROP TABLE IF EXISTS _graph_entities_staging", []); + let _ = conn.execute("DROP TABLE IF EXISTS _graph_aliases_staging", []); + let _ = conn.execute("DROP TABLE IF EXISTS _graph_relationships_staging", []); + let _ = conn.execute("DROP TABLE IF EXISTS _graph_evidence_staging", []); + Ok(stats) +} + +fn mark_graph_projection_building(pool: &DbPool) -> Result<()> { + let conn = pool.get()?; + let _guard = write_lock(); + conn.execute( + "UPDATE graph_projection_meta + SET projection_status = 'building', + last_started_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), + source_watermark = '', + source_row_count = 0, + entity_count = 0, + relationship_count = 0, + evidence_count = 0, + is_degraded = 0, + last_error = NULL, + last_runtime_ms = 0, + last_chunk_count = 0, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + WHERE id = 1", + [], + )?; + Ok(()) +} + +fn mark_graph_projection_progress( + conn: &rusqlite::Connection, + source_row_count: i64, + chunk_count: i64, +) -> Result<()> { + conn.execute( + "UPDATE graph_projection_meta + SET source_row_count = ?1, + last_chunk_count = ?2, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + WHERE id = 1 AND projection_status = 'building'", + params![source_row_count, chunk_count], + )?; + Ok(()) +} + +fn mark_graph_projection_failed(pool: &DbPool, err: &anyhow::Error) -> Result<()> { + let conn = pool.get()?; + let redacted = redact_error(&err.to_string()); + let _guard = write_lock(); + conn.execute( + "UPDATE graph_projection_meta + SET projection_status = 'failed', + is_degraded = 1, + last_error = ?1, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + WHERE id = 1", + [redacted], + )?; + Ok(()) +} + +fn create_graph_staging_tables(conn: &rusqlite::Connection) -> Result<()> { + conn.execute_batch( + "DROP TABLE IF EXISTS _graph_entities_staging; + DROP TABLE IF EXISTS _graph_aliases_staging; + DROP TABLE IF EXISTS _graph_relationships_staging; + DROP TABLE IF EXISTS _graph_evidence_staging; + + CREATE TEMP TABLE _graph_entities_staging ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entity_type TEXT NOT NULL, + canonical_key TEXT NOT NULL, + display_label TEXT NOT NULL, + source_kind TEXT NOT NULL DEFAULT '', + source_id TEXT NOT NULL DEFAULT '', + trust_level TEXT NOT NULL, + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(entity_type, canonical_key) + ); + CREATE TEMP TABLE _graph_aliases_staging ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entity_id INTEGER NOT NULL, + alias_type TEXT NOT NULL, + alias_key TEXT NOT NULL, + alias_value TEXT NOT NULL, + source_kind TEXT NOT NULL DEFAULT '', + trust_level TEXT NOT NULL, + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(entity_id, alias_type, alias_key, source_kind) + ); + CREATE TEMP TABLE _graph_relationships_staging ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relationship_key TEXT NOT NULL UNIQUE, + src_entity_id INTEGER NOT NULL, + dst_entity_id INTEGER NOT NULL, + relationship_type TEXT NOT NULL, + reason_code TEXT NOT NULL, + trust_level TEXT NOT NULL, + confidence REAL NOT NULL DEFAULT 0.0, + evidence_count INTEGER NOT NULL DEFAULT 0, + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(src_entity_id, dst_entity_id, relationship_type, relationship_key) + ); + CREATE TEMP TABLE _graph_evidence_staging ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relationship_id INTEGER NOT NULL, + evidence_key TEXT NOT NULL, + source_kind TEXT NOT NULL, + source_id TEXT NOT NULL DEFAULT '', + source_log_id INTEGER, + source_heartbeat_id INTEGER, + source_signature_hash TEXT, + observed_at TEXT NOT NULL, + reason_code TEXT NOT NULL, + reason_text TEXT, + confidence_delta REAL NOT NULL DEFAULT 0.0, + trust_level TEXT NOT NULL, + safe_excerpt TEXT, + metadata_path TEXT, + evidence_count INTEGER NOT NULL DEFAULT 1, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(relationship_id, evidence_key) + );", + )?; + Ok(()) +} + +fn fetch_log_graph_rows( + conn: &rusqlite::Connection, + after_id: i64, + limit: i64, +) -> Result> { + let mut stmt = conn.prepare( + "SELECT id, timestamp, hostname, app_name, source_ip, ai_tool, + ai_project, ai_session_id, metadata_json, message + FROM logs + WHERE id > ?1 + ORDER BY id ASC + LIMIT ?2", + )?; + let rows = stmt + .query_map(params![after_id, limit], |row| { + Ok(LogGraphRow { + id: row.get(0)?, + timestamp: row.get(1)?, + hostname: row.get(2)?, + app_name: row.get(3)?, + source_ip: row.get(4)?, + ai_tool: row.get(5)?, + ai_project: row.get(6)?, + ai_session_id: row.get(7)?, + metadata_json: row.get(8)?, + message: row.get(9)?, + }) + })? + .collect::>>()?; + Ok(rows) +} + +/// Per-chunk memo over `ensure_entity`, shared by every `extract_*_row` +/// projection (resolver and non-resolver alike): identical entity identities +/// collapse to one upsert per unique `(entity_type, canonical_key)` per chunk +/// transaction, instead of one upsert per log row that happens to name the +/// same host/container/app/etc. One instance is created per chunk (see the +/// `while cursor < max_log_id` loops in `project_graph_delta` and +/// `refresh_graph_projection_inner`) and threaded by `&mut` through +/// `extract_log_row` into every function it calls. "Staging" applies only to +/// the full rebuild path; incremental extraction writes the live tables +/// directly. +/// Tradeoff: within a chunk the memoised entity keeps the `last_seen_at` of +/// its first upsert, so it can lag later rows in the same chunk that +/// reference the same entity. This mirrors the resolver-path memo's +/// pre-existing tradeoff (syslog-mcp-g3fgk) — extended here to every +/// extract_*_row projection instead of the resolver path alone. +pub(crate) type EntityMemo = std::collections::HashMap<(&'static str, String), i64>; + +/// `metadata_json` is parsed exactly once per row (here, in the dispatcher) +/// and threaded down as `Option<&Value>` to every `extract_*_row` function +/// that needs it, instead of each one independently re-parsing the same +/// JSON string. +fn extract_log_row( + conn: &rusqlite::Connection, + row: &LogGraphRow, + memo: &mut EntityMemo, +) -> Result<()> { + let meta = parse_metadata(row.metadata_json.as_deref()); + let source_id = row.id.to_string(); + let source_entity = if let Some(key) = normalized(&row.source_ip) { + Some(ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_SOURCE_IP, + &key, + &row.source_ip, + SOURCE_KIND_LOG, + &source_id, + TRUST_VERIFIED, + Some(&row.timestamp), + Some(&row.timestamp), + )?) + } else { + None + }; + let host_entity = if let Some(key) = normalized(&row.hostname) { + Some(ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_HOST, + &key, + &row.hostname, + SOURCE_KIND_LOG, + &source_id, + TRUST_CLAIMED, + Some(&row.timestamp), + Some(&row.timestamp), + )?) + } else { + None + }; + + if let Some(host_id) = host_entity { + insert_alias( + conn, + host_id, + "hostname", + &normalize_key(&row.hostname), + &row.hostname, + SOURCE_KIND_LOG, + TRUST_CLAIMED, + Some(&row.timestamp), + Some(&row.timestamp), + )?; + } + if let (Some(source_id_entity), Some(host_id)) = (source_entity, host_entity) { + ensure_relationship_with_evidence( + conn, + source_id_entity, + host_id, + REL_OBSERVED_AS, + REASON_SYSLOG_CLAIMED_HOSTNAME, + TRUST_CLAIMED, + 0.6, + EvidenceInput { + evidence_key: evidence_bucket_key( + "log", + row.id, + REASON_SYSLOG_CLAIMED_HOSTNAME, + &row.timestamp, + ), + source_kind: SOURCE_KIND_LOG, + source_id: &source_id, + source_log_id: Some(row.id), + source_heartbeat_id: None, + source_signature_hash: None, + observed_at: &row.timestamp, + reason_text: Some("syslog header hostname claimed by sender"), + confidence_delta: 0.6, + trust_level: TRUST_CLAIMED, + safe_excerpt: Some(&row.hostname), + metadata_path: None, + }, + )?; + } + + // Nested slash-triplet app labels (`plex/plex/plex`) are stale defect + // shapes from the pre-resolver agent app-name format. They are never + // projected as `app` entities; structured agent-docker metadata carries + // the canonical identity instead. + let projectable_app = row + .app_name + .as_deref() + .and_then(normalized_value) + .filter(|app| { + !matches!( + crate::entity_resolution::classify_legacy_shape(app), + Some(crate::entity_resolution::LegacyShape::SlashTriplet) + ) + }); + if let Some(app_name) = projectable_app { + let app_id = ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_APP, + &normalize_key(app_name), + app_name, + SOURCE_KIND_LOG, + &source_id, + TRUST_INFERRED, + Some(&row.timestamp), + Some(&row.timestamp), + )?; + if let Some(host_id) = host_entity { + ensure_relationship_with_evidence( + conn, + app_id, + host_id, + REL_EMITTED_BY, + REASON_LOG_APP_NAME, + TRUST_INFERRED, + 0.5, + EvidenceInput { + evidence_key: evidence_bucket_key( + "log", + row.id, + REASON_LOG_APP_NAME, + &row.timestamp, + ), + source_kind: SOURCE_KIND_LOG, + source_id: &source_id, + source_log_id: Some(row.id), + source_heartbeat_id: None, + source_signature_hash: None, + observed_at: &row.timestamp, + reason_text: Some("log app_name observed on claimed host"), + confidence_delta: 0.5, + trust_level: TRUST_INFERRED, + safe_excerpt: Some(app_name), + metadata_path: Some("logs.app_name"), + }, + )?; + } + } + + extract_agent_command_row(conn, row, meta.as_ref(), memo)?; + extract_git_commit_row(conn, row, memo)?; + extract_user_device_row(conn, row, meta.as_ref(), memo)?; + extract_ai_log_row(conn, row, memo)?; + extract_docker_log_row(conn, row, meta.as_ref(), memo)?; + graph_resolver_projection::extract_agent_docker_row(conn, row, meta.as_ref(), memo)?; + Ok(()) +} + +/// Project user/device identity topology from identity-bearing log rows: +/// +/// * **AdGuard** DNS rows (`app_name` starts `adguard`): the `client` IP becomes +/// a `device` entity that `accessed` the queried `domain`. +/// * **Authelia** rows (`app_name == authelia`): the `username` becomes a `user` +/// entity that `authenticated_as` the host. +/// * **shell-history** rows (`shell-history://{host}/{user}/{shell}`): the user +/// segment becomes a `user` entity that `accessed` the host. +/// +/// These close "who/what did this" questions that previously dead-ended. +fn extract_user_device_row( + conn: &rusqlite::Connection, + row: &LogGraphRow, + meta: Option<&Value>, + memo: &mut EntityMemo, +) -> Result<()> { + let source_id = row.id.to_string(); + let app = row.app_name.as_deref().unwrap_or(""); + + // shell-history → user accessed host. + if let Some(rest) = row.source_ip.strip_prefix("shell-history://") { + let mut parts = rest.split('/'); + let host = parts.next().and_then(normalized_value); + let user = parts.next().and_then(normalized_value); + if let (Some(host), Some(user)) = (host, user) + && user != "unknown" + { + let user_key = format!("{}:{}", normalize_key(host), normalize_key(user)); + let user_id = ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_USER, + &user_key, + &user_key, + SOURCE_KIND_LOG, + &source_id, + TRUST_CLAIMED, + Some(&row.timestamp), + Some(&row.timestamp), + )?; + if let Some(host_id) = ensure_host_entity(conn, memo, host, &source_id, &row.timestamp)? + { + ensure_relationship_with_evidence( + conn, + user_id, + host_id, + REL_ACCESSED, + REASON_SHELL_HISTORY_USER, + TRUST_CLAIMED, + 0.7, + identity_evidence( + row, + &source_id, + REASON_SHELL_HISTORY_USER, + "shell history attributes commands to a user on this host", + 0.7, + TRUST_CLAIMED, + &user_key, + "logs.source_ip (shell-history)", + ), + )?; + } + } + return Ok(()); + } + + // AdGuard → device accessed domain. + if app.starts_with("adguard") { + let client = metadata_text(meta, &["client", "adguard.client"]).and_then(normalized_value); + let query = metadata_text(meta, &["query", "adguard.query"]).and_then(normalized_value); + if let (Some(client), Some(query)) = (client, query) { + let device_id = ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_DEVICE, + &normalize_key(client), + client, + SOURCE_KIND_LOG, + &source_id, + TRUST_VERIFIED, + Some(&row.timestamp), + Some(&row.timestamp), + )?; + let domain_id = ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_DOMAIN, + &normalize_key(query), + query, + SOURCE_KIND_LOG, + &source_id, + TRUST_INFERRED, + Some(&row.timestamp), + Some(&row.timestamp), + )?; + ensure_relationship_with_evidence( + conn, + device_id, + domain_id, + REL_ACCESSED, + REASON_ADGUARD_CLIENT_QUERY, + TRUST_INFERRED, + 0.9, + identity_evidence( + row, + &source_id, + REASON_ADGUARD_CLIENT_QUERY, + "adguard dns query links client device to domain", + 0.9, + TRUST_INFERRED, + query, + "metadata_json.client/query", + ), + )?; + } + return Ok(()); + } + + // Authelia → user authenticated_as host. + if app == "authelia" + && let Some(username) = + metadata_text(meta, &["username", "authelia.username"]).and_then(normalized_value) + && let Some(host) = normalized_value(&row.hostname) + { + let user_key = format!("{}:{}", normalize_key(host), normalize_key(username)); + let user_id = ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_USER, + &user_key, + &user_key, + SOURCE_KIND_LOG, + &source_id, + TRUST_CLAIMED, + Some(&row.timestamp), + Some(&row.timestamp), + )?; + if let Some(host_id) = ensure_host_entity(conn, memo, host, &source_id, &row.timestamp)? { + ensure_relationship_with_evidence( + conn, + user_id, + host_id, + REL_AUTHENTICATED_AS, + REASON_AUTHELIA_AUTH, + TRUST_CLAIMED, + 0.8, + identity_evidence( + row, + &source_id, + REASON_AUTHELIA_AUTH, + "authelia auth event links a user to this host", + 0.8, + TRUST_CLAIMED, + &user_key, + "metadata_json.username", + ), + )?; + } + } + Ok(()) +} + +/// Ensure a `host` entity (claimed) for a hostname, returning its id. +fn ensure_host_entity( + conn: &rusqlite::Connection, + memo: &mut EntityMemo, + hostname: &str, + source_id: &str, + timestamp: &str, +) -> Result> { + let Some(key) = normalized(hostname) else { + return Ok(None); + }; + Ok(Some(ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_HOST, + &key, + hostname, + SOURCE_KIND_LOG, + source_id, + TRUST_CLAIMED, + Some(timestamp), + Some(timestamp), + )?)) +} + +/// Build an `EvidenceInput` for an identity-projection relationship. +#[allow(clippy::too_many_arguments)] +fn identity_evidence<'a>( + row: &'a LogGraphRow, + source_id: &'a str, + reason_code: &'a str, + reason_text: &'a str, + confidence_delta: f64, + trust_level: &'a str, + safe_excerpt: &'a str, + metadata_path: &'a str, +) -> EvidenceInput<'a> { + EvidenceInput { + evidence_key: evidence_bucket_key("log", row.id, reason_code, &row.timestamp), + source_kind: SOURCE_KIND_LOG, + source_id, + source_log_id: Some(row.id), + source_heartbeat_id: None, + source_signature_hash: None, + observed_at: &row.timestamp, + reason_text: Some(reason_text), + confidence_delta, + trust_level, + safe_excerpt: Some(safe_excerpt), + metadata_path: Some(metadata_path), + } +} + +/// Source-IP prefix stamped on agent-command log rows by +/// `command_log::agent_record_to_entry`. These rows carry the raw `cwd` in the +/// `ai_project` column, so they are handled by `extract_agent_command_row` +/// rather than the generic AI extractor (which would key the session entity by +/// the full working-directory path and fragment it from transcript sessions). +const AGENT_COMMAND_SOURCE_PREFIX: &str = "agent-command://"; + +fn extract_ai_log_row( + conn: &rusqlite::Connection, + row: &LogGraphRow, + memo: &mut EntityMemo, +) -> Result<()> { + // Agent-command rows are owned by extract_agent_command_row: their + // `ai_project` is the raw cwd, not a clean project key. + if row.source_ip.starts_with(AGENT_COMMAND_SOURCE_PREFIX) { + return Ok(()); + } + let Some(project) = row.ai_project.as_deref().and_then(normalized_value) else { + return Ok(()); + }; + let Some(session) = row.ai_session_id.as_deref().and_then(normalized_value) else { + return Ok(()); + }; + let tool = row + .ai_tool + .as_deref() + .and_then(normalized_value) + .unwrap_or("unknown"); + let source_id = row.id.to_string(); + let project_id = ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_AI_PROJECT, + &normalize_key(project), + project, + SOURCE_KIND_LOG, + &source_id, + TRUST_VERIFIED, + Some(&row.timestamp), + Some(&row.timestamp), + )?; + let session_key = format!( + "{}:{}:{}", + normalize_key(project), + normalize_key(tool), + session + ); + let session_label = format!("{project}/{tool}/{session}"); + let session_id = ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_AI_SESSION, + &session_key, + &session_label, + SOURCE_KIND_LOG, + &source_id, + TRUST_VERIFIED, + Some(&row.timestamp), + Some(&row.timestamp), + )?; + ensure_relationship_with_evidence( + conn, + session_id, + project_id, + REL_WORKED_ON, + REASON_AI_SESSION_PROJECT, + TRUST_VERIFIED, + 0.9, + EvidenceInput { + evidence_key: evidence_bucket_key( + "log", + row.id, + REASON_AI_SESSION_PROJECT, + &row.timestamp, + ), + source_kind: SOURCE_KIND_LOG, + source_id: &source_id, + source_log_id: Some(row.id), + source_heartbeat_id: None, + source_signature_hash: None, + observed_at: &row.timestamp, + reason_text: Some("AI transcript metadata links session to project"), + confidence_delta: 0.9, + trust_level: TRUST_VERIFIED, + safe_excerpt: Some(&session_label), + metadata_path: Some("logs.ai_project/logs.ai_session_id"), + }, + )?; + Ok(()) +} + +/// Project the explicit agent-command → AI-session topology from a single +/// agent-command log row. +/// +/// Agent-command rows (`source_ip` starts with `agent-command://`) carry a hard +/// `session_id` FK and the executing host, plus the raw `cwd` in `ai_project`. +/// This builds two edges anchored on the session entity: +/// * session `REL_WORKED_ON` host — verified (0.95), the session provably ran +/// commands on this host (reason `agent_command_session`). +/// * session `REL_WORKED_ON` ai_project — inferred (0.7) from the cwd basename +/// (reason `agent_command_cwd_infer`), only when a project can be inferred. +/// +/// The session entity key reuses `extract_ai_log_row`'s +/// `{project}:{tool}:{session}` shape with the *inferred* project so +/// agent-command sessions converge with transcript-derived session entities for +/// the same session id, instead of fragmenting on the full cwd path. +fn extract_agent_command_row( + conn: &rusqlite::Connection, + row: &LogGraphRow, + meta: Option<&Value>, + memo: &mut EntityMemo, +) -> Result<()> { + if !row.source_ip.starts_with(AGENT_COMMAND_SOURCE_PREFIX) { + return Ok(()); + } + let Some(session) = row.ai_session_id.as_deref().and_then(normalized_value) else { + return Ok(()); + }; + let Some(host) = normalized(&row.hostname) else { + return Ok(()); + }; + let tool = row + .ai_tool + .as_deref() + .and_then(normalized_value) + .unwrap_or("unknown"); + let source_id = row.id.to_string(); + + // The cwd is stored in `ai_project` for these rows; fall back to the + // structured metadata copy if the column is empty. + let cwd = row + .ai_project + .as_deref() + .and_then(normalized_value) + .or_else(|| metadata_text(meta, &["agent_command.cwd"])); + let inferred_project = cwd.and_then(infer_project_from_cwd); + + let project_key_part = inferred_project + .as_deref() + .map(normalize_key) + .unwrap_or_else(|| "unknown".to_string()); + let project_label_part = inferred_project.as_deref().unwrap_or("unknown"); + let session_key = format!("{project_key_part}:{}:{session}", normalize_key(tool)); + let session_label = format!("{project_label_part}/{tool}/{session}"); + + let session_entity = ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_AI_SESSION, + &session_key, + &session_label, + SOURCE_KIND_LOG, + &source_id, + TRUST_VERIFIED, + Some(&row.timestamp), + Some(&row.timestamp), + )?; + + let host_entity = ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_HOST, + &host, + &row.hostname, + SOURCE_KIND_LOG, + &source_id, + TRUST_CLAIMED, + Some(&row.timestamp), + Some(&row.timestamp), + )?; + + // Verified anchor: the session executed commands on this host. + ensure_relationship_with_evidence( + conn, + session_entity, + host_entity, + REL_WORKED_ON, + REASON_AGENT_COMMAND_SESSION, + TRUST_VERIFIED, + 0.95, + EvidenceInput { + evidence_key: evidence_bucket_key( + "log", + row.id, + REASON_AGENT_COMMAND_SESSION, + &row.timestamp, + ), + source_kind: SOURCE_KIND_LOG, + source_id: &source_id, + source_log_id: Some(row.id), + source_heartbeat_id: None, + source_signature_hash: None, + observed_at: &row.timestamp, + reason_text: Some("agent command executed in this session on this host"), + confidence_delta: 0.95, + trust_level: TRUST_VERIFIED, + safe_excerpt: Some(&session_label), + metadata_path: Some("logs.ai_session_id/logs.hostname"), + }, + )?; + + // Inferred lane: the session worked on the project inferred from the cwd. + if let Some(project) = inferred_project.as_deref() { + let project_entity = ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_AI_PROJECT, + &normalize_key(project), + project, + SOURCE_KIND_LOG, + &source_id, + TRUST_INFERRED, + Some(&row.timestamp), + Some(&row.timestamp), + )?; + ensure_relationship_with_evidence( + conn, + session_entity, + project_entity, + REL_WORKED_ON, + REASON_AGENT_COMMAND_CWD_INFER, + TRUST_INFERRED, + 0.7, + EvidenceInput { + evidence_key: evidence_bucket_key( + "log", + row.id, + REASON_AGENT_COMMAND_CWD_INFER, + &row.timestamp, + ), + source_kind: SOURCE_KIND_LOG, + source_id: &source_id, + source_log_id: Some(row.id), + source_heartbeat_id: None, + source_signature_hash: None, + observed_at: &row.timestamp, + reason_text: Some("project inferred from agent command working directory"), + confidence_delta: 0.7, + trust_level: TRUST_INFERRED, + safe_excerpt: Some(project), + metadata_path: Some("logs.ai_project (cwd)"), + }, + )?; + } + Ok(()) +} + +/// Infer a clean project name from an agent command's working directory. +/// +/// Prefers the segment immediately following a `workspace` path component (the +/// homelab convention `~/workspace/`), so deep worktree paths like +/// `~/workspace/cortex/.claude/worktrees/foo` still resolve to `cortex`. Falls +/// back to the final path segment. Returns `None` for empty/`/`-only paths. +fn infer_project_from_cwd(cwd: &str) -> Option { + let segments: Vec<&str> = cwd + .split('/') + .map(str::trim) + .filter(|s| !s.is_empty()) + .collect(); + if let Some(pos) = segments.iter().position(|s| *s == "workspace") + && let Some(name) = segments.get(pos + 1) + { + return normalized_value(name).map(str::to_string); + } + segments + .last() + .and_then(|s| normalized_value(s).map(str::to_string)) +} + +/// True when a command surface is a `git commit` or `git push` invocation. +fn is_git_commit_command(message: &str) -> bool { + let lower = message.to_ascii_lowercase(); + lower.contains("git commit") || lower.contains("git push") +} + +/// Project a `git_commit` entity from an agent-command or shell-history row +/// whose command is a `git commit` / `git push`. +/// +/// Agent-command rows (which carry a session id and the cwd in `ai_project`) +/// produce a commit keyed by `{inferred_project}:{timestamp}`, linked back to +/// both the AI session (`worked_on`) and the project (`has_artifact`). Shell- +/// history rows carry no project/session, so they produce a commit keyed by +/// `{hostname}:{timestamp}` linked to the host (`emitted_by`). All edges are +/// inferred — the row proves a commit happened but not the exact SHA. +fn extract_git_commit_row( + conn: &rusqlite::Connection, + row: &LogGraphRow, + memo: &mut EntityMemo, +) -> Result<()> { + if !is_git_commit_command(&row.message) { + return Ok(()); + } + let source_id = row.id.to_string(); + let is_agent_command = row.source_ip.starts_with(AGENT_COMMAND_SOURCE_PREFIX); + let is_shell_history = row.source_ip.starts_with("shell-history://"); + if !is_agent_command && !is_shell_history { + return Ok(()); + } + + if is_agent_command { + let Some(session) = row.ai_session_id.as_deref().and_then(normalized_value) else { + return Ok(()); + }; + let tool = row + .ai_tool + .as_deref() + .and_then(normalized_value) + .unwrap_or("unknown"); + let inferred_project = row + .ai_project + .as_deref() + .and_then(normalized_value) + .and_then(infer_project_from_cwd); + let project_key_part = inferred_project + .as_deref() + .map(normalize_key) + .unwrap_or_else(|| "unknown".to_string()); + + let commit_key = format!("{project_key_part}:{}", row.timestamp); + let commit_entity = ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_GIT_COMMIT, + &commit_key, + &commit_key, + SOURCE_KIND_LOG, + &source_id, + TRUST_INFERRED, + Some(&row.timestamp), + Some(&row.timestamp), + )?; + + // session worked_on commit + let session_key = format!("{project_key_part}:{}:{session}", normalize_key(tool)); + let session_entity = ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_AI_SESSION, + &session_key, + &session_key, + SOURCE_KIND_LOG, + &source_id, + TRUST_VERIFIED, + Some(&row.timestamp), + Some(&row.timestamp), + )?; + ensure_relationship_with_evidence( + conn, + session_entity, + commit_entity, + REL_WORKED_ON, + REASON_AGENT_COMMAND_GIT_COMMIT, + TRUST_INFERRED, + 0.8, + EvidenceInput { + evidence_key: evidence_bucket_key( + "log", + row.id, + REASON_AGENT_COMMAND_GIT_COMMIT, + &row.timestamp, + ), + source_kind: SOURCE_KIND_LOG, + source_id: &source_id, + source_log_id: Some(row.id), + source_heartbeat_id: None, + source_signature_hash: None, + observed_at: &row.timestamp, + reason_text: Some("agent command ran a git commit/push in this session"), + confidence_delta: 0.8, + trust_level: TRUST_INFERRED, + safe_excerpt: Some(&commit_key), + metadata_path: Some("logs.message (git commit)"), + }, + )?; + + // commit has_artifact project + if let Some(project) = inferred_project.as_deref() { + let project_entity = ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_AI_PROJECT, + &normalize_key(project), + project, + SOURCE_KIND_LOG, + &source_id, + TRUST_INFERRED, + Some(&row.timestamp), + Some(&row.timestamp), + )?; + ensure_relationship_with_evidence( + conn, + commit_entity, + project_entity, + REL_HAS_ARTIFACT, + REASON_AGENT_COMMAND_GIT_COMMIT, + TRUST_INFERRED, + 0.9, + EvidenceInput { + evidence_key: evidence_bucket_key( + "log", + row.id, + REASON_AGENT_COMMAND_GIT_COMMIT, + &row.timestamp, + ), + source_kind: SOURCE_KIND_LOG, + source_id: &source_id, + source_log_id: Some(row.id), + source_heartbeat_id: None, + source_signature_hash: None, + observed_at: &row.timestamp, + reason_text: Some("git commit attributed to project via cwd"), + confidence_delta: 0.9, + trust_level: TRUST_INFERRED, + safe_excerpt: Some(project), + metadata_path: Some("logs.ai_project (cwd)"), + }, + )?; + } + return Ok(()); + } + + // Shell-history row: no session/project — key by host and link to the host. + let Some(host) = normalized(&row.hostname) else { + return Ok(()); + }; + let commit_key = format!("{host}:{}", row.timestamp); + let commit_entity = ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_GIT_COMMIT, + &commit_key, + &commit_key, + SOURCE_KIND_LOG, + &source_id, + TRUST_INFERRED, + Some(&row.timestamp), + Some(&row.timestamp), + )?; + let host_entity = ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_HOST, + &host, + &row.hostname, + SOURCE_KIND_LOG, + &source_id, + TRUST_CLAIMED, + Some(&row.timestamp), + Some(&row.timestamp), + )?; + ensure_relationship_with_evidence( + conn, + commit_entity, + host_entity, + REL_EMITTED_BY, + REASON_SHELL_HISTORY_GIT_COMMIT, + TRUST_INFERRED, + 0.7, + EvidenceInput { + evidence_key: evidence_bucket_key( + "log", + row.id, + REASON_SHELL_HISTORY_GIT_COMMIT, + &row.timestamp, + ), + source_kind: SOURCE_KIND_LOG, + source_id: &source_id, + source_log_id: Some(row.id), + source_heartbeat_id: None, + source_signature_hash: None, + observed_at: &row.timestamp, + reason_text: Some("shell history ran a git commit/push on this host"), + confidence_delta: 0.7, + trust_level: TRUST_INFERRED, + safe_excerpt: Some(&commit_key), + metadata_path: Some("logs.message (git commit)"), + }, + )?; + Ok(()) +} + +fn extract_docker_log_row( + conn: &rusqlite::Connection, + row: &LogGraphRow, + meta: Option<&Value>, + memo: &mut EntityMemo, +) -> Result<()> { + if !row.source_ip.starts_with("docker://") && !row.source_ip.starts_with("docker-event://") { + return Ok(()); + } + let parsed = parse_docker_source(&row.source_ip); + let docker_host = metadata_text(meta, &["docker_host", "docker.host"]) + .or(parsed.host) + .and_then(normalized_value); + let container = metadata_text(meta, &["container_id", "docker.container_id"]) + .or_else(|| metadata_text(meta, &["container_name", "docker.container_name"])) + .or(parsed.container) + .and_then(normalized_value); + let Some(docker_host) = docker_host else { + return Ok(()); + }; + let Some(container) = container else { + return Ok(()); + }; + let source_id = row.id.to_string(); + let host_id = ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_HOST, + &normalize_key(docker_host), + docker_host, + SOURCE_KIND_LOG, + &source_id, + TRUST_VERIFIED, + Some(&row.timestamp), + Some(&row.timestamp), + )?; + let container_key = format!( + "{}:{}", + normalize_key(docker_host), + normalize_key(container) + ); + let container_label = format!("{docker_host}/{container}"); + let container_id = ensure_entity_memoized( + conn, + memo, + ENTITY_TYPE_CONTAINER, + &container_key, + &container_label, + SOURCE_KIND_LOG, + &source_id, + TRUST_VERIFIED, + Some(&row.timestamp), + Some(&row.timestamp), + )?; + ensure_relationship_with_evidence( + conn, + container_id, + host_id, + REL_RUNS_ON, + REASON_DOCKER_CONTAINER_ID, + TRUST_VERIFIED, + 0.9, + EvidenceInput { + evidence_key: evidence_bucket_key( + "log", + row.id, + REASON_DOCKER_CONTAINER_ID, + &row.timestamp, + ), + source_kind: SOURCE_KIND_LOG, + source_id: &source_id, + source_log_id: Some(row.id), + source_heartbeat_id: None, + source_signature_hash: None, + observed_at: &row.timestamp, + reason_text: Some("docker source identity links container to host"), + confidence_delta: 0.9, + trust_level: TRUST_VERIFIED, + safe_excerpt: Some(&container_label), + metadata_path: Some("logs.source_ip/metadata_json"), + }, + )?; + + // Hard break (entity_resolution_v2): central-pull rows keep verified + // host/container edges only. Legacy `service` topology + // (`host:project:service`) is no longer emitted from any projection + // path; canonical service identity comes exclusively from resolver + // decisions over structured agent-docker metadata and verified + // inventory (`logical_service` / `service_instance`). + Ok(()) +} + +fn extract_heartbeat_latest(conn: &rusqlite::Connection) -> Result { + let mut stmt = conn.prepare( + "SELECT heartbeat_id, host_id, hostname, sampled_at + FROM host_heartbeats_latest + ORDER BY hostname ASC", + )?; + let rows = stmt + .query_map([], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + )) + })? + .collect::>>()?; + for (heartbeat_id, host_id_value, hostname, sampled_at) in &rows { + let Some(host_key) = normalized(hostname) else { + continue; + }; + let host_id = ensure_entity( + conn, + ENTITY_TYPE_HOST, + &host_key, + hostname, + SOURCE_KIND_HEARTBEAT, + &heartbeat_id.to_string(), + TRUST_VERIFIED, + Some(sampled_at), + Some(sampled_at), + )?; + insert_alias( + conn, + host_id, + "hostname", + &host_key, + hostname, + SOURCE_KIND_HEARTBEAT, + TRUST_VERIFIED, + Some(sampled_at), + Some(sampled_at), + )?; + insert_alias( + conn, + host_id, + "heartbeat_host_id", + &normalize_key(host_id_value), + host_id_value, + SOURCE_KIND_HEARTBEAT, + TRUST_VERIFIED, + Some(sampled_at), + Some(sampled_at), + )?; + } + Ok(rows.len() as i64) +} + +fn extract_error_signatures(conn: &rusqlite::Connection) -> Result { + let mut stmt = conn.prepare( + "SELECT signature_hash, normalizer_version, template, sample_hostname, + sample_app_name, first_seen_at, last_seen_at, total_count + FROM error_signatures + ORDER BY last_seen_at DESC", + )?; + let rows = stmt + .query_map([], |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + row.get::<_, Option>(4)?, + row.get::<_, String>(5)?, + row.get::<_, String>(6)?, + row.get::<_, i64>(7)?, + )) + })? + .collect::>>()?; + for (hash, version, template, hostname, app_name, first_seen, last_seen, total_count) in &rows { + let signature_key = format!("{hash}:{version}"); + let signature_id = ensure_entity( + conn, + ENTITY_TYPE_ERROR_SIGNATURE, + &signature_key, + &template.chars().take(120).collect::(), + SOURCE_KIND_ERROR_SIGNATURE, + &signature_key, + TRUST_INFERRED, + Some(first_seen), + Some(last_seen), + )?; + if let Some(app) = app_name.as_deref().and_then(normalized_value) { + let app_id = ensure_entity( + conn, + ENTITY_TYPE_APP, + &normalize_key(app), + app, + SOURCE_KIND_ERROR_SIGNATURE, + &signature_key, + TRUST_INFERRED, + Some(first_seen), + Some(last_seen), + )?; + ensure_relationship_with_evidence( + conn, + app_id, + signature_id, + REL_MATCHES_SIGNATURE, + REASON_ERROR_SIGNATURE_MATCH, + TRUST_INFERRED, + 0.7, + EvidenceInput { + evidence_key: format!("signature:{signature_key}:app"), + source_kind: SOURCE_KIND_ERROR_SIGNATURE, + source_id: &signature_key, + source_log_id: None, + source_heartbeat_id: None, + source_signature_hash: Some(hash), + observed_at: last_seen, + reason_text: Some("error signature projection links app to template"), + confidence_delta: 0.7, + trust_level: TRUST_INFERRED, + safe_excerpt: Some(template), + metadata_path: Some("error_signatures"), + }, + )?; + } + if let Some(host_key) = normalized(hostname) { + let host_id = ensure_entity( + conn, + ENTITY_TYPE_HOST, + &host_key, + hostname, + SOURCE_KIND_ERROR_SIGNATURE, + &signature_key, + TRUST_CLAIMED, + Some(first_seen), + Some(last_seen), + )?; + ensure_relationship_with_evidence( + conn, + host_id, + signature_id, + REL_MATCHES_SIGNATURE, + REASON_ERROR_SIGNATURE_MATCH, + TRUST_INFERRED, + 0.5, + EvidenceInput { + evidence_key: format!("signature:{signature_key}:host"), + source_kind: SOURCE_KIND_ERROR_SIGNATURE, + source_id: &signature_key, + source_log_id: None, + source_heartbeat_id: None, + source_signature_hash: Some(hash), + observed_at: last_seen, + reason_text: Some("error signature projection links claimed host to template"), + confidence_delta: 0.5, + trust_level: TRUST_INFERRED, + safe_excerpt: Some(template), + metadata_path: Some("error_signatures"), + }, + )?; + } + let _ = total_count; + } + Ok(rows.len() as i64) +} + +#[allow(clippy::too_many_arguments)] +fn ensure_entity( + conn: &rusqlite::Connection, + entity_type: &str, + canonical_key: &str, + display_label: &str, + source_kind: &str, + source_id: &str, + trust_level: &str, + first_seen_at: Option<&str>, + last_seen_at: Option<&str>, +) -> Result { + // prepare_cached throughout this helper and its siblings: these run 6-8 + // times PER LOG ROW during a full rebuild — re-parsing the SQL each call + // dominated rebuild time on large DBs (full-review PH2). + conn.prepare_cached( + "INSERT INTO _graph_entities_staging + (entity_type, canonical_key, display_label, source_kind, source_id, + trust_level, first_seen_at, last_seen_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) + ON CONFLICT(entity_type, canonical_key) DO UPDATE SET + display_label = CASE + WHEN _graph_entities_staging.display_label = '' THEN excluded.display_label + ELSE _graph_entities_staging.display_label END, + first_seen_at = CASE + WHEN _graph_entities_staging.first_seen_at IS NULL THEN excluded.first_seen_at + WHEN excluded.first_seen_at IS NULL THEN _graph_entities_staging.first_seen_at + WHEN excluded.first_seen_at < _graph_entities_staging.first_seen_at THEN excluded.first_seen_at + ELSE _graph_entities_staging.first_seen_at END, + last_seen_at = CASE + WHEN _graph_entities_staging.last_seen_at IS NULL THEN excluded.last_seen_at + WHEN excluded.last_seen_at IS NULL THEN _graph_entities_staging.last_seen_at + WHEN excluded.last_seen_at > _graph_entities_staging.last_seen_at THEN excluded.last_seen_at + ELSE _graph_entities_staging.last_seen_at END, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')", + )? + .execute(params![ + entity_type, + canonical_key, + display_label, + source_kind, + source_id, + trust_level, + first_seen_at, + last_seen_at + ])?; + conn.prepare_cached( + "SELECT id FROM _graph_entities_staging + WHERE entity_type = ?1 AND canonical_key = ?2", + )? + .query_row(params![entity_type, canonical_key], |row| row.get(0)) + .map_err(Into::into) +} + +/// Test-only counter of actual `ensure_entity` upserts (memo hits are not +/// counted), so tests can assert the chunk-scoped `EntityMemo` collapsed +/// repeated `(entity_type, canonical_key)` pairs to a single upsert instead +/// of one per occurrence. +#[cfg(test)] +pub(crate) static ENSURE_ENTITY_CALLS: std::sync::atomic::AtomicU64 = + std::sync::atomic::AtomicU64::new(0); + +/// Memoized wrapper over [`ensure_entity`]: within one chunk transaction, a +/// repeated `(entity_type, canonical_key)` pair resolves from `memo` instead +/// of re-running the upsert. See [`EntityMemo`] for the scope and the +/// `last_seen_at` tradeoff this implies. +#[allow(clippy::too_many_arguments)] +pub(crate) fn ensure_entity_memoized( + conn: &rusqlite::Connection, + memo: &mut EntityMemo, + entity_type: &'static str, + canonical_key: &str, + display_label: &str, + source_kind: &str, + source_id: &str, + trust_level: &str, + first_seen_at: Option<&str>, + last_seen_at: Option<&str>, +) -> Result { + let memo_key = (entity_type, canonical_key.to_string()); + if let Some(id) = memo.get(&memo_key) { + return Ok(*id); + } + #[cfg(test)] + ENSURE_ENTITY_CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + let id = ensure_entity( + conn, + entity_type, + canonical_key, + display_label, + source_kind, + source_id, + trust_level, + first_seen_at, + last_seen_at, + )?; + memo.insert(memo_key, id); + Ok(id) +} + +#[allow(clippy::too_many_arguments)] +fn insert_alias( + conn: &rusqlite::Connection, + entity_id: i64, + alias_type: &str, + alias_key: &str, + alias_value: &str, + source_kind: &str, + trust_level: &str, + first_seen_at: Option<&str>, + last_seen_at: Option<&str>, +) -> Result<()> { + conn.prepare_cached( + "INSERT INTO _graph_aliases_staging + (entity_id, alias_type, alias_key, alias_value, source_kind, + trust_level, first_seen_at, last_seen_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8) + ON CONFLICT(entity_id, alias_type, alias_key, source_kind) DO UPDATE SET + last_seen_at = CASE + WHEN excluded.last_seen_at > _graph_aliases_staging.last_seen_at THEN excluded.last_seen_at + ELSE _graph_aliases_staging.last_seen_at END, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')", + )? + .execute(params![ + entity_id, + alias_type, + alias_key, + alias_value, + source_kind, + trust_level, + first_seen_at, + last_seen_at + ])?; + Ok(()) +} + +pub(crate) struct EvidenceInput<'a> { + pub(crate) evidence_key: String, + pub(crate) source_kind: &'a str, + pub(crate) source_id: &'a str, + pub(crate) source_log_id: Option, + pub(crate) source_heartbeat_id: Option, + pub(crate) source_signature_hash: Option<&'a str>, + pub(crate) observed_at: &'a str, + pub(crate) reason_text: Option<&'a str>, + pub(crate) confidence_delta: f64, + pub(crate) trust_level: &'a str, + pub(crate) safe_excerpt: Option<&'a str>, + pub(crate) metadata_path: Option<&'a str>, +} + +#[allow(clippy::too_many_arguments)] +pub(crate) fn ensure_relationship_with_evidence( + conn: &rusqlite::Connection, + src_entity_id: i64, + dst_entity_id: i64, + relationship_type: &str, + reason_code: &str, + trust_level: &str, + confidence: f64, + evidence: EvidenceInput<'_>, +) -> Result<()> { + let relationship_key = format!("{src_entity_id}:{relationship_type}:{dst_entity_id}"); + conn.prepare_cached( + "INSERT INTO _graph_relationships_staging + (relationship_key, src_entity_id, dst_entity_id, relationship_type, + reason_code, trust_level, confidence, evidence_count, + first_seen_at, last_seen_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 0, ?8, ?8) + ON CONFLICT(relationship_key) DO UPDATE SET + confidence = MAX(_graph_relationships_staging.confidence, excluded.confidence), + first_seen_at = CASE + WHEN excluded.first_seen_at < _graph_relationships_staging.first_seen_at THEN excluded.first_seen_at + ELSE _graph_relationships_staging.first_seen_at END, + last_seen_at = CASE + WHEN excluded.last_seen_at > _graph_relationships_staging.last_seen_at THEN excluded.last_seen_at + ELSE _graph_relationships_staging.last_seen_at END, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')", + )? + .execute(params![ + relationship_key, + src_entity_id, + dst_entity_id, + relationship_type, + reason_code, + trust_level, + confidence, + evidence.observed_at + ])?; + let relationship_id: i64 = conn + .prepare_cached("SELECT id FROM _graph_relationships_staging WHERE relationship_key = ?1")? + .query_row([relationship_key], |row| row.get(0))?; + conn.prepare_cached( + "INSERT INTO _graph_evidence_staging + (relationship_id, evidence_key, source_kind, source_id, source_log_id, + source_heartbeat_id, source_signature_hash, observed_at, reason_code, + reason_text, confidence_delta, trust_level, safe_excerpt, metadata_path, + evidence_count) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, 1) + ON CONFLICT(relationship_id, evidence_key) DO UPDATE SET + evidence_count = _graph_evidence_staging.evidence_count + 1, + observed_at = CASE + WHEN excluded.observed_at > _graph_evidence_staging.observed_at THEN excluded.observed_at + ELSE _graph_evidence_staging.observed_at END", + )? + .execute(params![ + relationship_id, + evidence.evidence_key, + evidence.source_kind, + evidence.source_id, + evidence.source_log_id, + evidence.source_heartbeat_id, + evidence.source_signature_hash, + evidence.observed_at, + reason_code, + evidence.reason_text, + evidence.confidence_delta, + evidence.trust_level, + evidence.safe_excerpt.map(truncate_safe_excerpt), + evidence.metadata_path + ])?; + conn.prepare_cached( + "UPDATE _graph_relationships_staging + SET evidence_count = ( + SELECT COALESCE(SUM(evidence_count), 0) + FROM _graph_evidence_staging + WHERE relationship_id = ?1 + ) + WHERE id = ?1", + )? + .execute([relationship_id])?; + Ok(()) +} + +fn swap_graph_projection( + conn: &mut rusqlite::Connection, + source_row_count: i64, + source_watermark: &str, + runtime_ms: i64, + chunk_count: i64, +) -> Result { + let entity_count = table_count(conn, "_graph_entities_staging")?; + let relationship_count = table_count(conn, "_graph_relationships_staging")?; + let evidence_count = table_count(conn, "_graph_evidence_staging")?; + + let _guard = write_lock(); + let tx = conn.transaction()?; + tx.execute("DELETE FROM graph_relationship_evidence", [])?; + tx.execute("DELETE FROM graph_relationships", [])?; + tx.execute("DELETE FROM graph_entity_aliases", [])?; + tx.execute("DELETE FROM graph_entities", [])?; + tx.execute( + "INSERT INTO graph_entities + (id, entity_type, canonical_key, display_label, source_kind, source_id, + trust_level, first_seen_at, last_seen_at, created_at, updated_at) + SELECT id, entity_type, canonical_key, display_label, source_kind, source_id, + trust_level, first_seen_at, last_seen_at, created_at, updated_at + FROM _graph_entities_staging", + [], + )?; + tx.execute( + "INSERT INTO graph_entity_aliases + (id, entity_id, alias_type, alias_key, alias_value, source_kind, + trust_level, first_seen_at, last_seen_at, created_at, updated_at) + SELECT id, entity_id, alias_type, alias_key, alias_value, source_kind, + trust_level, first_seen_at, last_seen_at, created_at, updated_at + FROM _graph_aliases_staging", + [], + )?; + tx.execute( + "INSERT INTO graph_relationships + (id, relationship_key, src_entity_id, dst_entity_id, relationship_type, + reason_code, trust_level, confidence, evidence_count, first_seen_at, + last_seen_at, created_at, updated_at) + SELECT id, relationship_key, src_entity_id, dst_entity_id, relationship_type, + reason_code, trust_level, confidence, evidence_count, first_seen_at, + last_seen_at, created_at, updated_at + FROM _graph_relationships_staging", + [], + )?; + tx.execute( + "INSERT INTO graph_relationship_evidence + (id, relationship_id, evidence_key, source_kind, source_id, source_log_id, + source_heartbeat_id, source_signature_hash, observed_at, reason_code, + reason_text, confidence_delta, trust_level, safe_excerpt, metadata_path, + evidence_count, created_at) + SELECT id, relationship_id, evidence_key, source_kind, source_id, source_log_id, + source_heartbeat_id, source_signature_hash, observed_at, reason_code, + reason_text, confidence_delta, trust_level, safe_excerpt, metadata_path, + evidence_count, created_at + FROM _graph_evidence_staging", + [], + )?; + tx.execute( + "UPDATE graph_projection_meta + SET projection_status = 'ready', + last_completed_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), + source_watermark = ?1, + source_row_count = ?2, + entity_count = ?3, + relationship_count = ?4, + evidence_count = ?5, + is_degraded = 0, + last_error = NULL, + last_runtime_ms = ?6, + last_chunk_count = ?7, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + WHERE id = 1", + params![ + source_watermark, + source_row_count, + entity_count, + relationship_count, + evidence_count, + runtime_ms, + chunk_count + ], + )?; + tx.commit()?; + Ok(GraphRebuildStats { + source_row_count, + entity_count, + relationship_count, + evidence_count, + source_watermark: source_watermark.to_string(), + runtime_ms, + chunk_count, + }) +} + +fn graph_source_watermark(conn: &rusqlite::Connection) -> Result { + let max_log_id: i64 = + conn.query_row("SELECT COALESCE(MAX(id), 0) FROM logs", [], |r| r.get(0))?; + let max_heartbeat_id: i64 = conn.query_row( + "SELECT COALESCE(MAX(heartbeat_id), 0) FROM host_heartbeats_latest", + [], + |r| r.get(0), + )?; + let signature_count: i64 = + conn.query_row("SELECT COUNT(*) FROM error_signatures", [], |r| r.get(0))?; + Ok(format!( + "logs:{max_log_id};heartbeats:{max_heartbeat_id};signatures:{signature_count}" + )) +} + +fn table_count(conn: &rusqlite::Connection, table: &str) -> Result { + let sql = format!("SELECT COUNT(*) FROM {table}"); + conn.query_row(&sql, [], |row| row.get(0)) + .map_err(Into::into) +} + +/// Test-only counter of `parse_metadata` calls, so tests can assert +/// `extract_log_row` parses each row's `metadata_json` exactly once and +/// threads the result down to every `extract_*_row` function that needs it, +/// instead of each one independently re-parsing the same JSON string. +#[cfg(test)] +pub(crate) static PARSE_METADATA_CALLS: std::sync::atomic::AtomicU64 = + std::sync::atomic::AtomicU64::new(0); + +fn parse_metadata(input: Option<&str>) -> Option { + #[cfg(test)] + PARSE_METADATA_CALLS.fetch_add(1, std::sync::atomic::Ordering::Relaxed); + input.and_then(|raw| serde_json::from_str::(raw).ok()) +} + +fn metadata_text<'a>(meta: Option<&'a Value>, paths: &[&str]) -> Option<&'a str> { + let value = meta?; + for path in paths { + let mut current = value; + let mut found = true; + for segment in path.split('.') { + if let Some(next) = current.get(segment) { + current = next; + } else { + found = false; + break; + } + } + if found && let Some(text) = current.as_str().and_then(normalized_value) { + return Some(text); + } + } + None +} + +#[derive(Debug, Default)] +struct DockerSourceParts<'a> { + host: Option<&'a str>, + container: Option<&'a str>, +} + +fn parse_docker_source(source: &str) -> DockerSourceParts<'_> { + let Some(rest) = source + .strip_prefix("docker://") + .or_else(|| source.strip_prefix("docker-event://")) + else { + return DockerSourceParts::default(); + }; + let mut parts = rest.split('/'); + DockerSourceParts { + host: parts.next().and_then(normalized_value), + container: parts.next().and_then(normalized_value), + } +} + +fn normalized(value: &str) -> Option { + normalized_value(value).map(normalize_key) +} + +pub(crate) fn normalized_value(value: &str) -> Option<&str> { + let trimmed = value.trim(); + if trimmed.is_empty() { + None + } else { + Some(trimmed) + } +} + +fn normalize_key(value: &str) -> String { + value.trim().to_ascii_lowercase() +} + +pub(crate) fn evidence_bucket_key( + prefix: &str, + source_id: i64, + reason: &str, + timestamp: &str, +) -> String { + let bucket = timestamp.get(0..13).unwrap_or(timestamp); + format!("{prefix}:{reason}:{source_id}:{bucket}") +} + +fn truncate_safe_excerpt(value: &str) -> String { + value.chars().take(512).collect() +} + +fn redact_error(value: &str) -> String { + value + .chars() + .filter(|ch| !ch.is_control()) + .take(2048) + .collect() +} + +#[cfg(test)] +#[path = "graph_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/graph_findings.rs b/crates/shared/cortex/storage-sqlite/src/graph_findings.rs new file mode 100644 index 00000000..87f12bb5 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/graph_findings.rs @@ -0,0 +1,186 @@ +#[cfg(test)] +#[path = "graph_findings_tests.rs"] +mod tests; + +use anyhow::Result; +use rusqlite::params; + +use crate::DbPool; +use crate::graph; + +#[derive(Debug, Clone, PartialEq)] +pub struct PublicRouteFindingRow { + pub domain_key: String, + pub domain_label: String, + pub proxy_key: String, + pub proxy_label: String, + pub service_key: Option, + pub service_label: Option, + pub exposes_confidence: f64, + pub routes_confidence: Option, + pub exposes_evidence_id: Option, + pub exposes_excerpt: Option, + pub routes_evidence_id: Option, + pub routes_excerpt: Option, +} + +#[derive(Debug, Clone, PartialEq)] +pub struct MountRelationshipFindingRow { + pub service_key: String, + pub service_label: String, + pub storage_key: String, + pub storage_label: String, + pub confidence: f64, + pub evidence_id: Option, + pub safe_excerpt: Option, +} + +pub fn list_public_route_findings(pool: &DbPool, limit: u32) -> Result> { + let conn = pool.get()?; + let mut stmt = conn.prepare( + "SELECT + domain.canonical_key, + domain.display_label, + proxy.canonical_key, + proxy.display_label, + service.canonical_key, + service.display_label, + exposes.confidence, + routes.confidence, + exposes_ev.id, + exposes_ev.safe_excerpt, + routes_ev.id, + routes_ev.safe_excerpt + FROM graph_relationships exposes INDEXED BY idx_graph_relationships_type_seen + JOIN graph_entities proxy ON proxy.id = exposes.src_entity_id + JOIN graph_entities domain ON domain.id = exposes.dst_entity_id + LEFT JOIN graph_relationship_evidence exposes_ev + ON exposes_ev.id = ( + SELECT id + FROM graph_relationship_evidence + WHERE relationship_id = exposes.id + ORDER BY observed_at DESC, id DESC + LIMIT 1 + ) + LEFT JOIN graph_relationships routes + ON routes.src_entity_id = proxy.id + AND routes.relationship_type = ?2 + LEFT JOIN graph_entities service ON service.id = routes.dst_entity_id + LEFT JOIN graph_relationship_evidence routes_ev + ON routes_ev.id = ( + SELECT id + FROM graph_relationship_evidence + WHERE relationship_id = routes.id + ORDER BY observed_at DESC, id DESC + LIMIT 1 + ) + WHERE exposes.relationship_type = ?1 + AND proxy.entity_type = ?3 + AND domain.entity_type = ?4 + ORDER BY exposes.confidence DESC, exposes.last_seen_at DESC, exposes.id DESC + LIMIT ?5", + )?; + let rows = stmt + .query_map( + params![ + graph::REL_EXPOSES_DOMAIN, + graph::REL_ROUTES_TO, + graph::ENTITY_TYPE_REVERSE_PROXY, + graph::ENTITY_TYPE_DOMAIN, + i64::from(limit), + ], + |row| { + Ok(PublicRouteFindingRow { + domain_key: row.get(0)?, + domain_label: row.get(1)?, + proxy_key: row.get(2)?, + proxy_label: row.get(3)?, + service_key: row.get(4)?, + service_label: row.get(5)?, + exposes_confidence: row.get(6)?, + routes_confidence: row.get(7)?, + exposes_evidence_id: row.get(8)?, + exposes_excerpt: row.get(9)?, + routes_evidence_id: row.get(10)?, + routes_excerpt: row.get(11)?, + }) + }, + )? + .collect::>>()?; + Ok(rows) +} + +pub fn list_mount_relationship_findings( + pool: &DbPool, + limit: u32, +) -> Result> { + let conn = pool.get()?; + let mut stmt = conn.prepare( + "SELECT + service.canonical_key, + service.display_label, + storage.canonical_key, + storage.display_label, + mounts.confidence, + evidence.id, + evidence.safe_excerpt + FROM graph_relationships mounts INDEXED BY idx_graph_relationships_type_seen + JOIN graph_entities service ON service.id = mounts.src_entity_id + JOIN graph_entities storage ON storage.id = mounts.dst_entity_id + LEFT JOIN graph_relationship_evidence evidence + ON evidence.id = ( + SELECT id + FROM graph_relationship_evidence + WHERE relationship_id = mounts.id + ORDER BY observed_at DESC, id DESC + LIMIT 1 + ) + WHERE mounts.relationship_type = ?1 + AND service.entity_type = ?2 + AND storage.entity_type = ?3 + ORDER BY mounts.confidence DESC, mounts.last_seen_at DESC, mounts.id DESC + LIMIT ?4", + )?; + let rows = stmt + .query_map( + params![ + graph::REL_MOUNTS, + graph::ENTITY_TYPE_SERVICE_INSTANCE, + graph::ENTITY_TYPE_STORAGE, + i64::from(limit), + ], + |row| { + Ok(MountRelationshipFindingRow { + service_key: row.get(0)?, + service_label: row.get(1)?, + storage_key: row.get(2)?, + storage_label: row.get(3)?, + confidence: row.get(4)?, + evidence_id: row.get(5)?, + safe_excerpt: row.get(6)?, + }) + }, + )? + .collect::>>()?; + Ok(rows) +} + +#[cfg(test)] +pub(crate) fn relationship_type_query_plan( + pool: &DbPool, + relationship_type: &str, +) -> Result> { + let conn = pool.get()?; + let mut stmt = conn.prepare( + "EXPLAIN QUERY PLAN + SELECT id + FROM graph_relationships INDEXED BY idx_graph_relationships_type_seen + WHERE relationship_type = ?1 + ORDER BY last_seen_at DESC + LIMIT 10", + )?; + let rows = stmt + .query_map([relationship_type], |row| row.get::<_, String>(3))? + .collect::>>()?; + Ok(rows) +} diff --git a/crates/shared/cortex/storage-sqlite/src/graph_findings_tests.rs b/crates/shared/cortex/storage-sqlite/src/graph_findings_tests.rs new file mode 100644 index 00000000..466fb060 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/graph_findings_tests.rs @@ -0,0 +1,151 @@ +use super::*; +use crate::config::StorageConfig; +use cortex_inventory::{ + HomelabInventory, InventoryNode, InventoryService, MountRef, Provenance, ReverseProxyRoute, + TrustLevel, +}; + +fn test_pool() -> (DbPool, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let storage = StorageConfig::for_test(dir.path().join("graph-findings.db")); + let pool = crate::init_pool(&storage).unwrap(); + (pool, dir) +} + +fn provenance(source: &str) -> Provenance { + Provenance::new(source, "app_inventory", "2026-01-01T00:00:00Z".to_string()) +} + +fn seed_inventory(pool: &DbPool, services: usize) { + let mut inventory = HomelabInventory::empty( + "graph-findings-test".to_string(), + "2026-01-01T00:00:00Z".to_string(), + ); + inventory.nodes.push(InventoryNode { + id: "node:edgehost".to_string(), + hostname: "edgehost".to_string(), + trust_level: TrustLevel::Observed, + provenance: provenance("ssh:edgehost"), + roles: Vec::new(), + ips: Vec::new(), + os: None, + cpu: None, + memory: None, + listeners: Vec::new(), + storage: Vec::new(), + extras: Default::default(), + }); + for idx in 0..services { + let name = format!("svc-{idx}"); + inventory.services.push(InventoryService { + id: format!("container:edgehost:{name}"), + name: name.clone(), + kind: "container".to_string(), + trust_level: TrustLevel::Observed, + provenance: provenance("docker:edgehost"), + host: Some("edgehost".to_string()), + image: None, + status: Some("running".to_string()), + domains: Vec::new(), + ports: Vec::new(), + mounts: vec![MountRef { + source: Some("/var/run/docker.sock".to_string()), + target: "/var/run/docker.sock".to_string(), + read_only: false, + }], + env_keys: Vec::new(), + labels: Default::default(), + details: Default::default(), + }); + } + inventory.reverse_proxies.push(ReverseProxyRoute { + id: "proxy:one.example.test".to_string(), + server_names: vec!["one.example.test".to_string()], + upstreams: vec!["svc-0:80".to_string()], + provenance: provenance("swag:edgehost:/redacted.conf"), + }); + let _guard = crate::graph::GRAPH_TEST_LOCK.lock(); + crate::graph::refresh_graph_projection(pool).unwrap(); + crate::graph_inventory::project_inventory(pool, &inventory).unwrap(); +} + +fn seed_inventory_without_route_target(pool: &DbPool) { + let mut inventory = HomelabInventory::empty( + "graph-findings-no-target-test".to_string(), + "2026-01-01T00:00:00Z".to_string(), + ); + inventory.nodes.push(InventoryNode { + id: "node:edgehost".to_string(), + hostname: "edgehost".to_string(), + trust_level: TrustLevel::Observed, + provenance: provenance("ssh:edgehost"), + roles: Vec::new(), + ips: Vec::new(), + os: None, + cpu: None, + memory: None, + listeners: Vec::new(), + storage: Vec::new(), + extras: Default::default(), + }); + inventory.reverse_proxies.push(ReverseProxyRoute { + id: "proxy:orphan.example.test".to_string(), + server_names: vec!["orphan.example.test".to_string()], + upstreams: vec!["missing-service:80".to_string()], + provenance: provenance("swag:edgehost:/redacted.conf"), + }); + let _guard = crate::graph::GRAPH_TEST_LOCK.lock(); + crate::graph::refresh_graph_projection(pool).unwrap(); + crate::graph_inventory::project_inventory(pool, &inventory).unwrap(); +} + +#[test] +fn public_route_findings_return_route_target_and_evidence() { + let (pool, _dir) = test_pool(); + seed_inventory(&pool, 2); + + let rows = list_public_route_findings(&pool, 10).unwrap(); + + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].domain_key, "one.example.test"); + assert_eq!(rows[0].proxy_key, "proxy:one.example.test"); + // Canonical service-instance key (`host/service`), never `host:service`. + assert_eq!(rows[0].service_key.as_deref(), Some("edgehost/svc-0")); + assert!(rows[0].exposes_evidence_id.is_some()); + assert!(rows[0].routes_evidence_id.is_some()); +} + +#[test] +fn public_route_findings_return_domain_without_route_target() { + let (pool, _dir) = test_pool(); + seed_inventory_without_route_target(&pool); + + let rows = list_public_route_findings(&pool, 10).unwrap(); + + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].domain_key, "orphan.example.test"); + assert_eq!(rows[0].proxy_key, "proxy:orphan.example.test"); + assert!(rows[0].service_key.is_none()); + assert!(rows[0].routes_evidence_id.is_none()); + assert!(rows[0].exposes_evidence_id.is_some()); +} + +#[test] +fn mount_findings_are_bounded_and_use_relationship_type_index() { + let (pool, _dir) = test_pool(); + seed_inventory(&pool, 25); + + let rows = list_mount_relationship_findings(&pool, 5).unwrap(); + let plan = relationship_type_query_plan(&pool, crate::graph::REL_MOUNTS).unwrap(); + + assert_eq!(rows.len(), 5); + assert!( + plan.iter() + .any(|row| row.contains("idx_graph_relationships_type_seen")), + "expected type-specific graph index in query plan: {plan:?}" + ); + assert!( + !plan.iter().any(|row| row == "SCAN graph_relationships"), + "findings query must not broad-scan graph relationships: {plan:?}" + ); +} diff --git a/crates/shared/cortex/storage-sqlite/src/graph_inventory.rs b/crates/shared/cortex/storage-sqlite/src/graph_inventory.rs new file mode 100644 index 00000000..df19075a --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/graph_inventory.rs @@ -0,0 +1,767 @@ +mod sql; +#[cfg(test)] +#[path = "graph_inventory_tests.rs"] +mod tests; + +use std::collections::{BTreeMap, btree_map::Entry}; + +use anyhow::{Context, Result}; + +use crate::graph; +use crate::graph_resolver_projection::trust_to_graph; +use crate::{DbPool, entity_resolution, write_lock}; +use cortex_inventory::HomelabInventory; + +use self::sql::{ + add_alias, add_relationship, canonical, canonical_or_raw, graph_counts, + prune_previous_inventory_projection, safe_inventory_source_id, scoped_inventory_key, trust, + update_projection_meta, upsert_entity, +}; + +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct InventoryGraphStats { + pub source_row_count: i64, + pub entity_count: i64, + pub relationship_count: i64, + pub evidence_count: i64, +} + +pub fn project_inventory( + pool: &DbPool, + inventory: &HomelabInventory, +) -> Result { + let plan = build_projection_plan(inventory); + warn_skipped_services(&plan); + apply_projection_plan(pool, &plan, || {}) +} + +/// One warning per inventory projection listing the services whose identity +/// failed canonicalization and were therefore left out of the graph. +fn warn_skipped_services(plan: &InventoryProjectionPlan) { + if !plan.skipped_service_ids.is_empty() { + tracing::warn!( + skipped = plan.skipped_service_ids.len(), + service_ids = ?plan.skipped_service_ids, + "inventory services skipped from graph projection: service name failed canonicalization" + ); + } +} + +#[cfg(test)] +fn project_inventory_with_apply_hook( + pool: &DbPool, + inventory: &HomelabInventory, + before_apply: impl FnOnce(), +) -> Result { + let plan = build_projection_plan(inventory); + warn_skipped_services(&plan); + apply_projection_plan(pool, &plan, before_apply) +} + +fn apply_projection_plan( + pool: &DbPool, + plan: &InventoryProjectionPlan, + before_apply: impl FnOnce(), +) -> Result { + before_apply(); + let mut conn = pool.get().context("borrow sqlite connection")?; + let _guard = write_lock(); + let tx = conn + .transaction() + .context("start inventory graph transaction")?; + + prune_previous_inventory_projection(&tx)?; + + // Heartbeats are the authoritative proof that a fleet host exists. Keep + // them resolvable even when an SSH/Docker inventory collector fails and + // therefore contributes no InventoryNode for that refresh. + project_heartbeat_hosts(&tx)?; + + let mut entities = BTreeMap::new(); + for entity in &plan.entities { + let entity_ref = upsert_entity( + &tx, + entity.key.kind, + &entity.key.key, + &entity.display_label, + entity.source_kind, + &entity.source_id, + entity.trust_level, + &entity.observed_at, + )?; + entities.insert(entity.key.clone(), entity_ref); + } + + for alias in &plan.aliases { + let entity = entities + .get(&alias.entity) + .with_context(|| format!("missing planned entity {}", alias.entity.key))?; + add_alias( + &tx, + entity.id, + alias.alias_type, + &alias.alias_key, + &alias.alias_value, + alias.source_kind, + alias.trust_level, + &alias.observed_at, + )?; + } + + for relationship in &plan.relationships { + let src = entities + .get(&relationship.src) + .with_context(|| format!("missing planned source entity {}", relationship.src.key))?; + let dst = entities.get(&relationship.dst).with_context(|| { + format!( + "missing planned destination entity {}", + relationship.dst.key + ) + })?; + add_relationship( + &tx, + src, + dst, + relationship.relationship_type, + relationship.reason_code, + relationship.source_kind, + &relationship.source_id, + &relationship.observed_at, + relationship.trust_level, + relationship.confidence, + &relationship.safe_excerpt, + )?; + } + + let stats = graph_counts(&tx)?; + update_projection_meta(&tx, &stats)?; + tx.commit().context("commit inventory graph projection")?; + Ok(stats) +} + +fn project_heartbeat_hosts(conn: &rusqlite::Connection) -> Result<()> { + let rows = { + let mut stmt = conn.prepare( + "SELECT heartbeat_id, host_id, hostname, sampled_at + FROM host_heartbeats_latest + ORDER BY hostname ASC", + )?; + stmt.query_map([], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, String>(1)?, + row.get::<_, String>(2)?, + row.get::<_, String>(3)?, + )) + })? + .collect::>>()? + }; + for (heartbeat_id, host_id, hostname, sampled_at) in rows { + let Some(key) = canonical(&hostname) else { + continue; + }; + let entity = upsert_entity( + conn, + graph::ENTITY_TYPE_HOST, + &key, + &hostname, + graph::SOURCE_KIND_HEARTBEAT, + &heartbeat_id.to_string(), + graph::TRUST_VERIFIED, + &sampled_at, + )?; + add_alias( + conn, + entity.id, + "hostname", + &key, + &hostname, + graph::SOURCE_KIND_HEARTBEAT, + graph::TRUST_VERIFIED, + &sampled_at, + )?; + add_alias( + conn, + entity.id, + "heartbeat_host_id", + &canonical_or_raw(&host_id), + &host_id, + graph::SOURCE_KIND_HEARTBEAT, + graph::TRUST_VERIFIED, + &sampled_at, + )?; + } + Ok(()) +} + +fn build_projection_plan(inventory: &HomelabInventory) -> InventoryProjectionPlan { + let mut plan = InventoryProjectionPlan::default(); + let mut hosts = BTreeMap::new(); + for node in &inventory.nodes { + let Some(key) = canonical(&node.hostname) else { + continue; + }; + let entity = plan.entity( + graph::ENTITY_TYPE_HOST, + &key, + &node.hostname, + graph::SOURCE_KIND_SOURCE_INVENTORY, + &node.id, + trust(&node.trust_level), + &node.provenance.collected_at, + ); + plan.alias( + &entity, + "hostname", + &key, + &node.hostname, + graph::SOURCE_KIND_SOURCE_INVENTORY, + trust(&node.trust_level), + &node.provenance.collected_at, + ); + for ip in &node.ips { + if let Some(alias_key) = canonical(ip) { + plan.alias( + &entity, + "ip", + &alias_key, + ip, + graph::SOURCE_KIND_SOURCE_INVENTORY, + trust(&node.trust_level), + &node.provenance.collected_at, + ); + } + } + hosts.insert(key, entity); + } + + let mut services = BTreeMap::new(); + let mut unique_service_aliases = BTreeMap::new(); + let mut logical_services: BTreeMap = BTreeMap::new(); + for service in &inventory.services { + // Canonical service identity (entity_resolution_v2): inventory + // services flow through the shared resolver adapter, projecting as + // `logical_service` (`plex`) plus a host-scoped `service_instance` + // (`nashost/plex`). Legacy `service` entities (`host:name`) are never + // emitted. + let observations = entity_resolution::observations_from_inventory_service(service); + let decisions = entity_resolution::resolve_observations(&observations); + let logical_decision = decisions + .iter() + .find(|d| d.entity_type == graph::ENTITY_TYPE_LOGICAL_SERVICE); + let instance_decision = decisions + .iter() + .find(|d| d.entity_type == graph::ENTITY_TYPE_SERVICE_INSTANCE); + let Some(logical_decision) = logical_decision else { + plan.skipped_service_ids.push(service.id.clone()); + continue; + }; + let logical_key = logical_decision.canonical_key.clone(); + let logical_entity = match logical_services.entry(logical_key.clone()) { + Entry::Occupied(entry) => entry.get().clone(), + Entry::Vacant(entry) => { + let entity = plan.entity( + graph::ENTITY_TYPE_LOGICAL_SERVICE, + &logical_key, + &service.name, + graph::SOURCE_KIND_APP_INVENTORY, + &service.id, + trust_to_graph(logical_decision.trust), + &service.provenance.collected_at, + ); + entry.insert(entity.clone()); + entity + } + }; + + let Some(instance_decision) = instance_decision else { + // No host context: the logical service exists, but there is no + // deployment topology to assert. Ambiguity stays visible instead + // of being guessed into an `unknown/` instance. + insert_unique_alias( + &mut unique_service_aliases, + canonical_or_raw(&service.name), + logical_entity.clone(), + ); + continue; + }; + let instance_key = instance_decision.canonical_key.clone(); + let service_entity = plan.entity( + graph::ENTITY_TYPE_SERVICE_INSTANCE, + &instance_key, + &instance_key, + graph::SOURCE_KIND_APP_INVENTORY, + &service.id, + trust_to_graph(instance_decision.trust), + &service.provenance.collected_at, + ); + plan.relationship( + &service_entity, + &logical_entity, + graph::REL_INSTANCE_OF, + graph::REASON_RESOLVER_INSTANCE_OF, + graph::SOURCE_KIND_APP_INVENTORY, + &service.id, + &service.provenance.collected_at, + trust_to_graph(instance_decision.trust), + 0.95, + &format!("{} is an instance of {}", instance_key, logical_key), + ); + services.insert(instance_key.clone(), service_entity.clone()); + insert_unique_alias( + &mut unique_service_aliases, + canonical_or_raw(&service.name), + service_entity.clone(), + ); + + for domain in &service.domains { + if let Some(alias_key) = canonical(domain) { + plan.alias( + &service_entity, + "domain", + &alias_key, + domain, + graph::SOURCE_KIND_APP_INVENTORY, + trust(&service.trust_level), + &service.provenance.collected_at, + ); + } + } + + if let Some(host) = service + .host + .as_ref() + .and_then(|h| hosts.get(&canonical_or_raw(h))) + { + plan.relationship( + &service_entity, + host, + graph::REL_RUNS_ON, + graph::REASON_INVENTORY_SERVICE, + graph::SOURCE_KIND_APP_INVENTORY, + &service.id, + &service.provenance.collected_at, + graph::TRUST_INFERRED, + 0.85, + &format!("{} observed on {}", service.name, host.key), + ); + } + + for mount in &service.mounts { + let storage_key = canonical_or_raw(&format!( + "{}:{}", + service.host.as_deref().unwrap_or("unknown"), + mount.target + )); + let storage = plan.entity( + graph::ENTITY_TYPE_STORAGE, + &storage_key, + &mount.target, + graph::SOURCE_KIND_APP_INVENTORY, + &storage_key, + graph::TRUST_INFERRED, + &service.provenance.collected_at, + ); + plan.relationship( + &service_entity, + &storage, + graph::REL_MOUNTS, + graph::REASON_STORAGE_PROBE, + graph::SOURCE_KIND_APP_INVENTORY, + &service.id, + &service.provenance.collected_at, + graph::TRUST_INFERRED, + 0.65, + &format!("{} mounts {}", service.name, mount.target), + ); + } + } + services.extend( + unique_service_aliases + .into_iter() + .filter_map(|(key, service)| service.map(|service| (key, service))), + ); + + let mut artifacts = BTreeMap::new(); + for artifact in &inventory.artifact_refs { + let display = format!("{} artifact {}", artifact.kind, artifact.id); + let entity = plan.entity( + graph::ENTITY_TYPE_CONFIG_ARTIFACT, + &canonical_or_raw(&artifact.id), + &display, + graph::SOURCE_KIND_APP_INVENTORY, + &artifact.id, + graph::TRUST_VERIFIED, + &inventory.generated_at, + ); + artifacts.insert(artifact.id.clone(), entity.clone()); + if let Some(path) = &artifact.source_path { + artifacts.insert(path.clone(), entity); + } + } + + for project in &inventory.compose_projects { + let project_key = scoped_inventory_key(&project.provenance.source, &project.name); + let project_source = safe_inventory_source_id(&project.provenance.source); + let project_entity = plan.entity( + graph::ENTITY_TYPE_COMPOSE_PROJECT, + &project_key, + &project.name, + graph::SOURCE_KIND_APP_INVENTORY, + &project_source, + graph::TRUST_VERIFIED, + &project.provenance.collected_at, + ); + for service_name in &project.services { + if let Some(service_entity) = + match_service_name_key(service_name, &project.provenance.source, &services) + { + plan.relationship( + &project_entity, + service_entity, + graph::REL_DEFINES_SERVICE, + graph::REASON_COMPOSE_CONFIG, + graph::SOURCE_KIND_APP_INVENTORY, + &project_source, + &project.provenance.collected_at, + graph::TRUST_VERIFIED, + 0.90, + &format!("compose project {} defines {}", project.name, service_name), + ); + } + } + for compose_file in &project.compose_files { + if let Some(artifact) = artifacts.get(compose_file) { + let artifact_id = artifact.key.clone(); + plan.relationship( + &project_entity, + artifact, + graph::REL_HAS_ARTIFACT, + graph::REASON_CONFIG_ARTIFACT, + graph::SOURCE_KIND_APP_INVENTORY, + &project_source, + &project.provenance.collected_at, + graph::TRUST_VERIFIED, + 0.95, + &format!("compose artifact {}", artifact_id), + ); + } + } + } + + for route in &inventory.reverse_proxies { + let proxy_key = canonical_or_raw(&route.id); + let proxy = plan.entity( + graph::ENTITY_TYPE_REVERSE_PROXY, + &proxy_key, + route.server_names.first().unwrap_or(&route.id), + graph::SOURCE_KIND_APP_INVENTORY, + &route.id, + graph::TRUST_VERIFIED, + &route.provenance.collected_at, + ); + for domain in &route.server_names { + let domain_key = canonical_or_raw(domain); + let domain_entity = plan.entity( + graph::ENTITY_TYPE_DOMAIN, + &domain_key, + domain, + graph::SOURCE_KIND_APP_INVENTORY, + &route.id, + graph::TRUST_VERIFIED, + &route.provenance.collected_at, + ); + plan.relationship( + &proxy, + &domain_entity, + graph::REL_EXPOSES_DOMAIN, + graph::REASON_REVERSE_PROXY_CONFIG, + graph::SOURCE_KIND_APP_INVENTORY, + &route.id, + &route.provenance.collected_at, + graph::TRUST_VERIFIED, + 0.95, + &format!("proxy exposes {}", domain), + ); + } + for upstream in &route.upstreams { + if let Some(service) = match_upstream_key(upstream, &route.provenance.source, &services) + { + plan.relationship( + &proxy, + service, + graph::REL_ROUTES_TO, + graph::REASON_REVERSE_PROXY_CONFIG, + graph::SOURCE_KIND_APP_INVENTORY, + &route.id, + &route.provenance.collected_at, + graph::TRUST_VERIFIED, + 0.85, + &format!("proxy routes to {}", upstream), + ); + } + } + } + + for network in &inventory.networks { + let network_source = safe_inventory_source_id(&network.provenance.source); + let network_entity = plan.entity( + graph::ENTITY_TYPE_NETWORK, + &scoped_inventory_key(&network.provenance.source, &network.name), + &network.name, + graph::SOURCE_KIND_APP_INVENTORY, + &network_source, + graph::TRUST_VERIFIED, + &network.provenance.collected_at, + ); + for member in &network.members { + if let Some(service) = + match_service_name_key(member, &network.provenance.source, &services) + { + plan.relationship( + service, + &network_entity, + graph::REL_ATTACHED_TO, + graph::REASON_DOCKER_NETWORK, + graph::SOURCE_KIND_APP_INVENTORY, + &network_source, + &network.provenance.collected_at, + graph::TRUST_VERIFIED, + 0.80, + &format!("{} attached to network {}", member, network.name), + ); + } + } + } + + for storage in &inventory.storage { + let entity = plan.entity( + graph::ENTITY_TYPE_STORAGE, + &canonical_or_raw(&storage.id), + &storage.mount, + graph::SOURCE_KIND_SOURCE_INVENTORY, + &storage.id, + graph::TRUST_VERIFIED, + &storage.provenance.collected_at, + ); + if let Some(host) = storage + .id + .split(':') + .nth(1) + .and_then(|host| hosts.get(&canonical_or_raw(host))) + { + plan.relationship( + host, + &entity, + graph::REL_BACKED_BY, + graph::REASON_STORAGE_PROBE, + graph::SOURCE_KIND_SOURCE_INVENTORY, + &storage.id, + &storage.provenance.collected_at, + graph::TRUST_VERIFIED, + 0.75, + &format!("{} storage mounted at {}", host.key, storage.mount), + ); + } + } + + plan +} + +pub fn mark_inventory_projection_failed(pool: &DbPool, error: &str) -> Result<()> { + let conn = pool.get().context("borrow sqlite connection")?; + let _guard = write_lock(); + sql::mark_projection_degraded(&conn, error) +} + +fn insert_unique_alias( + aliases: &mut BTreeMap>, + key: String, + service: PlannedEntityKey, +) { + match aliases.entry(key) { + Entry::Vacant(entry) => { + entry.insert(Some(service)); + } + Entry::Occupied(mut entry) => { + if entry + .get() + .as_ref() + .is_some_and(|existing| existing != &service) + { + entry.insert(None); + } + } + } +} + +#[derive(Debug, Clone, PartialEq, Eq, PartialOrd, Ord)] +struct PlannedEntityKey { + kind: &'static str, + key: String, +} + +#[derive(Debug, Clone)] +struct EntityPlan { + key: PlannedEntityKey, + display_label: String, + source_kind: &'static str, + source_id: String, + trust_level: &'static str, + observed_at: String, +} + +#[derive(Debug, Clone)] +struct AliasPlan { + entity: PlannedEntityKey, + alias_type: &'static str, + alias_key: String, + alias_value: String, + source_kind: &'static str, + trust_level: &'static str, + observed_at: String, +} + +#[derive(Debug, Clone)] +struct RelationshipPlan { + src: PlannedEntityKey, + dst: PlannedEntityKey, + relationship_type: &'static str, + reason_code: &'static str, + source_kind: &'static str, + source_id: String, + observed_at: String, + trust_level: &'static str, + confidence: f64, + safe_excerpt: String, +} + +#[derive(Debug, Default)] +struct InventoryProjectionPlan { + entities: Vec, + aliases: Vec, + relationships: Vec, + /// Inventory service ids skipped because their names failed + /// canonicalization (no logical-service decision). Surfaced by the + /// caller so silently absent services are diagnosable. + skipped_service_ids: Vec, +} + +impl InventoryProjectionPlan { + #[allow(clippy::too_many_arguments)] + fn entity( + &mut self, + kind: &'static str, + key: &str, + display_label: &str, + source_kind: &'static str, + source_id: &str, + trust_level: &'static str, + observed_at: &str, + ) -> PlannedEntityKey { + let key = PlannedEntityKey { + kind, + key: key.to_string(), + }; + self.entities.push(EntityPlan { + key: key.clone(), + display_label: display_label.to_string(), + source_kind, + source_id: source_id.to_string(), + trust_level, + observed_at: observed_at.to_string(), + }); + key + } + + #[allow(clippy::too_many_arguments)] + fn alias( + &mut self, + entity: &PlannedEntityKey, + alias_type: &'static str, + alias_key: &str, + alias_value: &str, + source_kind: &'static str, + trust_level: &'static str, + observed_at: &str, + ) { + self.aliases.push(AliasPlan { + entity: entity.clone(), + alias_type, + alias_key: alias_key.to_string(), + alias_value: alias_value.to_string(), + source_kind, + trust_level, + observed_at: observed_at.to_string(), + }); + } + + #[allow(clippy::too_many_arguments)] + fn relationship( + &mut self, + src: &PlannedEntityKey, + dst: &PlannedEntityKey, + relationship_type: &'static str, + reason_code: &'static str, + source_kind: &'static str, + source_id: &str, + observed_at: &str, + trust_level: &'static str, + confidence: f64, + safe_excerpt: &str, + ) { + self.relationships.push(RelationshipPlan { + src: src.clone(), + dst: dst.clone(), + relationship_type, + reason_code, + source_kind, + source_id: source_id.to_string(), + observed_at: observed_at.to_string(), + trust_level, + confidence, + safe_excerpt: safe_excerpt.to_string(), + }); + } +} + +fn match_upstream_key<'a>( + upstream: &str, + source: &str, + services: &'a BTreeMap, +) -> Option<&'a PlannedEntityKey> { + let normalized = canonical_or_raw(upstream); + let prefix = upstream + .split([':', '/', '@']) + .find(|part| !part.is_empty() && !is_url_scheme_token(part)) + .map(canonical_or_raw); + prefix + .and_then(|key| match_service_name_key(&key, source, services)) + .or_else(|| match_service_name_key(&normalized, source, services)) +} + +fn is_url_scheme_token(part: &str) -> bool { + part.eq_ignore_ascii_case("http") || part.eq_ignore_ascii_case("https") +} + +fn match_service_name_key<'a>( + name: &str, + source: &str, + services: &'a BTreeMap, +) -> Option<&'a PlannedEntityKey> { + source_host(source) + .and_then(|host| entity_resolution::service_instance_key(host, name)) + .and_then(|key| services.get(&key)) + .or_else(|| services.get(&canonical_or_raw(name))) +} + +fn source_host(source: &str) -> Option<&str> { + let mut parts = source.split(':'); + let _collector = parts.next()?; + let host = parts.next()?.trim(); + if host.is_empty() || host.starts_with('/') { + None + } else { + Some(host) + } +} diff --git a/crates/shared/cortex/storage-sqlite/src/graph_inventory/sql.rs b/crates/shared/cortex/storage-sqlite/src/graph_inventory/sql.rs new file mode 100644 index 00000000..16423a68 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/graph_inventory/sql.rs @@ -0,0 +1,341 @@ +use anyhow::{Context, Result}; +use rusqlite::{Connection, params}; + +use crate::graph; +use cortex_inventory::TrustLevel; + +use super::InventoryGraphStats; + +#[derive(Debug, Clone)] +pub(super) struct EntityRef { + pub(super) id: i64, + pub(super) kind: &'static str, + pub(super) key: String, +} + +pub(super) fn prune_previous_inventory_projection(conn: &Connection) -> Result<()> { + // Resolver-vocabulary edges (`instance_of` with reason + // `resolver_instance_of`) are shared with the log-driven projection, so + // they are pruned symmetrically with the evidence criteria: only rows + // backed by inventory-sourced evidence are inventory-owned. This must run + // before the evidence delete below, which removes the identifying rows. + conn.execute( + "DELETE FROM graph_relationships + WHERE reason_code = ?1 + AND id IN ( + SELECT relationship_id FROM graph_relationship_evidence + WHERE source_kind IN ('source_inventory', 'app_inventory') + )", + [graph::REASON_RESOLVER_INSTANCE_OF], + )?; + conn.execute( + "DELETE FROM graph_relationship_evidence + WHERE source_kind IN ('source_inventory', 'app_inventory')", + [], + )?; + conn.execute( + "DELETE FROM graph_relationships + WHERE reason_code IN ( + 'inventory_node', 'inventory_service', 'compose_config', + 'reverse_proxy_config', 'docker_network', 'storage_probe', + 'config_artifact' + )", + [], + )?; + conn.execute( + "DELETE FROM graph_entity_aliases + WHERE source_kind IN ('source_inventory', 'app_inventory')", + [], + )?; + conn.execute( + "DELETE FROM graph_entities + WHERE source_kind IN ('source_inventory', 'app_inventory') + AND id NOT IN (SELECT src_entity_id FROM graph_relationships) + AND id NOT IN (SELECT dst_entity_id FROM graph_relationships)", + [], + )?; + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn upsert_entity( + conn: &Connection, + entity_type: &'static str, + canonical_key: &str, + display_label: &str, + source_kind: &str, + source_id: &str, + trust_level: &str, + observed_at: &str, +) -> Result { + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, source_kind, source_id, + trust_level, first_seen_at, last_seen_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?7) + ON CONFLICT(entity_type, canonical_key) DO UPDATE SET + display_label = excluded.display_label, + last_seen_at = excluded.last_seen_at, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')", + params![ + entity_type, + canonical_key, + display_label, + source_kind, + source_id, + trust_level, + observed_at + ], + )?; + let id = conn.query_row( + "SELECT id FROM graph_entities WHERE entity_type = ?1 AND canonical_key = ?2", + params![entity_type, canonical_key], + |row| row.get(0), + )?; + Ok(EntityRef { + id, + kind: entity_type, + key: canonical_key.to_string(), + }) +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn add_alias( + conn: &Connection, + entity_id: i64, + alias_type: &str, + alias_key: &str, + alias_value: &str, + source_kind: &str, + trust_level: &str, + observed_at: &str, +) -> Result<()> { + conn.execute( + "INSERT INTO graph_entity_aliases + (entity_id, alias_type, alias_key, alias_value, source_kind, + trust_level, first_seen_at, last_seen_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?7) + ON CONFLICT(entity_id, alias_type, alias_key, source_kind) DO UPDATE SET + alias_value = excluded.alias_value, + trust_level = excluded.trust_level, + last_seen_at = excluded.last_seen_at, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')", + params![ + entity_id, + alias_type, + alias_key, + alias_value, + source_kind, + trust_level, + observed_at + ], + )?; + Ok(()) +} + +#[allow(clippy::too_many_arguments)] +pub(super) fn add_relationship( + conn: &Connection, + src: &EntityRef, + dst: &EntityRef, + relationship_type: &str, + reason_code: &str, + source_kind: &str, + source_id: &str, + observed_at: &str, + trust_level: &str, + confidence: f64, + safe_excerpt: &str, +) -> Result<()> { + let key = format!( + "{}:{}->{}:{}:{}", + src.kind, src.key, dst.kind, dst.key, relationship_type + ); + conn.execute( + "INSERT INTO graph_relationships + (relationship_key, src_entity_id, dst_entity_id, relationship_type, + reason_code, trust_level, confidence, evidence_count, first_seen_at, last_seen_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, 1, ?8, ?8) + ON CONFLICT(src_entity_id, dst_entity_id, relationship_type, relationship_key) + DO UPDATE SET + reason_code = excluded.reason_code, + trust_level = excluded.trust_level, + confidence = MAX(graph_relationships.confidence, excluded.confidence), + last_seen_at = excluded.last_seen_at, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')", + params![ + key, + src.id, + dst.id, + relationship_type, + reason_code, + trust_level, + confidence, + observed_at + ], + )?; + let rel_id: i64 = conn.query_row( + "SELECT id FROM graph_relationships WHERE relationship_key = ?1", + [&key], + |row| row.get(0), + )?; + conn.execute( + "INSERT INTO graph_relationship_evidence + (relationship_id, evidence_key, source_kind, source_id, observed_at, + reason_code, reason_text, confidence_delta, trust_level, safe_excerpt, + evidence_count) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, 1) + ON CONFLICT(relationship_id, evidence_key) DO UPDATE SET + observed_at = excluded.observed_at, + reason_text = excluded.reason_text, + confidence_delta = excluded.confidence_delta, + trust_level = excluded.trust_level, + safe_excerpt = excluded.safe_excerpt, + evidence_count = excluded.evidence_count", + params![ + rel_id, + format!( + "{source_kind}:{source_id}:{reason_code}:{}:{}", + src.key, dst.key + ), + source_kind, + source_id, + observed_at, + reason_code, + reason_code.replace('_', " "), + confidence, + trust_level, + truncate_excerpt(safe_excerpt) + ], + )?; + conn.execute( + "UPDATE graph_relationships + SET evidence_count = ( + SELECT COALESCE(SUM(evidence_count), 0) + FROM graph_relationship_evidence + WHERE relationship_id = ?1 + ) + WHERE id = ?1", + [rel_id], + )?; + Ok(()) +} + +pub(super) fn update_projection_meta( + conn: &Connection, + counts: &InventoryGraphStats, +) -> Result<()> { + conn.execute( + "UPDATE graph_projection_meta + SET projection_status = 'ready', + last_started_at = COALESCE(last_started_at, strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + last_completed_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), + entity_count = ?1, + relationship_count = ?2, + evidence_count = ?3, + is_degraded = 0, + last_error = NULL, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + WHERE id = 1", + params![ + counts.entity_count, + counts.relationship_count, + counts.evidence_count + ], + )?; + Ok(()) +} + +pub(super) fn graph_counts(conn: &Connection) -> Result { + Ok(InventoryGraphStats { + source_row_count: conn + .query_row( + "SELECT source_row_count FROM graph_projection_meta WHERE id = 1", + [], + |row| row.get(0), + ) + .unwrap_or(0), + entity_count: table_count(conn, "graph_entities")?, + relationship_count: table_count(conn, "graph_relationships")?, + evidence_count: table_count(conn, "graph_relationship_evidence")?, + }) +} + +pub(super) fn mark_projection_degraded(conn: &Connection, error: &str) -> Result<()> { + conn.execute( + "UPDATE graph_projection_meta + SET is_degraded = 1, + last_error = ?1, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + WHERE id = 1", + [truncate_excerpt(error)], + )?; + Ok(()) +} + +fn table_count(conn: &Connection, table: &str) -> Result { + conn.query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .with_context(|| format!("count {table}")) +} + +pub(super) fn scoped_inventory_key(source: &str, name: &str) -> String { + let scope = source_host(source).unwrap_or("unknown"); + canonical_or_raw(&format!("{scope}:{name}")) +} + +pub(super) fn safe_inventory_source_id(source: &str) -> String { + match source_host(source) { + Some(host) => format!( + "{}:{}", + source.split(':').next().unwrap_or("inventory").trim(), + host + ), + None => source + .split(':') + .next() + .filter(|collector| !collector.trim().is_empty()) + .unwrap_or("inventory") + .to_string(), + } +} + +pub(super) fn canonical_or_raw(value: &str) -> String { + canonical(value).unwrap_or_else(|| value.trim().to_ascii_lowercase()) +} + +pub(super) fn canonical(value: &str) -> Option { + graph::canonical_graph_key(value) +} + +fn source_host(source: &str) -> Option<&str> { + let mut parts = source.split(':'); + let _collector = parts.next()?; + let host = parts.next()?.trim(); + if host.is_empty() || host.starts_with('/') { + None + } else { + Some(host) + } +} + +pub(super) fn trust(value: &TrustLevel) -> &'static str { + match value { + TrustLevel::Verified | TrustLevel::Observed => graph::TRUST_VERIFIED, + TrustLevel::Claimed => graph::TRUST_CLAIMED, + TrustLevel::Inferred => graph::TRUST_INFERRED, + } +} + +fn truncate_excerpt(value: &str) -> String { + const MAX: usize = 512; + if value.len() <= MAX { + return value.to_string(); + } + value.chars().take(MAX).collect() +} + +#[cfg(test)] +#[path = "sql_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/graph_inventory/sql_tests.rs b/crates/shared/cortex/storage-sqlite/src/graph_inventory/sql_tests.rs new file mode 100644 index 00000000..f182d98f --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/graph_inventory/sql_tests.rs @@ -0,0 +1,25 @@ +use super::*; + +#[test] +fn inventory_scope_and_source_ids_are_host_bounded() { + assert_eq!(scoped_inventory_key("docker:NAS", "Plex"), "nas:plex"); + assert_eq!( + safe_inventory_source_id("compose:nas:/opt/stack/compose.yaml"), + "compose:nas" + ); + assert_eq!(safe_inventory_source_id("inventory"), "inventory"); +} + +#[test] +fn inventory_trust_maps_to_graph_vocabulary() { + assert_eq!(trust(&TrustLevel::Verified), graph::TRUST_VERIFIED); + assert_eq!(trust(&TrustLevel::Observed), graph::TRUST_VERIFIED); + assert_eq!(trust(&TrustLevel::Claimed), graph::TRUST_CLAIMED); + assert_eq!(trust(&TrustLevel::Inferred), graph::TRUST_INFERRED); +} + +#[test] +fn projection_errors_are_bounded() { + let value = "x".repeat(600); + assert_eq!(truncate_excerpt(&value).len(), 512); +} diff --git a/crates/shared/cortex/storage-sqlite/src/graph_inventory_tests.rs b/crates/shared/cortex/storage-sqlite/src/graph_inventory_tests.rs new file mode 100644 index 00000000..c6ad350c --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/graph_inventory_tests.rs @@ -0,0 +1,1017 @@ +use super::*; +use crate::config::StorageConfig; +use crate::{LogBatchEntry, graph, init_pool, insert_logs_batch}; +use cortex_inventory::{ + ArtifactRef, ComposeProject, HomelabInventory, InventoryNode, InventoryService, NetworkSegment, + PortMapping, Provenance, RedactionStatus, ReverseProxyRoute, TrustLevel, +}; +use std::sync::mpsc; +use std::thread; +use std::time::{Duration, Instant}; + +fn count(conn: &rusqlite::Connection, sql: &str) -> i64 { + conn.query_row(sql, [], |row| row.get(0)).unwrap() +} + +fn relationship_count(conn: &rusqlite::Connection, rel_type: &str, reason: &str) -> i64 { + conn.query_row( + "SELECT COUNT(*) + FROM graph_relationships r + JOIN graph_entities src ON src.id = r.src_entity_id + JOIN graph_entities dst ON dst.id = r.dst_entity_id + WHERE r.relationship_type = ?1 + AND r.reason_code = ?2 + AND src.canonical_key <> '' + AND dst.canonical_key <> ''", + rusqlite::params![rel_type, reason], + |row| row.get(0), + ) + .unwrap() +} + +fn provenance(source: &str, kind: &str) -> Provenance { + Provenance::new(source, kind, "2026-01-01T00:00:00Z".to_string()) +} + +fn basic_inventory() -> HomelabInventory { + let mut inventory = + HomelabInventory::empty("inv-test".to_string(), "2026-01-01T00:00:00Z".to_string()); + inventory.nodes.push(InventoryNode { + id: "node:devhost".to_string(), + hostname: "devhost".to_string(), + trust_level: TrustLevel::Observed, + provenance: provenance("ssh:devhost", "source_inventory"), + roles: Vec::new(), + ips: vec!["192.0.2.6".to_string()], + os: Some("Ubuntu".to_string()), + cpu: None, + memory: None, + listeners: Vec::new(), + storage: Vec::new(), + extras: Default::default(), + }); + inventory +} + +fn log_entry(message: &str) -> LogBatchEntry { + LogBatchEntry { + timestamp: "2026-01-01T00:00:00Z".to_string(), + hostname: "writer-test".to_string(), + facility: Some("daemon".to_string()), + severity: "info".to_string(), + app_name: Some("test".to_string()), + process_id: None, + message: message.to_string(), + raw: format!("<14>{message}"), + source_ip: "127.0.0.1:1514".to_string(), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + } +} + +#[test] +fn inventory_projection_marks_never_built_graph_ready() { + let _guard = graph::GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test( + dir.path().join("inventory-graph-status.db"), + )) + .unwrap(); + + assert_eq!( + graph::graph_projection_status(&pool) + .unwrap() + .projection_status, + "never_built" + ); + + project_inventory(&pool, &basic_inventory()).unwrap(); + + let status = graph::graph_projection_status(&pool).unwrap(); + assert_eq!(status.projection_status, "ready"); + assert!(status.last_completed_at.is_some()); + assert!(!status.is_degraded); +} + +#[test] +fn inventory_projection_keeps_heartbeat_only_hosts_resolvable() { + let _guard = graph::GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test( + dir.path().join("inventory-heartbeat-host.db"), + )) + .unwrap(); + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO host_heartbeats ( + host_id, hostname, source_ip, sampled_at, received_at, boot_id, + uptime_secs, sequence, collection_ms, partial, agent_version, + os, architecture, metadata_json + ) VALUES ('host-devhost', 'devhost', '192.0.2.6:1514', + '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z', 'boot-a', + 60, 1, 5, 0, '0.1.0', 'linux', 'x86_64', '{}')", + [], + ) + .unwrap(); + let heartbeat_id = conn.last_insert_rowid(); + conn.execute( + "INSERT INTO host_heartbeats_latest ( + host_id, heartbeat_id, hostname, sampled_at, received_at, + partial, agent_version, os, architecture, metadata_json + ) VALUES ('host-devhost', ?1, 'devhost', '2026-01-01T00:00:00Z', + '2026-01-01T00:00:00Z', 0, '0.1.0', 'linux', 'x86_64', '{}')", + [heartbeat_id], + ) + .unwrap(); + drop(conn); + + let inventory = + HomelabInventory::empty("inv-test".to_string(), "2026-01-01T00:00:00Z".to_string()); + project_inventory(&pool, &inventory).unwrap(); + + let conn = pool.get().unwrap(); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities + WHERE entity_type = 'host' AND canonical_key = 'devhost'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) + FROM graph_entity_aliases a + JOIN graph_entities e ON e.id = a.entity_id + WHERE e.entity_type = 'host' + AND e.canonical_key = 'devhost' + AND a.alias_type = 'hostname' + AND a.alias_key = 'devhost'" + ), + 1 + ); +} + +#[test] +fn project_inventory_does_not_hold_write_lock_while_preparing_projection() { + let _guard = graph::GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test( + dir.path().join("inventory-graph-lock-scope.db"), + )) + .unwrap(); + graph::refresh_graph_projection(&pool).unwrap(); + + let inventory = basic_inventory(); + let projection_pool = pool.clone(); + let (prepared_tx, prepared_rx) = mpsc::channel(); + let (continue_tx, continue_rx) = mpsc::channel(); + let projection = thread::spawn(move || { + project_inventory_with_apply_hook(&projection_pool, &inventory, || { + prepared_tx.send(()).unwrap(); + continue_rx.recv().unwrap(); + }) + .unwrap(); + }); + + prepared_rx.recv_timeout(Duration::from_secs(2)).unwrap(); + let writer_pool = pool.clone(); + let (insert_done_tx, insert_done_rx) = mpsc::channel(); + let writer = thread::spawn(move || { + let started = Instant::now(); + insert_logs_batch( + &writer_pool, + &[log_entry("write while projection prepares")], + ) + .unwrap(); + insert_done_tx.send(started.elapsed()).unwrap(); + }); + + let writer_timeout = Duration::from_secs(5); + match insert_done_rx.recv_timeout(writer_timeout) { + Ok(elapsed) => assert!( + elapsed < writer_timeout, + "insert waited for projection preparation for {elapsed:?}" + ), + Err(error) => { + continue_tx.send(()).unwrap(); + projection.join().unwrap(); + writer.join().unwrap(); + panic!("insert was blocked by projection preparation: {error}"); + } + } + + continue_tx.send(()).unwrap(); + projection.join().unwrap(); + writer.join().unwrap(); +} + +#[test] +fn project_inventory_adds_topology_entities_relationships_and_evidence() { + let _guard = graph::GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test( + dir.path().join("inventory-graph.db"), + )) + .unwrap(); + graph::refresh_graph_projection(&pool).unwrap(); + + let mut inventory = + HomelabInventory::empty("inv-test".to_string(), "2026-01-01T00:00:00Z".to_string()); + inventory.nodes.push(InventoryNode { + id: "node:edgehost".to_string(), + hostname: "edgehost".to_string(), + trust_level: TrustLevel::Observed, + provenance: provenance("ssh:edgehost", "source_inventory"), + roles: vec!["edge".to_string()], + ips: vec!["192.0.2.8".to_string()], + os: Some("Ubuntu".to_string()), + cpu: None, + memory: None, + listeners: Vec::new(), + storage: Vec::new(), + extras: Default::default(), + }); + inventory.services.push(InventoryService { + id: "container:edgehost:swag".to_string(), + name: "swag".to_string(), + kind: "container".to_string(), + trust_level: TrustLevel::Observed, + provenance: provenance("docker:edgehost", "app_inventory"), + host: Some("edgehost".to_string()), + image: Some("lscr.io/linuxserver/swag:latest".to_string()), + status: Some("running".to_string()), + domains: vec!["example.example.invalid".to_string()], + ports: vec![PortMapping { + host_ip: Some("0.0.0.0".to_string()), + host_port: Some(443), + container_port: Some(443), + protocol: "tcp".to_string(), + }], + mounts: Vec::new(), + env_keys: vec!["URL".to_string()], + labels: Default::default(), + details: Default::default(), + }); + inventory.compose_projects.push(ComposeProject { + name: "edge".to_string(), + provenance: provenance("compose:edgehost:/opt/edge/compose.yaml", "app_inventory"), + services: vec!["swag".to_string()], + compose_files: vec!["/opt/edge/compose.yaml".to_string()], + domains: vec!["example.example.invalid".to_string()], + ports: Vec::new(), + }); + inventory.reverse_proxies.push(ReverseProxyRoute { + id: "proxy:example.example.invalid".to_string(), + server_names: vec!["example.example.invalid".to_string()], + upstreams: vec!["swag:443".to_string()], + provenance: provenance("swag:edgehost:/config/nginx/proxy.conf", "app_inventory"), + }); + inventory.artifact_refs.push(ArtifactRef { + id: "artifact:compose:edgehost:edge".to_string(), + kind: "compose".to_string(), + collector: "raw_configs".to_string(), + source_host: Some("edgehost".to_string()), + source_path: Some("/opt/edge/compose.yaml".to_string()), + cache_path: "/home/jmagar/.cortex/inventory/artifacts/edge.yaml".to_string(), + redaction: RedactionStatus::Redacted, + byte_len: 42, + truncated: false, + }); + + let stats = project_inventory(&pool, &inventory).unwrap(); + assert_eq!(stats.source_row_count, 0); + assert!(stats.entity_count >= 6); + assert!(stats.relationship_count >= 4); + assert!(stats.evidence_count >= 4); + + let conn = pool.get().unwrap(); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities + WHERE entity_type = 'host' AND canonical_key = 'edgehost'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entity_aliases + WHERE alias_type = 'ip' AND alias_key = '192.0.2.8'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entity_aliases + WHERE alias_type = 'domain' AND alias_key = 'example.example.invalid'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities + WHERE entity_type = 'compose_project' AND canonical_key = 'edgehost:edge'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities + WHERE entity_type = 'reverse_proxy' AND canonical_key = 'proxy:example.example.invalid'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities + WHERE entity_type = 'domain' AND canonical_key = 'example.example.invalid'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities + WHERE entity_type = 'config_artifact' + AND canonical_key = 'artifact:compose:edgehost:edge'" + ), + 1 + ); + assert_eq!(relationship_count(&conn, "runs_on", "inventory_service"), 1); + assert_eq!( + relationship_count(&conn, "defines_service", "compose_config"), + 1 + ); + assert_eq!( + relationship_count(&conn, "routes_to", "reverse_proxy_config"), + 1 + ); + assert_eq!( + relationship_count(&conn, "exposes_domain", "reverse_proxy_config"), + 1 + ); + assert_eq!( + relationship_count(&conn, "has_artifact", "config_artifact"), + 1 + ); + // 6 evidence rows: runs_on, instance_of, defines_service, routes_to, + // exposes_domain, has_artifact. + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_relationship_evidence + WHERE source_kind IN ('source_inventory', 'app_inventory')" + ), + 6 + ); + // Hard break: inventory services project as service_instance + + // logical_service, never legacy `service` rows. + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'service'" + ), + 0 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities + WHERE entity_type = 'service_instance' AND canonical_key = 'edgehost/swag'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities + WHERE entity_type = 'logical_service' AND canonical_key = 'swag'" + ), + 1 + ); +} + +#[test] +fn project_inventory_preserves_existing_entity_ownership_and_hides_config_paths() { + let _guard = graph::GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test( + dir.path().join("inventory-graph-ownership.db"), + )) + .unwrap(); + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, source_kind, source_id, trust_level) + VALUES ('host', 'edgehost', 'edgehost', 'log', '42', 'claimed')", + [], + ) + .unwrap(); + drop(conn); + + let mut inventory = + HomelabInventory::empty("inv-test".to_string(), "2026-01-01T00:00:00Z".to_string()); + inventory.nodes.push(InventoryNode { + id: "node:edgehost".to_string(), + hostname: "edgehost".to_string(), + trust_level: TrustLevel::Observed, + provenance: provenance("ssh:edgehost", "source_inventory"), + roles: Vec::new(), + ips: Vec::new(), + os: None, + cpu: None, + memory: None, + listeners: Vec::new(), + storage: Vec::new(), + extras: Default::default(), + }); + inventory.artifact_refs.push(ArtifactRef { + id: "artifact:compose:edgehost:edge".to_string(), + kind: "compose".to_string(), + collector: "raw_configs".to_string(), + source_host: Some("edgehost".to_string()), + source_path: Some("/opt/edge/compose.yaml".to_string()), + cache_path: "/home/jmagar/.cortex/inventory/raw/inv/edge.txt".to_string(), + redaction: RedactionStatus::Redacted, + byte_len: 42, + truncated: false, + }); + inventory.compose_projects.push(ComposeProject { + name: "edge".to_string(), + provenance: provenance("compose:edgehost:/opt/edge/compose.yaml", "app_inventory"), + services: Vec::new(), + compose_files: vec!["/opt/edge/compose.yaml".to_string()], + domains: Vec::new(), + ports: Vec::new(), + }); + + project_inventory(&pool, &inventory).unwrap(); + let conn = pool.get().unwrap(); + let (source_kind, source_id, trust_level): (String, String, String) = conn + .query_row( + "SELECT source_kind, source_id, trust_level + FROM graph_entities + WHERE entity_type = 'host' AND canonical_key = 'edgehost'", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!(source_kind, "log"); + assert_eq!(source_id, "42"); + assert_eq!(trust_level, "claimed"); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities + WHERE entity_type = 'config_artifact' + AND (display_label LIKE '%/opt/%' OR display_label LIKE '%/.cortex/%')" + ), + 0 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entity_aliases + WHERE alias_type = 'path' + OR alias_value LIKE '%/opt/%' + OR alias_value LIKE '%/.cortex/%'" + ), + 0 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_relationship_evidence + WHERE source_id LIKE '%/opt/%' + OR safe_excerpt LIKE '%/opt/%'" + ), + 0 + ); +} + +#[test] +fn project_inventory_does_not_route_to_ambiguous_service_name() { + let _guard = graph::GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test( + dir.path().join("inventory-graph-ambiguous.db"), + )) + .unwrap(); + graph::refresh_graph_projection(&pool).unwrap(); + + let mut inventory = + HomelabInventory::empty("inv-test".to_string(), "2026-01-01T00:00:00Z".to_string()); + for host in ["edgehost", "nashost"] { + inventory.nodes.push(InventoryNode { + id: format!("node:{host}"), + hostname: host.to_string(), + trust_level: TrustLevel::Observed, + provenance: provenance(&format!("ssh:{host}"), "source_inventory"), + roles: Vec::new(), + ips: Vec::new(), + os: None, + cpu: None, + memory: None, + listeners: Vec::new(), + storage: Vec::new(), + extras: Default::default(), + }); + inventory.services.push(InventoryService { + id: format!("container:{host}:swag"), + name: "swag".to_string(), + kind: "container".to_string(), + trust_level: TrustLevel::Observed, + provenance: provenance(&format!("docker:{host}"), "app_inventory"), + host: Some(host.to_string()), + image: None, + status: Some("running".to_string()), + domains: Vec::new(), + ports: Vec::new(), + mounts: Vec::new(), + env_keys: Vec::new(), + labels: Default::default(), + details: Default::default(), + }); + } + inventory.reverse_proxies.push(ReverseProxyRoute { + id: "proxy:ambiguous.example.invalid".to_string(), + server_names: vec!["ambiguous.example.invalid".to_string()], + upstreams: vec!["swag:443".to_string()], + provenance: provenance("swag:/config/nginx/proxy.conf", "app_inventory"), + }); + + project_inventory(&pool, &inventory).unwrap(); + let conn = pool.get().unwrap(); + assert_eq!(relationship_count(&conn, "runs_on", "inventory_service"), 2); + assert_eq!( + relationship_count(&conn, "routes_to", "reverse_proxy_config"), + 0 + ); +} + +#[test] +fn project_inventory_routes_to_service_name_beginning_with_http() { + let _guard = graph::GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test( + dir.path().join("inventory-graph-http-prefix.db"), + )) + .unwrap(); + graph::refresh_graph_projection(&pool).unwrap(); + + let mut inventory = + HomelabInventory::empty("inv-test".to_string(), "2026-01-01T00:00:00Z".to_string()); + inventory.nodes.push(InventoryNode { + id: "node:edgehost".to_string(), + hostname: "edgehost".to_string(), + trust_level: TrustLevel::Observed, + provenance: provenance("ssh:edgehost", "source_inventory"), + roles: Vec::new(), + ips: Vec::new(), + os: None, + cpu: None, + memory: None, + listeners: Vec::new(), + storage: Vec::new(), + extras: Default::default(), + }); + inventory.services.push(InventoryService { + id: "container:edgehost:http-api".to_string(), + name: "http-api".to_string(), + kind: "container".to_string(), + trust_level: TrustLevel::Observed, + provenance: provenance("docker:edgehost", "app_inventory"), + host: Some("edgehost".to_string()), + image: None, + status: Some("running".to_string()), + domains: Vec::new(), + ports: Vec::new(), + mounts: Vec::new(), + env_keys: Vec::new(), + labels: Default::default(), + details: Default::default(), + }); + inventory.reverse_proxies.push(ReverseProxyRoute { + id: "proxy:http-api.example.invalid".to_string(), + server_names: vec!["http-api.example.invalid".to_string()], + upstreams: vec!["http://http-api:8080".to_string()], + provenance: provenance("swag:edgehost:/config/nginx/proxy.conf", "app_inventory"), + }); + + project_inventory(&pool, &inventory).unwrap(); + let conn = pool.get().unwrap(); + assert_eq!( + relationship_count(&conn, "routes_to", "reverse_proxy_config"), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) + FROM graph_relationships rel + JOIN graph_entities dst ON dst.id = rel.dst_entity_id + WHERE rel.relationship_type = 'routes_to' + AND dst.entity_type = 'service_instance' + AND dst.canonical_key = 'edgehost/http-api'" + ), + 1 + ); +} + +#[test] +fn project_inventory_scopes_compose_projects_and_networks_by_source_host() { + let _guard = graph::GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test( + dir.path().join("inventory-graph-scoped.db"), + )) + .unwrap(); + graph::refresh_graph_projection(&pool).unwrap(); + + let mut inventory = + HomelabInventory::empty("inv-test".to_string(), "2026-01-01T00:00:00Z".to_string()); + for host in ["edgehost", "nashost"] { + inventory.nodes.push(InventoryNode { + id: format!("node:{host}"), + hostname: host.to_string(), + trust_level: TrustLevel::Observed, + provenance: provenance(&format!("ssh:{host}"), "source_inventory"), + roles: Vec::new(), + ips: Vec::new(), + os: None, + cpu: None, + memory: None, + listeners: Vec::new(), + storage: Vec::new(), + extras: Default::default(), + }); + inventory.services.push(InventoryService { + id: format!("container:{host}:swag"), + name: "swag".to_string(), + kind: "container".to_string(), + trust_level: TrustLevel::Observed, + provenance: provenance(&format!("docker:{host}"), "app_inventory"), + host: Some(host.to_string()), + image: None, + status: Some("running".to_string()), + domains: Vec::new(), + ports: Vec::new(), + mounts: Vec::new(), + env_keys: Vec::new(), + labels: Default::default(), + details: Default::default(), + }); + inventory.compose_projects.push(ComposeProject { + name: "edge".to_string(), + provenance: provenance( + &format!("compose:{host}:/opt/edge/compose.yaml"), + "app_inventory", + ), + services: vec!["swag".to_string()], + compose_files: Vec::new(), + domains: Vec::new(), + ports: Vec::new(), + }); + inventory.networks.push(NetworkSegment { + name: "bridge".to_string(), + kind: "docker".to_string(), + members: vec!["swag".to_string()], + provenance: provenance(&format!("docker:{host}"), "app_inventory"), + details: Default::default(), + }); + } + + project_inventory(&pool, &inventory).unwrap(); + let conn = pool.get().unwrap(); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'compose_project'" + ), + 2 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'network'" + ), + 2 + ); + assert_eq!( + relationship_count(&conn, "defines_service", "compose_config"), + 2 + ); + assert_eq!( + relationship_count(&conn, "attached_to", "docker_network"), + 2 + ); +} + +#[test] +fn reprojection_prunes_stale_resolver_instance_of_edges_when_service_moves_hosts() { + let _guard = graph::GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test( + dir.path().join("inventory-reprojection-prune.db"), + )) + .unwrap(); + + let inventory_with_plex_on = |host: &str| { + let mut inventory = + HomelabInventory::empty("inv-test".to_string(), "2026-01-01T00:00:00Z".to_string()); + inventory.nodes.push(InventoryNode { + id: format!("node:{host}"), + hostname: host.to_string(), + trust_level: TrustLevel::Observed, + provenance: provenance(&format!("ssh:{host}"), "source_inventory"), + roles: Vec::new(), + ips: Vec::new(), + os: None, + cpu: None, + memory: None, + listeners: Vec::new(), + storage: Vec::new(), + extras: Default::default(), + }); + inventory.services.push(InventoryService { + id: format!("container:{host}:plex"), + name: "plex".to_string(), + kind: "container".to_string(), + trust_level: TrustLevel::Observed, + provenance: provenance(&format!("docker:{host}"), "app_inventory"), + host: Some(host.to_string()), + image: None, + status: Some("running".to_string()), + domains: Vec::new(), + ports: Vec::new(), + mounts: Vec::new(), + env_keys: Vec::new(), + labels: Default::default(), + details: Default::default(), + }); + inventory + }; + + project_inventory(&pool, &inventory_with_plex_on("nashost")).unwrap(); + { + let conn = pool.get().unwrap(); + assert_eq!( + relationship_count(&conn, "instance_of", "resolver_instance_of"), + 1 + ); + } + + // Plex moves to backuphost: re-projection must not leak the stale + // nashost/plex instance_of edge or leave orphan evidence behind. + project_inventory(&pool, &inventory_with_plex_on("backuphost")).unwrap(); + let conn = pool.get().unwrap(); + assert_eq!( + relationship_count(&conn, "instance_of", "resolver_instance_of"), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) + FROM graph_relationships r + JOIN graph_entities src ON src.id = r.src_entity_id + WHERE r.relationship_type = 'instance_of' + AND src.canonical_key = 'nashost/plex'" + ), + 0 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities + WHERE entity_type = 'service_instance' AND canonical_key = 'nashost/plex'" + ), + 0 + ); + // No orphan evidence: every evidence row must reference a live edge. + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_relationship_evidence e + WHERE NOT EXISTS ( + SELECT 1 FROM graph_relationships r WHERE r.id = e.relationship_id + )" + ), + 0 + ); +} + +#[test] +fn double_projection_with_log_and_inventory_instance_of_leaves_no_orphans() { + let _guard = graph::GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test( + dir.path().join("inventory-log-instance-overlap.db"), + )) + .unwrap(); + + // Log-driven instance_of: agent-docker structured metadata row for + // (nashost/plex, plex), projected by the log extraction path. + let mut entry = log_entry("Plex started"); + entry.hostname = "nashost".to_string(); + entry.app_name = Some("plex".to_string()); + entry.metadata_json = Some( + r#"{"source_kind":"agent-docker","agent_docker":{"host":"nashost","container_id":"abcdef1234567890","container_name":"plex","compose_project":"plex","compose_service":"plex","stream":"stdout"}}"# + .to_string(), + ); + insert_logs_batch(&pool, &[entry]).unwrap(); + graph::refresh_graph_projection(&pool).unwrap(); + + // Inventory-driven instance_of for the SAME (instance, logical) pair. + let mut inventory = + HomelabInventory::empty("inv-test".to_string(), "2026-01-01T00:00:00Z".to_string()); + inventory.services.push(InventoryService { + id: "container:nashost:plex".to_string(), + name: "plex".to_string(), + kind: "container".to_string(), + trust_level: TrustLevel::Observed, + provenance: provenance("docker:nashost", "app_inventory"), + host: Some("nashost".to_string()), + image: None, + status: Some("running".to_string()), + domains: Vec::new(), + ports: Vec::new(), + mounts: Vec::new(), + env_keys: Vec::new(), + labels: Default::default(), + details: Default::default(), + }); + project_inventory(&pool, &inventory).unwrap(); + + let snapshot = |conn: &rusqlite::Connection| -> (i64, i64) { + ( + count( + conn, + "SELECT COUNT(*) FROM graph_relationships + WHERE relationship_type = 'instance_of'", + ), + count(conn, "SELECT COUNT(*) FROM graph_relationship_evidence"), + ) + }; + let (rels_first, evidence_first) = { + let conn = pool.get().unwrap(); + snapshot(&conn) + }; + + // Double projection: re-project the same inventory. The log-driven and + // inventory-driven paths use distinct relationship_key shapes BY DESIGN, + // so we assert stability and zero orphans — NOT exactly-one edge. + project_inventory(&pool, &inventory).unwrap(); + let conn = pool.get().unwrap(); + let (rels_second, evidence_second) = snapshot(&conn); + assert_eq!( + rels_first, rels_second, + "instance_of rowcount must be stable" + ); + assert_eq!( + evidence_first, evidence_second, + "evidence rowcount must be stable" + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_relationship_evidence e + WHERE NOT EXISTS ( + SELECT 1 FROM graph_relationships r WHERE r.id = e.relationship_id + )" + ), + 0, + "no evidence row may reference a dead relationship" + ); +} + +#[test] +fn inventory_projection_links_service_instance_to_host_storage_compose_and_route() { + let _guard = graph::GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test( + dir.path().join("inventory-service-instance.db"), + )) + .unwrap(); + let mut inventory = + HomelabInventory::empty("plex-proof".to_string(), "2026-01-01T00:00:00Z".to_string()); + inventory.nodes.push(InventoryNode { + id: "node:nashost".to_string(), + hostname: "nashost".to_string(), + trust_level: TrustLevel::Observed, + provenance: provenance("ssh:nashost", "source_inventory"), + roles: Vec::new(), + ips: vec!["198.51.100.2".to_string()], + os: Some("Unraid".to_string()), + cpu: None, + memory: None, + listeners: Vec::new(), + storage: Vec::new(), + extras: Default::default(), + }); + inventory.services.push(InventoryService { + id: "service:nashost:plex".to_string(), + name: "plex".to_string(), + kind: "container".to_string(), + trust_level: TrustLevel::Observed, + provenance: provenance("docker:nashost", "app_inventory"), + host: Some("nashost".to_string()), + image: Some("lscr.io/linuxserver/plex:latest".to_string()), + status: Some("running".to_string()), + domains: vec!["plex.example.invalid".to_string()], + ports: vec![PortMapping { + host_ip: Some("0.0.0.0".to_string()), + host_port: Some(32400), + container_port: Some(32400), + protocol: "tcp".to_string(), + }], + mounts: vec![cortex_inventory::MountRef { + source: Some("/mnt/user/media".to_string()), + target: "/media".to_string(), + read_only: false, + }], + env_keys: Vec::new(), + labels: Default::default(), + details: Default::default(), + }); + inventory.compose_projects.push(ComposeProject { + name: "plex".to_string(), + provenance: provenance("compose:nashost:/opt/plex/compose.yaml", "app_inventory"), + services: vec!["plex".to_string()], + compose_files: Vec::new(), + domains: Vec::new(), + ports: Vec::new(), + }); + inventory.reverse_proxies.push(ReverseProxyRoute { + id: "proxy:plex.example.invalid".to_string(), + server_names: vec!["plex.example.invalid".to_string()], + upstreams: vec!["plex:32400".to_string()], + provenance: provenance("swag:nashost:/config/nginx/plex.conf", "app_inventory"), + }); + project_inventory(&pool, &inventory).unwrap(); + let conn = pool.get().unwrap(); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'service_instance' AND canonical_key = 'nashost/plex'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'logical_service' AND canonical_key = 'plex'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'service'" + ), + 0 + ); + assert_eq!( + relationship_count(&conn, "instance_of", "resolver_instance_of"), + 1 + ); + assert_eq!(relationship_count(&conn, "runs_on", "inventory_service"), 1); + assert_eq!( + relationship_count(&conn, "defines_service", "compose_config"), + 1 + ); + assert_eq!( + relationship_count(&conn, "routes_to", "reverse_proxy_config"), + 1 + ); + assert_eq!(relationship_count(&conn, "mounts", "storage_probe"), 1); + // Every service edge terminates at the service_instance, never at a + // legacy `service` node. + assert_eq!( + count( + &conn, + "SELECT COUNT(*) + FROM graph_relationships rel + JOIN graph_entities dst ON dst.id = rel.dst_entity_id + WHERE rel.relationship_type IN ('defines_service', 'routes_to') + AND dst.entity_type <> 'service_instance'" + ), + 0 + ); +} diff --git a/crates/shared/cortex/storage-sqlite/src/graph_resolver_projection.rs b/crates/shared/cortex/storage-sqlite/src/graph_resolver_projection.rs new file mode 100644 index 00000000..56212738 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/graph_resolver_projection.rs @@ -0,0 +1,355 @@ +//! Canonical-entity-resolution projection glue. +//! +//! Converts deterministic entity-resolver decisions (canonical +//! `logical_service` / `service_instance` identity derived from structured +//! agent-Docker metadata) into investigation-graph rows, plus the two other +//! genuinely resolver-specific, cleanly-separable pieces that consume that +//! projection: the bounded service-topic graph walk and the legacy +//! pre-resolver topology cleanup. +//! +//! Extracted from `graph.rs` (syslog-mcp-6ipjl). The shared extraction +//! machinery this glue calls into — `EntityMemo`, `ensure_entity_memoized`, +//! `ensure_relationship_with_evidence`, `LogGraphRow`, `EvidenceInput`, and +//! the rest of the `extract_*_row` dispatch family — stays in `graph.rs` +//! because it is general-purpose (shared by every extractor, not just the +//! resolver path); moving it here would drag along the whole dispatch tree. +//! This module owns only the pieces that are cleanly separable from that +//! machinery and specific to projecting resolver decisions. + +use anyhow::Result; +use serde_json::Value; + +use crate::graph::{self, EntityMemo, EvidenceInput, GraphWalkEntity, LogGraphRow}; +use crate::write_lock; + +/// Bounded entity cap for service-topic walks (final result LIMIT). +pub const GRAPH_SERVICE_TOPIC_ENTITY_CAP: usize = 250; +/// Per-depth allowance folded into the aggregate CTE row budget of +/// service-topic walks (`ENTITY_CAP + HOP_CAP * GRAPH_WALK_MAX_DEPTH`). +/// SQLite's recursive CTE `LIMIT` is a single overall budget — there is no +/// per-level cap. +pub const GRAPH_SERVICE_TOPIC_HOP_CAP: usize = 50; + +/// Relationship types a service-topic walk may traverse: only the edges +/// needed for the canonical service proof. Deliberately excludes the broad +/// log-identity edges (`observed_as`, `emitted_by`) so a service topic never +/// silently expands to all logs for the host running the service. +pub const GRAPH_SERVICE_TOPIC_RELATIONSHIPS: &[&str] = &[ + graph::REL_INSTANCE_OF, + graph::REL_RUNS_ON, + graph::REL_DEFINES_SERVICE, + graph::REL_ROUTES_TO, + graph::REL_EXPOSES_DOMAIN, + graph::REL_MOUNTS, + graph::REL_HAS_ARTIFACT, + graph::REL_MATCHES_SIGNATURE, + graph::REL_WORKED_ON, +]; + +/// Bounded breadth-first walk for service-topic lookups: traverses only +/// [`GRAPH_SERVICE_TOPIC_RELATIONSHIPS`], bounds the whole recursive +/// expansion at an aggregate CTE row budget of +/// `GRAPH_SERVICE_TOPIC_ENTITY_CAP + GRAPH_SERVICE_TOPIC_HOP_CAP * +/// GRAPH_WALK_MAX_DEPTH` (a single overall `LIMIT`, not a per-level cap), +/// and caps the final result at [`GRAPH_SERVICE_TOPIC_ENTITY_CAP`] entities. +/// +/// Returns `(entities, truncated)`: `truncated` is `true` when the walk +/// actually reached more than [`GRAPH_SERVICE_TOPIC_ENTITY_CAP`] distinct +/// entities, so callers can tell a silently-capped neighborhood apart from an +/// exhaustive one (mirrors [`crate::graph::graph_around_entity`]'s +/// `truncated` signal, detected the same way: fetch one row past the cap and +/// check whether it was there). +pub fn graph_walk_service_topic( + conn: &rusqlite::Connection, + start_keys: &[String], + max_depth: u8, +) -> Result<(Vec, bool)> { + if start_keys.is_empty() { + return Ok((Vec::new(), false)); + } + let depth = i64::from(max_depth.clamp(1, graph::GRAPH_WALK_MAX_DEPTH)); + let placeholders = vec!["?"; start_keys.len()].join(", "); + let rel_placeholders = vec!["?"; GRAPH_SERVICE_TOPIC_RELATIONSHIPS.len()].join(", "); + let sql = format!( + "WITH RECURSIVE graph_walk(entity_id, depth) AS ( + SELECT id, 0 FROM graph_entities WHERE canonical_key IN ({placeholders}) + UNION + SELECT CASE WHEN r.src_entity_id = gw.entity_id + THEN r.dst_entity_id ELSE r.src_entity_id END, + gw.depth + 1 + FROM graph_relationships r + JOIN graph_walk gw + ON r.src_entity_id = gw.entity_id OR r.dst_entity_id = gw.entity_id + WHERE gw.depth < ? + AND r.trust_level != 'refuted' + AND r.relationship_type IN ({rel_placeholders}) + LIMIT ? + ) + SELECT DISTINCT e.entity_type, e.canonical_key + FROM graph_entities e + JOIN graph_walk gw ON e.id = gw.entity_id + LIMIT ?" + ); + + let mut bindings: Vec = start_keys + .iter() + .map(|k| rusqlite::types::Value::Text(k.clone())) + .collect(); + bindings.push(rusqlite::types::Value::Integer(depth)); + for rel in GRAPH_SERVICE_TOPIC_RELATIONSHIPS { + bindings.push(rusqlite::types::Value::Text((*rel).to_string())); + } + bindings.push(rusqlite::types::Value::Integer( + (GRAPH_SERVICE_TOPIC_ENTITY_CAP + + GRAPH_SERVICE_TOPIC_HOP_CAP * graph::GRAPH_WALK_MAX_DEPTH as usize) as i64, + )); + // Fetch one row past the cap so we can detect truncation (see doc + // comment), then trim back down to the advertised cap below. + bindings.push(rusqlite::types::Value::Integer( + (GRAPH_SERVICE_TOPIC_ENTITY_CAP + 1) as i64, + )); + + let mut stmt = conn.prepare(&sql)?; + let mut entities = stmt + .query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + Ok(GraphWalkEntity { + entity_type: row.get(0)?, + canonical_key: row.get(1)?, + }) + })? + .collect::>>()?; + let truncated = entities.len() > GRAPH_SERVICE_TOPIC_ENTITY_CAP; + entities.truncate(GRAPH_SERVICE_TOPIC_ENTITY_CAP); + Ok((entities, truncated)) +} + +/// Rows deleted per chunk by [`cleanup_legacy_service_topology`]. Mirrors the +/// repo's chunked-deletion convention (`purge_old_logs` chunks, the storage +/// enforcement `cleanup_chunk_size` default) so the write lock is released +/// between chunks instead of held across one unbounded transaction. +const LEGACY_TOPOLOGY_CLEANUP_CHUNK: i64 = 2_000; + +/// Subquery selecting the stale pre-resolver entity ids: every `service` +/// entity (old `host:name` / `host:project:service` canonical keys) and +/// nested `app` labels shaped like `plex/plex/plex`. +const LEGACY_TOPOLOGY_ENTITY_IDS: &str = "SELECT id FROM graph_entities + WHERE entity_type = 'service' + OR (entity_type = 'app' AND canonical_key LIKE '%/%/%')"; + +/// Remove stale pre-resolver service topology rows from the graph projection: +/// every `service` entity (old `host:name` / `host:project:service` canonical +/// keys) and nested `app` labels shaped like `plex/plex/plex`, plus their +/// aliases, relationships, and evidence. The canonical replacement is the +/// resolver-owned `logical_service` / `service_instance` projection; old keys +/// are deleted, never migrated. +/// +/// Deletes run in [`LEGACY_TOPOLOGY_CLEANUP_CHUNK`]-row chunks, committing +/// and releasing [`write_lock`] between chunks so a large legacy projection +/// never pins the writer. The phase order (evidence → relationships → +/// aliases → entities) keeps every commit boundary referentially safe: a +/// child row is always gone before its parent. +pub fn cleanup_legacy_service_topology(conn: &mut rusqlite::Connection) -> Result<()> { + // Same `src IN (…) OR dst IN (…)` shape for evidence and relationships: + // an inner JOIN on both endpoints would skip relationships whose other + // endpoint dangles, orphaning their evidence. + let phases = [ + format!( + "DELETE FROM graph_relationship_evidence + WHERE id IN ( + SELECT id FROM graph_relationship_evidence + WHERE relationship_id IN ( + SELECT id FROM graph_relationships + WHERE src_entity_id IN ({LEGACY_TOPOLOGY_ENTITY_IDS}) + OR dst_entity_id IN ({LEGACY_TOPOLOGY_ENTITY_IDS}) + ) + LIMIT ?1 + )" + ), + format!( + "DELETE FROM graph_relationships + WHERE id IN ( + SELECT id FROM graph_relationships + WHERE src_entity_id IN ({LEGACY_TOPOLOGY_ENTITY_IDS}) + OR dst_entity_id IN ({LEGACY_TOPOLOGY_ENTITY_IDS}) + LIMIT ?1 + )" + ), + format!( + "DELETE FROM graph_entity_aliases + WHERE id IN ( + SELECT id FROM graph_entity_aliases + WHERE entity_id IN ({LEGACY_TOPOLOGY_ENTITY_IDS}) + LIMIT ?1 + )" + ), + format!( + "DELETE FROM graph_entities + WHERE id IN (SELECT id FROM ({LEGACY_TOPOLOGY_ENTITY_IDS}) LIMIT ?1)" + ), + ]; + for sql in &phases { + loop { + let deleted = { + let _guard = write_lock(); + let tx = conn.transaction()?; + let deleted = tx.execute(sql, [LEGACY_TOPOLOGY_CLEANUP_CHUNK])?; + tx.commit()?; + deleted + }; + if (deleted as i64) < LEGACY_TOPOLOGY_CLEANUP_CHUNK { + break; + } + } + } + Ok(()) +} + +/// Project canonical service identity from structured agent Docker metadata +/// (`metadata_json.agent_docker`) through the deterministic resolver. This is +/// the supported Docker identity source for the `logical_service` / +/// `service_instance` graph contract; central-pull `docker://` / +/// `docker-event://` rows are not resolver proof and are skipped here. +pub(crate) fn extract_agent_docker_row( + conn: &rusqlite::Connection, + row: &LogGraphRow, + meta: Option<&Value>, + memo: &mut EntityMemo, +) -> Result<()> { + let observations = agent_docker_observations_from_log_row(row, meta); + if observations.is_empty() { + return Ok(()); + } + let decisions = crate::entity_resolution::resolve_observations(&observations); + project_resolver_decisions(conn, row, &decisions, memo) +} + +/// Read `metadata_json.agent_docker` into resolver observations. Returns +/// empty when the row has no structured agent identity or is a central-pull +/// Docker row (`docker://` / `docker-event://`), which is not proof. +/// +/// `meta` is the already-parsed `metadata_json` from the dispatcher +/// (`extract_log_row`) — no re-parse here. The former "cheap prefilter" byte +/// scan for `"agent_docker"` before parsing is gone because the parse now +/// always happens exactly once upstream regardless of whether this function +/// needs it. +fn agent_docker_observations_from_log_row( + row: &LogGraphRow, + meta: Option<&Value>, +) -> Vec { + if row.source_ip.starts_with("docker://") || row.source_ip.starts_with("docker-event://") { + return Vec::new(); + } + let Some(agent) = meta + .and_then(|value| value.get("agent_docker")) + .filter(|value| value.is_object()) + else { + return Vec::new(); + }; + let text = |field: &str| { + agent + .get(field) + .and_then(Value::as_str) + .and_then(graph::normalized_value) + .map(str::to_string) + }; + let (Some(agent_host), Some(container_id), Some(container_name), Some(stream)) = ( + text("host"), + text("container_id"), + text("container_name"), + text("stream"), + ) else { + return Vec::new(); + }; + let identity = crate::entity_resolution::AgentDockerIdentity { + agent_host, + container_id, + container_name, + compose_project: text("compose_project"), + compose_service: text("compose_service"), + image: text("image"), + stream, + observed_at: row.timestamp.clone(), + }; + crate::entity_resolution::observations_from_agent_docker_identity(&identity) +} + +/// Store resolver decisions as graph entities and link each +/// `service_instance` to its `logical_service` with an `instance_of` edge. +fn project_resolver_decisions( + conn: &rusqlite::Connection, + row: &LogGraphRow, + decisions: &[crate::entity_resolution::ResolvedEntityDecision], + memo: &mut EntityMemo, +) -> Result<()> { + let source_id = row.id.to_string(); + let mut logical_ids = std::collections::BTreeMap::new(); + let mut instance_ids = std::collections::BTreeMap::new(); + for decision in decisions { + let entity_id = graph::ensure_entity_memoized( + conn, + memo, + decision.entity_type, + &decision.canonical_key, + &decision.display_label, + graph::SOURCE_KIND_LOG, + &source_id, + trust_to_graph(decision.trust), + Some(&row.timestamp), + Some(&row.timestamp), + )?; + if decision.entity_type == graph::ENTITY_TYPE_LOGICAL_SERVICE { + logical_ids.insert(decision.canonical_key.clone(), entity_id); + } else if decision.entity_type == graph::ENTITY_TYPE_SERVICE_INSTANCE { + instance_ids.insert(decision.canonical_key.clone(), entity_id); + } + } + for (instance_key, instance_id) in instance_ids { + if let Some((_, service)) = + crate::entity_resolution::split_service_instance_key(&instance_key) + && let Some(logical_id) = logical_ids.get(service) + { + graph::ensure_relationship_with_evidence( + conn, + instance_id, + *logical_id, + graph::REL_INSTANCE_OF, + graph::REASON_RESOLVER_INSTANCE_OF, + graph::TRUST_VERIFIED, + 1.0, + EvidenceInput { + evidence_key: graph::evidence_bucket_key( + "log", + row.id, + graph::REASON_RESOLVER_INSTANCE_OF, + &row.timestamp, + ), + source_kind: graph::SOURCE_KIND_LOG, + source_id: &source_id, + source_log_id: Some(row.id), + source_heartbeat_id: None, + source_signature_hash: None, + observed_at: &row.timestamp, + reason_text: Some("resolver linked service instance to logical service"), + confidence_delta: 1.0, + trust_level: graph::TRUST_VERIFIED, + safe_excerpt: Some(&instance_key), + metadata_path: Some("metadata_json.agent_docker"), + }, + )?; + } + } + Ok(()) +} + +/// Map resolver trust levels onto graph trust vocabulary. +pub(crate) fn trust_to_graph(trust: crate::entity_resolution::ResolverTrust) -> &'static str { + match trust { + crate::entity_resolution::ResolverTrust::Verified => graph::TRUST_VERIFIED, + crate::entity_resolution::ResolverTrust::Claimed => graph::TRUST_CLAIMED, + crate::entity_resolution::ResolverTrust::Inferred => graph::TRUST_INFERRED, + } +} + +#[cfg(test)] +#[path = "graph_resolver_projection_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/graph_resolver_projection_tests.rs b/crates/shared/cortex/storage-sqlite/src/graph_resolver_projection_tests.rs new file mode 100644 index 00000000..93b4a003 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/graph_resolver_projection_tests.rs @@ -0,0 +1,190 @@ +use super::*; +use crate::config::StorageConfig; +use crate::{DbPool, graph, init_pool}; + +fn test_pool(name: &str) -> (tempfile::TempDir, DbPool) { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test(dir.path().join(name))).unwrap(); + (dir, pool) +} + +fn insert_entity(conn: &rusqlite::Connection, entity_type: &str, key: &str) -> i64 { + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, trust_level) + VALUES (?1, ?2, ?2, 'verified')", + rusqlite::params![entity_type, key], + ) + .unwrap(); + conn.last_insert_rowid() +} + +fn insert_rel(conn: &rusqlite::Connection, src: i64, dst: i64, rel: &str) { + conn.execute( + "INSERT INTO graph_relationships + (relationship_key, src_entity_id, dst_entity_id, relationship_type, + reason_code, trust_level, confidence, last_seen_at) + VALUES (?1, ?2, ?3, ?4, 'log_app_name', 'inferred', 0.5, + '2026-01-01T00:00:00Z')", + rusqlite::params![format!("{src}:{rel}:{dst}"), src, dst, rel], + ) + .unwrap(); +} + +fn keys(entities: &[graph::GraphWalkEntity]) -> Vec { + let mut k: Vec = entities.iter().map(|e| e.canonical_key.clone()).collect(); + k.sort(); + k +} + +#[test] +fn graph_walk_service_topic_traverses_proof_edges_and_caps_results() { + let (_dir, pool) = test_pool("service-topic-walk.db"); + let conn = pool.get().unwrap(); + let logical = insert_entity(&conn, "logical_service", "plex"); + let instance = insert_entity(&conn, "service_instance", "nashost/plex"); + let host = insert_entity(&conn, "host", "nashost"); + let app = insert_entity(&conn, "app", "kernel"); + insert_rel(&conn, instance, logical, "instance_of"); + insert_rel(&conn, instance, host, "runs_on"); + // Broad log-identity edge from the host: must NOT be traversed. + insert_rel(&conn, app, host, "emitted_by"); + + let (entities, truncated) = graph_walk_service_topic(&conn, &["plex".to_string()], 3).unwrap(); + let walked = keys(&entities); + assert!(walked.contains(&"plex".to_string())); + assert!(walked.contains(&"nashost/plex".to_string())); + assert!(walked.contains(&"nashost".to_string())); + assert!( + !walked.contains(&"kernel".to_string()), + "service-topic walk must not traverse broad log-identity edges: {walked:?}" + ); + // Exact reach: seed + instance + host, and nothing else. (A `<= CAP` + // assertion would be vacuous on a 4-node fixture.) + assert_eq!(entities.len(), 3, "walk must reach exactly {walked:?}"); + assert!(!truncated, "small fixture must not report truncation"); +} + +#[test] +fn graph_walk_service_topic_reports_truncated_when_cap_hit() { + let (_dir, pool) = test_pool("service-topic-walk-cap.db"); + let conn = pool.get().unwrap(); + let seed = insert_entity(&conn, "logical_service", "plex"); + // One more neighbor than GRAPH_SERVICE_TOPIC_ENTITY_CAP so the walk + // (seed + neighbors) exceeds the cap and must report truncation. + // Batched in one transaction — one autocommit per row makes this ~1000x + // slower for no benefit in a test fixture. + conn.execute_batch("BEGIN;").unwrap(); + for i in 0..(GRAPH_SERVICE_TOPIC_ENTITY_CAP + 1) { + let neighbor = insert_entity(&conn, "host", &format!("host-{i}")); + insert_rel(&conn, seed, neighbor, "runs_on"); + } + conn.execute_batch("COMMIT;").unwrap(); + + let (entities, truncated) = graph_walk_service_topic(&conn, &["plex".to_string()], 1).unwrap(); + assert_eq!( + entities.len(), + GRAPH_SERVICE_TOPIC_ENTITY_CAP, + "result must be capped at GRAPH_SERVICE_TOPIC_ENTITY_CAP" + ); + assert!( + truncated, + "walk reaching more than the cap must report truncated=true" + ); +} + +#[test] +fn stale_service_topology_cleanup_removes_old_canonical_rows() { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test( + dir.path().join("stale-service-cleanup.db"), + )) + .unwrap(); + let mut conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, source_kind, source_id, trust_level) + VALUES + ('service', 'nashost:plex', 'plex', 'log', 'fixture', 'inferred'), + ('service', 'nashost:plex:plex', 'nashost/plex/plex', 'log', 'fixture', 'inferred'), + ('app', 'plex/plex/plex', 'plex/plex/plex', 'log', 'fixture', 'claimed')", + [], + ) + .unwrap(); + // Seed more legacy rows than one cleanup chunk (2000) so the chunked + // delete loop must iterate, plus dependent alias/relationship/evidence + // rows to exercise every phase, and one unrelated host that must survive. + { + let tx = conn.transaction().unwrap(); + for i in 0..2_500 { + tx.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, source_kind, source_id, trust_level) + VALUES ('service', ?1, 'svc', 'log', 'fixture', 'inferred')", + [format!("host{i}:svc{i}")], + ) + .unwrap(); + } + tx.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, source_kind, source_id, trust_level) + VALUES ('host', 'nashost', 'nashost', 'log', 'fixture', 'verified')", + [], + ) + .unwrap(); + tx.execute_batch( + "INSERT INTO graph_entity_aliases + (entity_id, alias_type, alias_key, alias_value, source_kind, trust_level) + SELECT id, 'app_name', canonical_key, canonical_key, 'log', 'inferred' + FROM graph_entities WHERE entity_type = 'service'; + INSERT INTO graph_relationships + (relationship_key, src_entity_id, dst_entity_id, relationship_type, + reason_code, trust_level, confidence) + SELECT s.id || ':runs_on:' || h.id, s.id, h.id, 'runs_on', + 'docker_service_label', 'inferred', 0.5 + FROM graph_entities s, graph_entities h + WHERE s.entity_type = 'service' AND h.canonical_key = 'nashost'; + INSERT INTO graph_relationship_evidence + (relationship_id, evidence_key, source_kind, source_id, + observed_at, reason_code, trust_level) + SELECT id, 'ev:' || id, 'log', 'fixture', + '2026-01-01T00:00:00Z', 'docker_service_label', 'inferred' + FROM graph_relationships;", + ) + .unwrap(); + tx.commit().unwrap(); + } + cleanup_legacy_service_topology(&mut conn).unwrap(); + let count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM graph_entities + WHERE entity_type = 'service' + OR (entity_type = 'app' AND canonical_key = 'plex/plex/plex')", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(count, 0); + // Dependent rows are fully gone across every chunked phase. + for table in [ + "graph_entity_aliases", + "graph_relationships", + "graph_relationship_evidence", + ] { + let n: i64 = conn + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(n, 0, "{table} must be emptied by cleanup"); + } + // The unrelated host entity survives the cleanup. + let hosts: i64 = conn + .query_row( + "SELECT COUNT(*) FROM graph_entities WHERE canonical_key = 'nashost'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(hosts, 1); +} diff --git a/crates/shared/cortex/storage-sqlite/src/graph_tests.rs b/crates/shared/cortex/storage-sqlite/src/graph_tests.rs new file mode 100644 index 00000000..b73df1c7 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/graph_tests.rs @@ -0,0 +1,1673 @@ +use super::*; +use crate::config::StorageConfig; +use crate::{LogBatchEntry, init_pool, insert_logs_batch}; + +fn test_storage_config(db_path: std::path::PathBuf) -> StorageConfig { + StorageConfig::for_test(db_path) +} + +fn make_entry(ts: &str, host: &str, app: Option<&str>, msg: &str) -> LogBatchEntry { + LogBatchEntry { + timestamp: ts.to_string(), + hostname: host.to_string(), + facility: None, + severity: "info".to_string(), + app_name: app.map(str::to_string), + process_id: None, + message: msg.to_string(), + raw: msg.to_string(), + source_ip: "10.0.0.1:514".to_string(), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + } +} + +fn count(conn: &rusqlite::Connection, sql: &str) -> i64 { + conn.query_row(sql, [], |row| row.get(0)).unwrap() +} + +#[test] +fn refresh_graph_projection_builds_syslog_app_edges_and_is_idempotent() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("graph-rebuild.db")); + let pool = init_pool(&config).unwrap(); + + insert_logs_batch( + &pool, + &[ + make_entry( + "2026-01-01T00:00:00Z", + "Claimed-Host", + Some("sshd"), + "accepted publickey", + ), + make_entry( + "2026-01-01T00:12:00Z", + "claimed-host", + Some("sshd"), + "session opened", + ), + ], + ) + .unwrap(); + + let first = match refresh_graph_projection(&pool).unwrap() { + GraphRebuildOutcome::Rebuilt(stats) => stats, + GraphRebuildOutcome::AlreadyRunning => panic!("unexpected single-flight skip"), + }; + assert_eq!(first.source_row_count, 2); + assert_eq!(first.chunk_count, 1); + assert!(first.entity_count >= 3); + assert!(first.relationship_count >= 2); + + let conn = pool.get().unwrap(); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'source_ip'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'host'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'app'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_relationships WHERE relationship_type = 'observed_as'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT SUM(evidence_count) FROM graph_relationship_evidence + WHERE reason_code = 'syslog_claimed_hostname'" + ), + 2 + ); + drop(conn); + + let second = match refresh_graph_projection(&pool).unwrap() { + GraphRebuildOutcome::Rebuilt(stats) => stats, + GraphRebuildOutcome::AlreadyRunning => panic!("unexpected single-flight skip"), + }; + assert_eq!(first.entity_count, second.entity_count); + assert_eq!(first.relationship_count, second.relationship_count); + assert_eq!(first.evidence_count, second.evidence_count); +} + +#[test] +fn graph_evidence_by_id_returns_relationship_entities_and_source_log_summary() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("graph-evidence-lookup.db")); + let pool = init_pool(&config).unwrap(); + + insert_logs_batch( + &pool, + &[make_entry( + "2026-01-01T00:00:00Z", + "proof-host", + Some("sshd"), + "proof row", + )], + ) + .unwrap(); + refresh_graph_projection(&pool).unwrap(); + + let conn = pool.get().unwrap(); + let evidence_id: i64 = conn + .query_row( + "SELECT id FROM graph_relationship_evidence + WHERE source_log_id IS NOT NULL + ORDER BY id LIMIT 1", + [], + |row| row.get(0), + ) + .unwrap(); + drop(conn); + + let rows = graph_evidence_by_id(&pool, evidence_id).unwrap().unwrap(); + assert_eq!(rows.evidence.id, evidence_id); + assert_eq!(rows.evidence.relationship_id, rows.relationship.id); + assert!(rows.evidence.source_log_id.is_some()); + assert!(rows.source_log_summary.is_some()); + assert_eq!( + rows.source_log_summary.as_ref().unwrap().message, + "proof row" + ); + assert_eq!(rows.src_entity.id, rows.relationship.src_entity_id); + assert_eq!(rows.dst_entity.id, rows.relationship.dst_entity_id); +} + +#[test] +fn refresh_graph_projection_extracts_docker_from_metadata_and_source() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("graph-docker.db")); + let pool = init_pool(&config).unwrap(); + + let mut metadata_row = make_entry( + "2026-01-01T00:00:00Z", + "docker-host", + Some("cortex"), + "container log", + ); + metadata_row.source_ip = "docker://devhost/abcdef/stdout".to_string(); + metadata_row.metadata_json = Some( + r#"{"docker_host":"devhost","container_id":"abcdef","container_name":"cortex","compose_project":"infra","compose_service":"cortex"}"#.to_string(), + ); + let mut malformed_row = make_entry( + "2026-01-01T00:01:00Z", + "docker-host", + Some("other"), + "container log", + ); + malformed_row.source_ip = "docker://devhost/bad-json/stderr".to_string(); + malformed_row.metadata_json = Some("{not-json".to_string()); + + insert_logs_batch(&pool, &[metadata_row, malformed_row]).unwrap(); + refresh_graph_projection(&pool).unwrap(); + + let conn = pool.get().unwrap(); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'container'" + ), + 2 + ); + // Hard break: no legacy `service` topology from central-pull rows. + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'service'" + ), + 0 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_relationships WHERE reason_code = 'docker_container_id'" + ), + 2 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_relationships WHERE reason_code = 'docker_service_label'" + ), + 0 + ); +} + +#[test] +fn refresh_graph_projection_extracts_ai_heartbeat_and_signature_sources() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("graph-sources.db")); + let pool = init_pool(&config).unwrap(); + + let mut ai = make_entry( + "2026-01-01T00:00:00Z", + "agent-host", + Some("codex"), + "worked on cortex", + ); + ai.ai_tool = Some("codex".to_string()); + ai.ai_project = Some("cortex".to_string()); + ai.ai_session_id = Some("sess-1".to_string()); + insert_logs_batch(&pool, &[ai]).unwrap(); + + { + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO host_heartbeats_latest + (host_id, heartbeat_id, hostname, sampled_at, received_at, + partial, agent_version, os, architecture, metadata_json) + VALUES ('host-1', 42, 'agent-host', '2026-01-01T00:00:00Z', + '2026-01-01T00:00:01Z', 0, '1.0.0', 'linux', 'x86_64', NULL)", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO error_signatures + (signature_hash, normalizer_version, template, sample_message, + sample_hostname, sample_app_name, severity, first_seen_at, + last_seen_at, total_count) + VALUES ('abc123', 1, 'error ', 'error 1', 'agent-host', + 'codex', 'err', '2026-01-01T00:00:00Z', + '2026-01-01T00:05:00Z', 3)", + [], + ) + .unwrap(); + } + + refresh_graph_projection(&pool).unwrap(); + + let conn = pool.get().unwrap(); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'ai_project'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'ai_session'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'error_signature'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entity_aliases WHERE alias_type = 'heartbeat_host_id'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_relationships WHERE relationship_type = 'worked_on'" + ), + 1 + ); + assert!( + count( + &conn, + "SELECT COUNT(*) FROM graph_relationships WHERE relationship_type = 'matches_signature'" + ) >= 1 + ); +} + +#[test] +fn alias_lookup_deduplicates_multiple_sources_for_the_same_entity() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("graph-alias-dedup.db")); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, source_kind, source_id, + trust_level, first_seen_at, last_seen_at) + VALUES ('host', 'devhost', 'devhost', 'heartbeat', '42', 'verified', + '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z')", + [], + ) + .unwrap(); + let entity_id = conn.last_insert_rowid(); + for source_kind in ["heartbeat", "source_inventory"] { + conn.execute( + "INSERT INTO graph_entity_aliases + (entity_id, alias_type, alias_key, alias_value, source_kind, + trust_level, first_seen_at, last_seen_at) + VALUES (?1, 'hostname', 'devhost', 'devhost', ?2, 'verified', + '2026-01-01T00:00:00Z', '2026-01-01T00:00:00Z')", + rusqlite::params![entity_id, source_kind], + ) + .unwrap(); + } + drop(conn); + + let candidates = find_graph_entities_by_alias(&pool, "hostname", "devhost", 20).unwrap(); + assert_eq!(candidates.len(), 1); + assert_eq!(candidates[0].entity.id, entity_id); +} + +#[test] +fn refresh_graph_projection_removes_deleted_source_log_evidence() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("graph-ghost.db")); + let pool = init_pool(&config).unwrap(); + + insert_logs_batch( + &pool, + &[make_entry( + "2026-01-01T00:00:00Z", + "ghost-host", + Some("sshd"), + "temporary row", + )], + ) + .unwrap(); + refresh_graph_projection(&pool).unwrap(); + { + let conn = pool.get().unwrap(); + assert!(count(&conn, "SELECT COUNT(*) FROM graph_relationship_evidence") > 0); + conn.execute("DELETE FROM logs", []).unwrap(); + } + + refresh_graph_projection(&pool).unwrap(); + let conn = pool.get().unwrap(); + assert_eq!( + count(&conn, "SELECT COUNT(*) FROM graph_relationship_evidence"), + 0 + ); + assert_eq!(count(&conn, "SELECT COUNT(*) FROM graph_relationships"), 0); + assert_eq!(count(&conn, "SELECT COUNT(*) FROM graph_entities"), 0); +} + +#[test] +fn refresh_graph_projection_reports_status_failures_and_single_flight() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("graph-status.db")); + let pool = init_pool(&config).unwrap(); + + let held = GRAPH_REBUILD_LOCK.try_lock().unwrap(); + assert_eq!( + refresh_graph_projection(&pool).unwrap(), + GraphRebuildOutcome::AlreadyRunning + ); + drop(held); + + { + let conn = pool.get().unwrap(); + conn.execute("DROP TABLE graph_relationships", []).unwrap(); + } + let err = refresh_graph_projection(&pool).unwrap_err(); + assert!(err.to_string().contains("graph_relationships")); + let status = graph_projection_status(&pool).unwrap(); + assert_eq!(status.projection_status, "failed"); + assert!(status.is_degraded); + assert!(status.last_error.is_some()); +} + +#[test] +fn parse_log_watermark_extracts_log_cursor() { + assert_eq!( + parse_log_watermark("logs:42;heartbeats:3;signatures:7"), + Some(42) + ); + // Order-independent. + assert_eq!( + parse_log_watermark("heartbeats:3;logs:99;signatures:7"), + Some(99) + ); + assert_eq!(parse_log_watermark("logs:0"), Some(0)); + assert_eq!(parse_log_watermark(""), None); + assert_eq!(parse_log_watermark("heartbeats:3;signatures:7"), None); + assert_eq!(parse_log_watermark("logs:notanumber"), None); +} + +#[test] +fn incremental_projection_falls_back_to_full_build_when_unbuilt() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&test_storage_config( + dir.path().join("graph-inc-fallback.db"), + )) + .unwrap(); + + insert_logs_batch( + &pool, + &[make_entry( + "2026-01-01T00:00:00Z", + "host-a", + Some("sshd"), + "accepted publickey", + )], + ) + .unwrap(); + + // No prior projection: incremental must perform a full build. + let stats = match refresh_graph_projection_incremental(&pool).unwrap() { + GraphRebuildOutcome::Rebuilt(stats) => stats, + GraphRebuildOutcome::AlreadyRunning => panic!("unexpected single-flight skip"), + }; + assert!(stats.entity_count >= 3); + let status = graph_projection_status(&pool).unwrap(); + assert_eq!(status.projection_status, "ready"); + assert!(!status.is_degraded); +} + +/// The gold-standard correctness check: a full build followed by an incremental +/// delta must yield the same graph (entity/relationship/evidence counts and +/// accumulated evidence totals) as a single full rebuild over all the same logs. +#[test] +fn incremental_projection_matches_full_rebuild() { + let _guard = GRAPH_TEST_LOCK.lock(); + + let batch1 = || { + vec![ + make_entry("2026-01-01T00:00:00Z", "host-a", Some("sshd"), "login a"), + make_entry("2026-01-01T00:05:00Z", "host-a", Some("sshd"), "login b"), + ] + }; + // batch2 reuses host-a/sshd (accumulates evidence on existing edges) and + // introduces host-b/nginx (new entities + edges discovered incrementally). + let batch2 = || { + vec![ + make_entry("2026-01-01T01:00:00Z", "host-a", Some("sshd"), "login c"), + make_entry("2026-01-01T01:05:00Z", "host-b", Some("nginx"), "GET /"), + ] + }; + + // DB A: full build over batch1, then an incremental pass over batch2. + let dir_a = tempfile::tempdir().unwrap(); + let pool_a = init_pool(&test_storage_config(dir_a.path().join("graph-inc-a.db"))).unwrap(); + insert_logs_batch(&pool_a, &batch1()).unwrap(); + refresh_graph_projection(&pool_a).unwrap(); + insert_logs_batch(&pool_a, &batch2()).unwrap(); + let incremental = match refresh_graph_projection_incremental(&pool_a).unwrap() { + GraphRebuildOutcome::Rebuilt(stats) => stats, + GraphRebuildOutcome::AlreadyRunning => panic!("unexpected single-flight skip"), + }; + + // DB B: a single full rebuild over batch1 + batch2. + let dir_b = tempfile::tempdir().unwrap(); + let pool_b = init_pool(&test_storage_config(dir_b.path().join("graph-inc-b.db"))).unwrap(); + insert_logs_batch(&pool_b, &batch1()).unwrap(); + insert_logs_batch(&pool_b, &batch2()).unwrap(); + let full = match refresh_graph_projection(&pool_b).unwrap() { + GraphRebuildOutcome::Rebuilt(stats) => stats, + GraphRebuildOutcome::AlreadyRunning => panic!("unexpected single-flight skip"), + }; + + assert_eq!( + incremental.entity_count, full.entity_count, + "entity count must match a full rebuild" + ); + assert_eq!( + incremental.relationship_count, full.relationship_count, + "relationship count must match a full rebuild" + ); + assert_eq!( + incremental.evidence_count, full.evidence_count, + "evidence row count must match a full rebuild" + ); + + let conn_a = pool_a.get().unwrap(); + let conn_b = pool_b.get().unwrap(); + // Accumulated evidence totals must match (guards against double-counting or + // dropped evidence in the incremental merge). + let ev_sum = "SELECT COALESCE(SUM(evidence_count), 0) FROM graph_relationship_evidence"; + assert_eq!( + count(&conn_a, ev_sum), + count(&conn_b, ev_sum), + "summed evidence_count must match a full rebuild" + ); + let rel_ev_sum = "SELECT COALESCE(SUM(evidence_count), 0) FROM graph_relationships"; + assert_eq!( + count(&conn_a, rel_ev_sum), + count(&conn_b, rel_ev_sum), + "relationship evidence_count rollups must match a full rebuild" + ); + // host-b only appears in batch2, so the incremental pass must have created it. + assert_eq!( + count( + &conn_a, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type='host' AND canonical_key='host-b'" + ), + 1, + "incremental pass must create entities first seen in the delta" + ); +} + +/// A second incremental pass with no new logs must be a no-op for counts (the +/// bounded snapshot re-projection stays idempotent). +#[test] +fn incremental_projection_is_idempotent_without_new_logs() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&test_storage_config(dir.path().join("graph-inc-idem.db"))).unwrap(); + insert_logs_batch( + &pool, + &[ + make_entry("2026-01-01T00:00:00Z", "host-a", Some("sshd"), "login a"), + make_entry("2026-01-01T00:05:00Z", "host-a", Some("sshd"), "login b"), + ], + ) + .unwrap(); + refresh_graph_projection(&pool).unwrap(); + + let first = match refresh_graph_projection_incremental(&pool).unwrap() { + GraphRebuildOutcome::Rebuilt(stats) => stats, + GraphRebuildOutcome::AlreadyRunning => panic!("unexpected single-flight skip"), + }; + let second = match refresh_graph_projection_incremental(&pool).unwrap() { + GraphRebuildOutcome::Rebuilt(stats) => stats, + GraphRebuildOutcome::AlreadyRunning => panic!("unexpected single-flight skip"), + }; + assert_eq!(first.entity_count, second.entity_count); + assert_eq!(first.relationship_count, second.relationship_count); + assert_eq!(first.evidence_count, second.evidence_count); +} + +/// Downgrade → re-upgrade drift: a legacy `service` row present in a ready +/// projection must trigger cleanup + a full rebuild instead of an +/// incremental merge on top of a mixed-contract graph. +#[test] +fn incremental_projection_purges_legacy_service_rows_and_forces_full_rebuild() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&test_storage_config(dir.path().join("graph-inc-drift.db"))).unwrap(); + insert_logs_batch( + &pool, + &[make_entry( + "2026-01-01T00:00:00Z", + "host-a", + Some("sshd"), + "login a", + )], + ) + .unwrap(); + refresh_graph_projection(&pool).unwrap(); + + // Simulate a pre-resolver binary projecting legacy service topology into + // the ready projection (the CHECK constraint still allows 'service'). + { + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, trust_level) + VALUES ('service', 'host-a:plex', 'host-a:plex', 'inferred')", + [], + ) + .unwrap(); + } + + let outcome = refresh_graph_projection_incremental(&pool).unwrap(); + assert!(matches!(outcome, GraphRebuildOutcome::Rebuilt(_))); + let conn = pool.get().unwrap(); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'service'" + ), + 0, + "legacy service rows must be purged by the drift probe" + ); + drop(conn); + let status = graph_projection_status(&pool).unwrap(); + assert_eq!(status.projection_status, "ready"); +} + +/// Build a log entry shaped like `command_log::agent_record_to_entry` output: +/// `agent-command://` source_ip, agent in ai_tool/app_name, raw cwd in +/// ai_project, session id in ai_session_id. +fn make_agent_command_entry( + ts: &str, + host: &str, + agent: &str, + session: &str, + cwd: &str, +) -> LogBatchEntry { + let mut entry = make_entry(ts, host, Some(agent), "cargo test"); + entry.source_ip = format!("agent-command://{host}/{agent}/{session}"); + entry.ai_tool = Some(agent.to_string()); + entry.ai_project = Some(cwd.to_string()); + entry.ai_session_id = Some(session.to_string()); + entry.metadata_json = Some(format!( + r#"{{"source_kind":"agent-command","agent_command":{{"cwd":"{cwd}","session_id":"{session}"}}}}"# + )); + entry +} + +#[test] +fn infer_project_from_cwd_prefers_workspace_segment() { + assert_eq!( + infer_project_from_cwd("/home/jmagar/workspace/cortex"), + Some("cortex".to_string()) + ); + // Deep worktree path still resolves to the repo under workspace/. + assert_eq!( + infer_project_from_cwd("/home/jmagar/workspace/cortex/.claude/worktrees/foo"), + Some("cortex".to_string()) + ); + // No workspace component → final segment. + assert_eq!( + infer_project_from_cwd("/srv/projects/axon/"), + Some("axon".to_string()) + ); + assert_eq!(infer_project_from_cwd("/"), None); + assert_eq!(infer_project_from_cwd(""), None); +} + +#[test] +fn agent_command_row_creates_verified_session_host_and_inferred_project_edges() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&test_storage_config(dir.path().join("graph-agent-cmd.db"))).unwrap(); + + insert_logs_batch( + &pool, + &[make_agent_command_entry( + "2026-01-01T00:00:00Z", + "devhost", + "claude", + "sess-7", + "/home/jmagar/workspace/cortex", + )], + ) + .unwrap(); + refresh_graph_projection(&pool).unwrap(); + + let conn = pool.get().unwrap(); + + // Exactly one ai_session, keyed by the INFERRED project (not the raw cwd path). + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'ai_session'" + ), + 1 + ); + let session_key: String = conn + .query_row( + "SELECT canonical_key FROM graph_entities WHERE entity_type = 'ai_session'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(session_key, "cortex:claude:sess-7"); + + // Verified session→host edge (agent_command_session, 0.95). + let (trust, conf): (String, f64) = conn + .query_row( + "SELECT trust_level, confidence FROM graph_relationships + WHERE relationship_type = 'worked_on' AND reason_code = 'agent_command_session'", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(trust, "verified"); + assert!((conf - 0.95).abs() < 1e-9, "confidence was {conf}"); + + // Inferred session→project edge (agent_command_cwd_infer, 0.7) to ai_project:cortex. + let project_key: String = conn + .query_row( + "SELECT e.canonical_key FROM graph_relationships r + JOIN graph_entities e ON e.id = r.dst_entity_id + WHERE r.reason_code = 'agent_command_cwd_infer'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(project_key, "cortex"); +} + +#[test] +fn agent_command_session_converges_with_transcript_session_entity() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&test_storage_config(dir.path().join("graph-converge.db"))).unwrap(); + + // Transcript event: clean project "cortex", same tool + session id. + let mut transcript = make_entry( + "2026-01-01T00:00:00Z", + "devhost", + Some("claude"), + "thinking", + ); + transcript.ai_tool = Some("claude".to_string()); + transcript.ai_project = Some("cortex".to_string()); + transcript.ai_session_id = Some("sess-9".to_string()); + transcript.source_ip = "agent://devhost".to_string(); + + // Agent-command row: raw cwd, same session id → must converge on one entity. + let cmd = make_agent_command_entry( + "2026-01-01T00:01:00Z", + "devhost", + "claude", + "sess-9", + "/home/jmagar/workspace/cortex", + ); + + insert_logs_batch(&pool, &[transcript, cmd]).unwrap(); + refresh_graph_projection(&pool).unwrap(); + + let conn = pool.get().unwrap(); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'ai_session'" + ), + 1, + "transcript and agent-command rows for the same session must share one ai_session entity" + ); +} + +#[test] +fn agent_command_incremental_rebuild_adds_no_duplicate_edges() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&test_storage_config(dir.path().join("graph-agent-inc.db"))).unwrap(); + + insert_logs_batch( + &pool, + &[make_agent_command_entry( + "2026-01-01T00:00:00Z", + "devhost", + "claude", + "sess-3", + "/home/jmagar/workspace/cortex", + )], + ) + .unwrap(); + refresh_graph_projection(&pool).unwrap(); + let before: i64 = count( + &pool.get().unwrap(), + "SELECT COUNT(*) FROM graph_relationships WHERE reason_code LIKE 'agent_command%'", + ); + + // Second command in the same session → incremental rebuild, no new edges. + insert_logs_batch( + &pool, + &[make_agent_command_entry( + "2026-01-01T00:02:00Z", + "devhost", + "claude", + "sess-3", + "/home/jmagar/workspace/cortex", + )], + ) + .unwrap(); + refresh_graph_projection_incremental(&pool).unwrap(); + let after: i64 = count( + &pool.get().unwrap(), + "SELECT COUNT(*) FROM graph_relationships WHERE reason_code LIKE 'agent_command%'", + ); + + assert_eq!( + before, after, + "incremental rebuild must not duplicate agent-command edges" + ); + assert_eq!( + before, 2, + "one verified host edge + one inferred project edge" + ); +} + +fn make_git_commit_agent_entry(ts: &str, host: &str, session: &str, cwd: &str) -> LogBatchEntry { + let mut entry = make_agent_command_entry(ts, host, "claude", session, cwd); + entry.message = "git commit -m \"feat: add thing\"".to_string(); + entry.raw = entry.message.clone(); + entry +} + +#[test] +fn is_git_commit_command_matches_commit_and_push() { + assert!(is_git_commit_command("git commit -m x")); + assert!(is_git_commit_command("cd /tmp && GIT COMMIT")); + assert!(is_git_commit_command("git push origin main")); + assert!(!is_git_commit_command("cargo build")); + assert!(!is_git_commit_command("git status")); +} + +#[test] +fn agent_command_git_commit_creates_commit_session_and_project_edges() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&test_storage_config(dir.path().join("graph-gitcommit.db"))).unwrap(); + + insert_logs_batch( + &pool, + &[make_git_commit_agent_entry( + "2026-01-01T00:00:00Z", + "devhost", + "sess-7", + "/home/jmagar/workspace/cortex", + )], + ) + .unwrap(); + refresh_graph_projection(&pool).unwrap(); + + let conn = pool.get().unwrap(); + // One git_commit entity, keyed by inferred project + timestamp. + let commit_key: String = conn + .query_row( + "SELECT canonical_key FROM graph_entities WHERE entity_type = 'git_commit'", + [], + |row| row.get(0), + ) + .unwrap(); + assert!( + commit_key.starts_with("cortex:"), + "commit key keyed by inferred project: {commit_key}" + ); + + // session worked_on commit (inferred, 0.8). + let (trust, conf): (String, f64) = conn + .query_row( + "SELECT trust_level, confidence FROM graph_relationships + WHERE reason_code = 'agent_command_git_commit' + AND relationship_type = 'worked_on'", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(trust, "inferred"); + assert!((conf - 0.8).abs() < 1e-9, "confidence was {conf}"); + + // commit has_artifact project (inferred, 0.9). + let project_key: String = conn + .query_row( + "SELECT e.canonical_key FROM graph_relationships r + JOIN graph_entities e ON e.id = r.dst_entity_id + WHERE r.reason_code = 'agent_command_git_commit' + AND r.relationship_type = 'has_artifact'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(project_key, "cortex"); +} + +#[test] +fn non_git_command_creates_no_commit_entity() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&test_storage_config(dir.path().join("graph-nogit.db"))).unwrap(); + + // make_agent_command_entry's message is "cargo test" — not a git commit. + insert_logs_batch( + &pool, + &[make_agent_command_entry( + "2026-01-01T00:00:00Z", + "devhost", + "claude", + "sess-7", + "/home/jmagar/workspace/cortex", + )], + ) + .unwrap(); + refresh_graph_projection(&pool).unwrap(); + + let conn = pool.get().unwrap(); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'git_commit'" + ), + 0 + ); +} + +#[test] +fn shell_history_git_commit_links_commit_to_host() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&test_storage_config(dir.path().join("graph-shellgit.db"))).unwrap(); + + let mut entry = make_entry( + "2026-01-01T00:00:00Z", + "devhost", + Some("zsh"), + "git commit -am wip", + ); + entry.source_ip = "shell-history://devhost/jacob/zsh".to_string(); + entry.metadata_json = Some(r#"{"source_kind":"shell-history"}"#.to_string()); + insert_logs_batch(&pool, &[entry]).unwrap(); + refresh_graph_projection(&pool).unwrap(); + + let conn = pool.get().unwrap(); + let commit_key: String = conn + .query_row( + "SELECT canonical_key FROM graph_entities WHERE entity_type = 'git_commit'", + [], + |row| row.get(0), + ) + .unwrap(); + assert!( + commit_key.starts_with("devhost:"), + "shell commit keyed by host: {commit_key}" + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_relationships WHERE reason_code = 'shell_history_git_commit' AND relationship_type = 'emitted_by'" + ), + 1 + ); +} + +#[test] +fn docker_compose_label_no_longer_projects_legacy_service_topology() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&test_storage_config(dir.path().join("graph-compose.db"))).unwrap(); + + let mut row = make_entry( + "2026-01-01T00:00:00Z", + "devhost", + Some("axon-qdrant"), + "started", + ); + row.source_ip = "docker-event://devhost/axon-qdrant".to_string(); + row.metadata_json = Some( + r#"{"source_kind":"docker-event","compose_project":"axon","compose_service":"qdrant"}"# + .to_string(), + ); + insert_logs_batch(&pool, &[row]).unwrap(); + refresh_graph_projection(&pool).unwrap(); + + let conn = pool.get().unwrap(); + // Hard break: central-pull docker labels no longer synthesize legacy + // `service` topology (`devhost:axon:qdrant`) or compose_project edges to + // it. Canonical service identity requires agent-docker structured + // metadata or verified inventory (resolver decisions). + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'service'" + ), + 0 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'compose_project'" + ), + 0 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_relationships + WHERE reason_code IN ('compose_config', 'docker_service_label')" + ), + 0 + ); + // Verified host/container identity is still projected. + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_relationships WHERE reason_code = 'docker_container_id'" + ), + 1 + ); +} + +#[test] +fn reason_code_namespace_maps_to_hierarchical_v2() { + assert_eq!( + reason_code_namespace(REASON_DOCKER_CONTAINER_ID), + "source:docker:container_id" + ); + assert_eq!( + reason_code_namespace(REASON_AI_SESSION_PROJECT), + "derivation:ai:session_project" + ); + assert_eq!(reason_code_family(REASON_DOCKER_CONTAINER_ID), "source"); + assert_eq!(reason_code_family(REASON_AI_SESSION_PROJECT), "derivation"); + // Every registered reason code has a namespace (no unknown fallthrough). + for code in REASON_CODES { + assert_ne!( + reason_code_namespace(code), + "unknown:unknown:unknown", + "reason code {code} lacks a v2 namespace" + ); + } +} + +#[test] +fn graph_around_entity_excludes_refuted_edges() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&test_storage_config(dir.path().join("graph-refuted.db"))).unwrap(); + let conn = pool.get().unwrap(); + + let insert_entity = |etype: &str, key: &str| -> i64 { + conn.execute( + "INSERT INTO graph_entities (entity_type, canonical_key, display_label, trust_level) + VALUES (?1, ?2, ?2, 'verified')", + rusqlite::params![etype, key], + ) + .unwrap(); + conn.last_insert_rowid() + }; + let a = insert_entity(ENTITY_TYPE_HOST, "host-a"); + let b = insert_entity(ENTITY_TYPE_APP, "app-b"); + let insert_rel = |rel: &str, trust: &str| { + conn.execute( + "INSERT INTO graph_relationships + (relationship_key, src_entity_id, dst_entity_id, relationship_type, + reason_code, trust_level, confidence, last_seen_at) + VALUES (?1, ?2, ?3, ?4, 'log_app_name', ?5, 0.5, '2026-01-01T00:00:00Z')", + rusqlite::params![format!("{a}:{rel}:{b}"), a, b, rel, trust], + ) + .unwrap(); + }; + insert_rel("emitted_by", "verified"); + insert_rel("runs_on", "refuted"); + drop(conn); + + let around = graph_around_entity(&pool, a, 50, 0).unwrap(); + assert_eq!( + around.relationships.len(), + 1, + "refuted edge must be excluded" + ); + assert_eq!(around.relationships[0].relationship_type, "emitted_by"); +} + +#[test] +fn shell_history_row_creates_user_accessed_host() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&test_storage_config(dir.path().join("graph-user-shell.db"))).unwrap(); + + let mut entry = make_entry("2026-01-01T00:00:00Z", "devhost", Some("zsh"), "ls -la"); + entry.source_ip = "shell-history://devhost/jacob/zsh".to_string(); + entry.metadata_json = Some(r#"{"source_kind":"shell-history"}"#.to_string()); + insert_logs_batch(&pool, &[entry]).unwrap(); + refresh_graph_projection(&pool).unwrap(); + + let conn = pool.get().unwrap(); + let user_key: String = conn + .query_row( + "SELECT canonical_key FROM graph_entities WHERE entity_type = 'user'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(user_key, "devhost:jacob"); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_relationships WHERE reason_code = 'shell_history_user' AND relationship_type = 'accessed'" + ), + 1 + ); +} + +#[test] +fn adguard_row_creates_device_accessed_domain() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&test_storage_config(dir.path().join("graph-adguard.db"))).unwrap(); + + let mut entry = make_entry( + "2026-01-01T00:00:00Z", + "edgehost", + Some("adguard-query"), + "dns", + ); + entry.metadata_json = Some( + r#"{"source_kind":"adguard-api","client":"192.168.10.55","query":"doubleclick.net"}"# + .to_string(), + ); + insert_logs_batch(&pool, &[entry]).unwrap(); + refresh_graph_projection(&pool).unwrap(); + + let conn = pool.get().unwrap(); + let device_key: String = conn + .query_row( + "SELECT canonical_key FROM graph_entities WHERE entity_type = 'device'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(device_key, "192.168.10.55"); + let domain_key: String = conn + .query_row( + "SELECT e.canonical_key FROM graph_relationships r + JOIN graph_entities e ON e.id = r.dst_entity_id + WHERE r.reason_code = 'adguard_client_query' AND r.relationship_type = 'accessed'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(domain_key, "doubleclick.net"); +} + +#[test] +fn authelia_row_creates_user_authenticated_as_host() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&test_storage_config(dir.path().join("graph-authelia.db"))).unwrap(); + + let mut entry = make_entry( + "2026-01-01T00:00:00Z", + "edgehost", + Some("authelia"), + "auth ok", + ); + entry.metadata_json = Some(r#"{"source_kind":"syslog-udp","username":"alice"}"#.to_string()); + insert_logs_batch(&pool, &[entry]).unwrap(); + refresh_graph_projection(&pool).unwrap(); + + let conn = pool.get().unwrap(); + let user_key: String = conn + .query_row( + "SELECT canonical_key FROM graph_entities WHERE entity_type = 'user'", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(user_key, "edgehost:alice"); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_relationships WHERE reason_code = 'authelia_auth' AND relationship_type = 'authenticated_as'" + ), + 1 + ); +} + +fn rel_row(id: i64, last_seen: &str) -> GraphRelationshipRow { + GraphRelationshipRow { + id, + relationship_key: format!("k{id}"), + src_entity_id: 1, + dst_entity_id: id + 100, + relationship_type: "emitted_by".to_string(), + reason_code: "log:app_name".to_string(), + trust_level: "inferred".to_string(), + confidence: 0.5, + evidence_count: 1, + first_seen_at: Some(last_seen.to_string()), + last_seen_at: Some(last_seen.to_string()), + } +} + +#[test] +fn fair_share_gives_each_neighbor_type_a_slot() { + // 5 fresh error_signatures + 2 older apps; with a recency-only sort and a + // limit of 4, apps would be entirely crowded out. Fair-share must include + // both apps. + let candidates = vec![ + ( + "error_signature".to_string(), + rel_row(1, "2026-06-19T10:00:05Z"), + ), + ( + "error_signature".to_string(), + rel_row(2, "2026-06-19T10:00:04Z"), + ), + ( + "error_signature".to_string(), + rel_row(3, "2026-06-19T10:00:03Z"), + ), + ( + "error_signature".to_string(), + rel_row(4, "2026-06-19T10:00:02Z"), + ), + ( + "error_signature".to_string(), + rel_row(5, "2026-06-19T10:00:01Z"), + ), + ("app".to_string(), rel_row(6, "2026-06-19T09:00:00Z")), + ("app".to_string(), rel_row(7, "2026-06-19T08:00:00Z")), + ]; + let (selected, truncated) = fair_share_relationships(candidates, 4, false); + assert_eq!(selected.len(), 4); + let ids: std::collections::HashSet = selected.iter().map(|r| r.id).collect(); + // Both apps must survive (round-robin), not only error_signatures. + assert!(ids.contains(&6), "first app must be selected: {ids:?}"); + assert!(ids.contains(&7), "second app must be selected: {ids:?}"); + assert!( + truncated, + "5 error_signatures + 2 apps into limit 4 truncates" + ); +} + +#[test] +fn fair_share_not_truncated_when_everything_fits() { + let candidates = vec![ + ("app".to_string(), rel_row(1, "2026-06-19T10:00:00Z")), + ("source_ip".to_string(), rel_row(2, "2026-06-19T09:00:00Z")), + ]; + let (selected, truncated) = fair_share_relationships(candidates, 10, false); + assert_eq!(selected.len(), 2); + assert!(!truncated); + // Returned in recency order. + assert_eq!(selected[0].id, 1); + assert_eq!(selected[1].id, 2); +} + +#[test] +fn fair_share_reports_truncated_when_candidate_pool_capped() { + let candidates = vec![("app".to_string(), rel_row(1, "2026-06-19T10:00:00Z"))]; + let (selected, truncated) = fair_share_relationships(candidates, 10, true); + assert_eq!(selected.len(), 1); + assert!(truncated, "candidate-pool cap must propagate as truncated"); +} + +#[test] +fn evidence_bucket_key_distinguishes_by_source_id_within_same_hour_bucket() { + // Two evidence rows with the same prefix/reason and timestamps in the + // same hour bucket, but different source_ids, must not collapse into a + // single dedup key — each originating log/heartbeat/signature must be + // able to leave its own evidence row. + let a = evidence_bucket_key("log", 1, "log_app_name", "2026-01-01T00:00:00Z"); + let b = evidence_bucket_key("log", 2, "log_app_name", "2026-01-01T00:12:00Z"); + assert_ne!(a, b, "distinct source_ids must yield distinct bucket keys"); + + // Same source_id, same prefix/reason, same hour bucket still collapses + // (that's the intended within-source hourly dedup). + let c = evidence_bucket_key("log", 1, "log_app_name", "2026-01-01T00:45:00Z"); + assert_eq!( + a, c, + "same source_id within the same hour bucket must dedup" + ); + + // A different hour bucket for the same source_id must not collapse. + let d = evidence_bucket_key("log", 1, "log_app_name", "2026-01-01T01:00:00Z"); + assert_ne!(a, d, "different hour buckets must not dedup"); +} + +#[test] +fn graph_projection_emits_service_instance_not_nested_service_key() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&test_storage_config( + dir.path().join("resolver-graph-projection.db"), + )) + .unwrap(); + let mut entry = make_entry( + "2026-01-01T00:00:00Z", + "nashost", + Some("plex/plex/plex"), + "Plex started", + ); + entry.metadata_json = Some( + r#"{"source_kind":"agent-docker","agent_docker":{"host":"nashost","container_id":"abcdef1234567890","container_name":"plex","compose_project":"plex","compose_service":"plex","stream":"stdout"}}"# + .to_string(), + ); + insert_logs_batch(&pool, &[entry]).unwrap(); + refresh_graph_projection(&pool).unwrap(); + let conn = pool.get().unwrap(); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'logical_service' AND canonical_key = 'plex'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'service_instance' AND canonical_key = 'nashost/plex'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_relationships WHERE relationship_type = 'instance_of' AND reason_code = 'resolver_instance_of'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'service' AND canonical_key IN ('nashost:plex', 'nashost:plex:plex')" + ), + 0 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'app' AND canonical_key = 'plex/plex/plex'" + ), + 0 + ); +} + +#[test] +fn canonical_plex_proof_fixture_projects_only_resolver_identity() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&test_storage_config(dir.path().join("plex-proof.db"))).unwrap(); + + let agent_docker_meta = |host: &str| { + format!( + r#"{{"source_kind":"agent-docker","agent_docker":{{"host":"{host}","container_id":"abcdef1234567890","container_name":"plex","compose_project":"plex","compose_service":"plex","stream":"stdout"}}}}"# + ) + }; + let mut nashost_plex = make_entry( + "2026-01-01T00:00:00Z", + "nashost", + Some("plex/plex/plex"), + "Plex started", + ); + nashost_plex.metadata_json = Some(agent_docker_meta("nashost")); + let mut backuphost_plex = make_entry( + "2026-01-01T00:01:00Z", + "backuphost", + Some("plex/plex/plex"), + "Plex replica started", + ); + backuphost_plex.metadata_json = Some(agent_docker_meta("backuphost")); + // Raw syslog labels that merely contain "plex": never logical services. + let complex = make_entry( + "2026-01-01T00:02:00Z", + "nashost", + Some("complex"), + "complex event", + ); + let plex_backup = make_entry( + "2026-01-01T00:03:00Z", + "nashost", + Some("plex-backup"), + "backup ran", + ); + // AI command row whose project path mentions plex. + let mut ai_row = make_entry( + "2026-01-01T00:04:00Z", + "devhost", + Some("claude"), + "edited compose file", + ); + ai_row.source_ip = "agent-command://devhost/claude/sess-plex".to_string(); + ai_row.ai_tool = Some("claude".to_string()); + ai_row.ai_project = Some("/home/jmagar/workspace/plex-tools".to_string()); + ai_row.ai_session_id = Some("sess-plex".to_string()); + + insert_logs_batch( + &pool, + &[nashost_plex, backuphost_plex, complex, plex_backup, ai_row], + ) + .unwrap(); + refresh_graph_projection(&pool).unwrap(); + + let conn = pool.get().unwrap(); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'logical_service' AND canonical_key = 'plex'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'service_instance' AND canonical_key = 'nashost/plex'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'service_instance' AND canonical_key = 'backuphost/plex'" + ), + 1 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE canonical_key IN ('nashost:plex', 'nashost:plex:plex', 'plex/plex/plex')" + ), + 0 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE canonical_key = 'complex' AND entity_type = 'logical_service'" + ), + 0 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE canonical_key = 'plex-backup' AND entity_type = 'logical_service'" + ), + 0 + ); + // Both instances converge on ONE logical service via instance_of. + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_relationships WHERE relationship_type = 'instance_of'" + ), + 2 + ); +} + +#[test] +fn extract_log_row_parses_metadata_json_exactly_once_per_row() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&test_storage_config(dir.path().join("graph-parse-once.db"))).unwrap(); + + // Each of these rows previously drove metadata_json through TWO + // independent `parse_metadata` calls (extract_user_device_row parsed + // unconditionally for every row, plus one gated extractor re-parsed the + // same string): a docker:// row (extract_docker_log_row), an + // agent-command:// row with a cwd fallback in metadata + // (extract_agent_command_row), and a plain row carrying + // metadata_json.agent_docker (extract_agent_docker_row's resolver path). + let mut docker_row = make_entry("2026-01-01T00:00:00Z", "docker-host", Some("cortex"), "log"); + docker_row.source_ip = "docker://devhost/abcdef/stdout".to_string(); + docker_row.metadata_json = Some( + r#"{"docker_host":"devhost","container_id":"abcdef","container_name":"cortex"}"# + .to_string(), + ); + + let mut agent_command_row = make_entry( + "2026-01-01T00:01:00Z", + "devhost", + Some("claude"), + "ran a command", + ); + agent_command_row.source_ip = "agent-command://devhost/claude/sess-parse".to_string(); + agent_command_row.ai_tool = Some("claude".to_string()); + agent_command_row.ai_session_id = Some("sess-parse".to_string()); + agent_command_row.metadata_json = + Some(r#"{"agent_command":{"cwd":"/home/jmagar/workspace/cortex"}}"#.to_string()); + + let mut agent_docker_row = make_entry( + "2026-01-01T00:02:00Z", + "nashost", + Some("plex/plex/plex"), + "Plex started", + ); + agent_docker_row.metadata_json = Some( + r#"{"agent_docker":{"host":"nashost","container_id":"abcdef1234567890","container_name":"plex","stream":"stdout"}}"# + .to_string(), + ); + + // A malformed-JSON row: parse_metadata must still run exactly once for + // it (returning None), not fail partially in some extractors and + // succeed in others from independent re-parses. + let mut malformed_row = make_entry("2026-01-01T00:03:00Z", "docker-host", Some("x"), "log"); + malformed_row.source_ip = "docker://devhost/badjson/stderr".to_string(); + malformed_row.metadata_json = Some("{not-json".to_string()); + + insert_logs_batch( + &pool, + &[ + docker_row, + agent_command_row, + agent_docker_row, + malformed_row, + ], + ) + .unwrap(); + + PARSE_METADATA_CALLS.store(0, std::sync::atomic::Ordering::Relaxed); + refresh_graph_projection(&pool).unwrap(); + + assert_eq!( + PARSE_METADATA_CALLS.load(std::sync::atomic::Ordering::Relaxed), + 4, + "metadata_json must be parsed exactly once per row (4 rows in), not once per extractor that reads it" + ); + + // Correctness is preserved despite parsing once and threading the + // result down: the docker row and the resolver-identity row both still + // project their entities, and the malformed row degrades cleanly. + let conn = pool.get().unwrap(); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'container'" + ), + 2 + ); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'logical_service' AND canonical_key = 'plex'" + ), + 1 + ); +} + +#[test] +fn chunk_scoped_entity_memo_collapses_repeated_ensure_entity_upserts() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&test_storage_config( + dir.path().join("graph-entity-memo.db"), + )) + .unwrap(); + + // 50 rows all naming the same host/app/source_ip (make_entry's fixed + // source_ip), with strictly increasing timestamps, all well within one + // GRAPH_REBUILD_CHUNK_SIZE (10_000) chunk transaction. + const ROW_COUNT: usize = 50; + let entries: Vec<_> = (0..ROW_COUNT) + .map(|i| { + make_entry( + &format!("2026-01-01T00:{i:02}:00Z"), + "warden", + Some("sshd"), + "repeated event", + ) + }) + .collect(); + insert_logs_batch(&pool, &entries).unwrap(); + + ENSURE_ENTITY_CALLS.store(0, std::sync::atomic::Ordering::Relaxed); + refresh_graph_projection(&pool).unwrap(); + + // Correctness: still exactly one entity per distinct (entity_type, + // canonical_key) — source_ip, host, app — no matter how many rows + // referenced them. + let conn = pool.get().unwrap(); + assert_eq!( + count( + &conn, + "SELECT COUNT(*) FROM graph_entities WHERE entity_type IN ('source_ip', 'host', 'app')" + ), + 3 + ); + // Every row still contributes its own evidence (evidence_key includes + // the log row id), so nothing was silently dropped by memoizing the + // entity upsert. + assert_eq!( + count( + &conn, + "SELECT SUM(evidence_count) FROM graph_relationship_evidence + WHERE reason_code = 'syslog_claimed_hostname'" + ), + ROW_COUNT as i64 + ); + + // The memo collapsed the 3 distinct entities * 50 rows = 150 potential + // ensure_entity invocations down to exactly 3 real upserts (one per + // unique (entity_type, canonical_key) per chunk). + assert_eq!( + ENSURE_ENTITY_CALLS.load(std::sync::atomic::Ordering::Relaxed), + 3, + "memo must collapse repeated (entity_type, canonical_key) upserts within a chunk" + ); + + // Documented tradeoff: because rows 2..50 never reach ensure_entity at + // all (the memo short-circuits them), the host entity's first/last_seen + // window reflects only the FIRST row processed for that key in this + // chunk, not the widened window across all 50 rows. + let (first_seen, last_seen): (String, String) = conn + .query_row( + "SELECT first_seen_at, last_seen_at FROM graph_entities + WHERE entity_type = 'host' AND canonical_key = 'warden'", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + assert_eq!(first_seen, "2026-01-01T00:00:00Z"); + assert_eq!( + last_seen, "2026-01-01T00:00:00Z", + "memoized entity keeps the first upsert's last_seen_at within the chunk, per the documented tradeoff" + ); +} + +#[test] +fn graph_walk_n_hops_enforces_entity_cap() { + let _guard = GRAPH_TEST_LOCK.lock(); + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&test_storage_config(dir.path().join("graph-nhops-cap.db"))).unwrap(); + let conn = pool.get().unwrap(); + + // Star topology: one seed host directly connected to more apps than the + // cap allows, all reachable within a single hop. Without a row cap this + // walk would return seed + every app (well over GRAPH_WALK_N_HOPS_ENTITY_CAP). + let insert_entity = |etype: &str, key: &str| -> i64 { + conn.execute( + "INSERT INTO graph_entities (entity_type, canonical_key, display_label, trust_level) + VALUES (?1, ?2, ?2, 'verified')", + rusqlite::params![etype, key], + ) + .unwrap(); + conn.last_insert_rowid() + }; + let host = insert_entity(ENTITY_TYPE_HOST, "host-cap-seed"); + let extra_neighbours = GRAPH_WALK_N_HOPS_ENTITY_CAP + 50; + // Batch the fixture seed in one transaction — one autocommit per row + // makes this loop ~1000x slower and needlessly extends how long + // GRAPH_TEST_LOCK is held. + conn.execute_batch("BEGIN;").unwrap(); + for i in 0..extra_neighbours { + let app_key = format!("app-cap-{i}"); + let app = insert_entity(ENTITY_TYPE_APP, &app_key); + conn.execute( + "INSERT INTO graph_relationships + (relationship_key, src_entity_id, dst_entity_id, relationship_type, + reason_code, trust_level, confidence, last_seen_at) + VALUES (?1, ?2, ?3, 'emitted_by', 'log_app_name', 'verified', 0.5, '2026-01-01T00:00:00Z')", + rusqlite::params![format!("{app}:emitted_by:{host}"), app, host], + ) + .unwrap(); + } + conn.execute_batch("COMMIT;").unwrap(); + drop(conn); + + let reached = graph_walk_n_hops( + &pool.get().unwrap(), + &["host-cap-seed".to_string()], + GRAPH_WALK_MAX_DEPTH, + ) + .unwrap(); + + assert_eq!( + reached.len(), + GRAPH_WALK_N_HOPS_ENTITY_CAP, + "walk must return exactly the capped entity count, not the full {extra_neighbours}-neighbour reach" + ); +} diff --git a/crates/shared/cortex/storage-sqlite/src/heartbeat.rs b/crates/shared/cortex/storage-sqlite/src/heartbeat.rs new file mode 100644 index 00000000..a40ff9b2 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/heartbeat.rs @@ -0,0 +1,680 @@ +use anyhow::{Result, anyhow}; +use rusqlite::{OptionalExtension, params}; +use serde_json::Value; + +pub use cortex_domain::{ + HeartbeatHostState, HeartbeatSampleState, HeartbeatStateFlags, HeartbeatWindowSummary, +}; + +use tracing::warn; + +use super::pool::DbPool; + +const DISK_PRESSURE_SQL_FILTER: &str = " + used_percent IS NOT NULL + AND COALESCE(filesystem, '') NOT IN ( + 'autofs', + 'binfmt_misc', + 'bpf', + 'cgroup', + 'cgroup2', + 'configfs', + 'debugfs', + 'devpts', + 'devtmpfs', + 'efivarfs', + 'fuse.snapfuse', + 'fusectl', + 'hugetlbfs', + 'iso9660', + 'mqueue', + 'nsfs', + 'overlay', + 'proc', + 'pstore', + 'ramfs', + 'rootfs', + 'securityfs', + 'squashfs', + 'sysfs', + 'tmpfs', + 'tracefs' + ) + AND COALESCE(mountpoint, '') NOT IN ('', '/init') + AND COALESCE(mountpoint, '') NOT LIKE '/snap/%' + AND COALESCE(mountpoint, '') NOT LIKE '/mnt/wsl/docker-desktop/%' + AND COALESCE(mountpoint, '') NOT LIKE '/mnt/wslg/%' + AND COALESCE(mountpoint, '') NOT LIKE '/usr/lib/modules/%' + AND COALESCE(mountpoint, '') NOT LIKE '/usr/lib/wsl/%' + AND COALESCE(mountpoint, '') NOT LIKE '/run/%' + AND COALESCE(mountpoint, '') NOT LIKE '/var/run/%' +"; + +#[derive(Debug, Clone)] +pub enum HeartbeatHostLookup { + HostId(String), + Hostname(String), +} + +// ── Fleet-state types ───────────────────────────────────────────────────── + +/// One row from `host_heartbeats_latest` — the fleet-state cache table. +/// Holds only the fields needed to compute derived flags without joining +/// the main `host_heartbeats` table. +#[derive(Debug, Clone)] +pub struct HeartbeatLatestEntry { + pub host_id: String, + pub heartbeat_id: i64, + pub hostname: String, + pub sampled_at: String, + pub received_at: String, + pub partial: bool, + pub metadata_json: Option, +} + +/// Aggregated metric values for a single heartbeat_id. +/// Used by `app::heartbeat_flags::derive_flags` to compute pressure signals. +#[derive(Debug, Clone, Default)] +pub struct HeartbeatMetricSnapshot { + pub cpu_usage_percent: Option, + pub mem_used_percent: Option, + pub swap_total_bytes: Option, + pub swap_used_bytes: Option, + pub max_disk_used_percent: Option, + pub total_network_errors: Option, + pub container_unhealthy_count: Option, +} + +/// A host whose latest accepted heartbeat has gone stale. +#[derive(Debug, Clone)] +pub struct StaleHeartbeatHost { + pub host_id: String, + pub hostname: String, + pub received_at: String, + pub age_secs: u64, +} + +/// Hosts whose newest heartbeat is older than `threshold_secs` but younger +/// than `forget_secs` — the heartbeat_silence notification rule's input. +/// O(hosts) scan of the small `host_heartbeats_latest` cache; the forget +/// bound keeps decommissioned hosts from staying alert-eligible forever. +pub fn stale_heartbeat_hosts( + conn: &rusqlite::Connection, + threshold_secs: u64, + forget_secs: u64, +) -> Result> { + let age_expr = "CAST(strftime('%s','now') AS INTEGER) - \ + CAST(strftime('%s', received_at) AS INTEGER)"; + // Thresholds are trusted u64 config values, inlined because SQLite does + // not allow SELECT aliases in WHERE. + let sql = format!( + "SELECT host_id, hostname, received_at, {age_expr} AS age_secs + FROM host_heartbeats_latest + WHERE ({age_expr}) > {threshold_secs} AND ({age_expr}) < {forget_secs} + ORDER BY hostname ASC" + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt + .query_map([], |row| { + Ok(StaleHeartbeatHost { + host_id: row.get(0)?, + hostname: row.get(1)?, + received_at: row.get(2)?, + age_secs: row.get::<_, i64>(3)?.max(0) as u64, + }) + })? + .collect::>>()?; + Ok(rows) +} + +/// Return all entries from `host_heartbeats_latest`, ordered by hostname. +/// +/// This is an O(hosts) full scan of a small cache table — it deliberately +/// avoids scanning `host_heartbeats` (which may contain millions of rows). +/// EXPLAIN QUERY PLAN should show `SCAN host_heartbeats_latest`, never +/// `SCAN host_heartbeats`. +pub fn heartbeat_latest_all(pool: &DbPool) -> Result> { + let conn = pool.get()?; + let mut stmt = conn.prepare( + "SELECT host_id, heartbeat_id, hostname, sampled_at, received_at, + partial, metadata_json + FROM host_heartbeats_latest + ORDER BY hostname ASC", + )?; + let entries = stmt + .query_map([], |row| { + Ok(HeartbeatLatestEntry { + host_id: row.get(0)?, + heartbeat_id: row.get(1)?, + hostname: row.get(2)?, + sampled_at: row.get(3)?, + received_at: row.get(4)?, + partial: row.get::<_, i64>(5)? != 0, + metadata_json: row.get(6)?, + }) + })? + .collect::>>()?; + Ok(entries) +} + +/// Fetch aggregated metric values for one heartbeat by `heartbeat_id`. +/// +/// All five queries target indexed `heartbeat_id` columns. Each returns at +/// most one row (or one aggregate). Kept for targeted tests; production code +/// uses `heartbeat_metric_snapshot_batch`. +#[cfg(test)] +pub fn heartbeat_metric_snapshot( + pool: &DbPool, + heartbeat_id: i64, +) -> Result { + let conn = pool.get()?; + + let cpu: Option<(Option, Option)> = conn + .query_row( + "SELECT usage_percent, load1 + FROM heartbeat_cpu WHERE heartbeat_id = ?1", + [heartbeat_id], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .optional()?; + + let mem: Option<(Option, Option, Option)> = conn + .query_row( + "SELECT used_percent, swap_total_bytes, swap_used_bytes + FROM heartbeat_memory WHERE heartbeat_id = ?1", + [heartbeat_id], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .optional()?; + + let max_disk: Option = conn + .query_row( + &format!( + "SELECT MAX(used_percent) FROM heartbeat_disks \ + WHERE heartbeat_id = ?1 AND {DISK_PRESSURE_SQL_FILTER}" + ), + [heartbeat_id], + |row| row.get(0), + ) + .optional()? + .flatten(); + + let net_errors: Option = conn + .query_row( + "SELECT SUM(COALESCE(rx_errors, 0) + COALESCE(tx_errors, 0)) + FROM heartbeat_network WHERE heartbeat_id = ?1", + [heartbeat_id], + |row| row.get(0), + ) + .optional()? + .flatten(); + + let container_unhealthy: Option = conn + .query_row( + "SELECT MAX(COALESCE(unhealthy, 0)) + FROM heartbeat_containers WHERE heartbeat_id = ?1", + [heartbeat_id], + |row| row.get(0), + ) + .optional()? + .flatten(); + + Ok(HeartbeatMetricSnapshot { + cpu_usage_percent: cpu.as_ref().and_then(|(u, _)| *u), + mem_used_percent: mem.as_ref().and_then(|(u, _, _)| *u), + swap_total_bytes: mem.as_ref().and_then(|(_, t, _)| *t), + swap_used_bytes: mem.and_then(|(_, _, u)| u), + max_disk_used_percent: max_disk, + total_network_errors: net_errors, + container_unhealthy_count: container_unhealthy, + }) +} + +/// Fetch aggregated metric values for multiple heartbeat IDs in one pass. +/// +/// Returns a map from heartbeat_id → snapshot. IDs with no data are absent +/// from the map (callers should use `unwrap_or_default()`). +pub fn heartbeat_metric_snapshot_batch( + pool: &DbPool, + ids: &[i64], +) -> Result> { + if ids.is_empty() { + return Ok(std::collections::HashMap::new()); + } + let conn = pool.get()?; + let placeholders = ids + .iter() + .map(|id| id.to_string()) + .collect::>() + .join(","); + + let mut map: std::collections::HashMap = ids + .iter() + .map(|&id| (id, HeartbeatMetricSnapshot::default())) + .collect(); + + let cpu_sql = format!( + "SELECT heartbeat_id, usage_percent FROM heartbeat_cpu WHERE heartbeat_id IN ({placeholders})" + ); + let mut stmt = conn.prepare(&cpu_sql)?; + let rows = stmt.query_map([], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, Option>(1)?)) + })?; + for row in rows.flatten() { + map.entry(row.0).or_default().cpu_usage_percent = row.1; + } + + let mem_sql = format!( + "SELECT heartbeat_id, used_percent, swap_total_bytes, swap_used_bytes \ + FROM heartbeat_memory WHERE heartbeat_id IN ({placeholders})" + ); + let mut stmt = conn.prepare(&mem_sql)?; + let rows = stmt.query_map([], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, Option>(1)?, + row.get::<_, Option>(2)?, + row.get::<_, Option>(3)?, + )) + })?; + for row in rows.flatten() { + let e = map.entry(row.0).or_default(); + e.mem_used_percent = row.1; + e.swap_total_bytes = row.2; + e.swap_used_bytes = row.3; + } + + let disk_sql = format!( + "SELECT heartbeat_id, MAX(used_percent) FROM heartbeat_disks \ + WHERE heartbeat_id IN ({placeholders}) AND {DISK_PRESSURE_SQL_FILTER} \ + GROUP BY heartbeat_id" + ); + let mut stmt = conn.prepare(&disk_sql)?; + let rows = stmt.query_map([], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, Option>(1)?)) + })?; + for row in rows.flatten() { + map.entry(row.0).or_default().max_disk_used_percent = row.1; + } + + let net_sql = format!( + "SELECT heartbeat_id, SUM(COALESCE(rx_errors,0)+COALESCE(tx_errors,0)) \ + FROM heartbeat_network WHERE heartbeat_id IN ({placeholders}) GROUP BY heartbeat_id" + ); + let mut stmt = conn.prepare(&net_sql)?; + let rows = stmt.query_map([], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, Option>(1)?)) + })?; + for row in rows.flatten() { + map.entry(row.0).or_default().total_network_errors = row.1; + } + + let ctr_sql = format!( + "SELECT heartbeat_id, MAX(COALESCE(unhealthy,0)) FROM heartbeat_containers \ + WHERE heartbeat_id IN ({placeholders}) GROUP BY heartbeat_id" + ); + let mut stmt = conn.prepare(&ctr_sql)?; + let rows = stmt.query_map([], |row| { + Ok((row.get::<_, i64>(0)?, row.get::<_, Option>(1)?)) + })?; + for row in rows.flatten() { + map.entry(row.0).or_default().container_unhealthy_count = row.1; + } + + Ok(map) +} + +/// Build per-host heartbeat summaries for a time window. +/// +/// When `host_id` is `Some`, only that host is included (single-host +/// correlate_state). When `None`, all hosts with heartbeats in the window are +/// included. The query uses the `idx_host_heartbeats_received` index on +/// `received_at` as the primary range predicate to avoid broad table scans. +pub fn heartbeat_window_summaries( + pool: &DbPool, + from: &str, + to: &str, + host_id: Option<&str>, +) -> Result> { + let conn = pool.get()?; + + type WindowRow = (String, String, i64, i64, Option, Option); + let row_from = |row: &rusqlite::Row<'_>| -> rusqlite::Result { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + )) + }; + + // `max_cpu`/`min_mem` come from the most-recent heartbeat in the window + // for each host. The latest id must be resolved with a scalar subquery + // (`SELECT MAX(h2.id) ...`) rather than referencing `MAX(h.id)` inside a + // correlated subquery — SQLite rejects the latter as "misuse of aggregate" + // under GROUP BY. + let rows: Vec = if let Some(hid) = host_id { + let mut stmt = conn.prepare( + "SELECT h.host_id, h.hostname, + COUNT(*) AS samples, + SUM(h.partial) AS partial_samples, + (SELECT c.usage_percent FROM heartbeat_cpu c + WHERE c.heartbeat_id = ( + SELECT MAX(h2.id) FROM host_heartbeats h2 + WHERE h2.host_id = h.host_id + AND h2.received_at >= ?2 + AND h2.received_at <= ?3 + )) AS max_cpu, + (SELECT m.available_bytes FROM heartbeat_memory m + WHERE m.heartbeat_id = ( + SELECT MAX(h2.id) FROM host_heartbeats h2 + WHERE h2.host_id = h.host_id + AND h2.received_at >= ?2 + AND h2.received_at <= ?3 + )) AS min_mem + FROM host_heartbeats h + WHERE h.host_id = ?1 + AND h.received_at >= ?2 + AND h.received_at <= ?3 + GROUP BY h.host_id, h.hostname", + )?; + + stmt.query_map(params![hid, from, to], row_from)? + .collect::>>()? + } else { + let mut stmt = conn.prepare( + "SELECT h.host_id, h.hostname, + COUNT(*) AS samples, + SUM(h.partial) AS partial_samples, + (SELECT c.usage_percent FROM heartbeat_cpu c + WHERE c.heartbeat_id = ( + SELECT MAX(h2.id) FROM host_heartbeats h2 + WHERE h2.host_id = h.host_id + AND h2.received_at >= ?1 + AND h2.received_at <= ?2 + )) AS max_cpu, + (SELECT m.available_bytes FROM heartbeat_memory m + WHERE m.heartbeat_id = ( + SELECT MAX(h2.id) FROM host_heartbeats h2 + WHERE h2.host_id = h.host_id + AND h2.received_at >= ?1 + AND h2.received_at <= ?2 + )) AS min_mem + FROM host_heartbeats h + WHERE h.received_at >= ?1 + AND h.received_at <= ?2 + GROUP BY h.host_id, h.hostname + ORDER BY h.hostname ASC", + )?; + + stmt.query_map(params![from, to], row_from)? + .collect::>>()? + }; + + Ok(rows + .into_iter() + .map( + |(host_id, hostname, samples, partial_samples, max_cpu, min_mem)| { + HeartbeatWindowSummary { + host_id, + hostname, + samples: samples as usize, + partial_samples: partial_samples as usize, + max_cpu_usage_percent: max_cpu, + min_mem_available_bytes: min_mem, + pressure_flags: Vec::new(), // filled by service layer + } + }, + ) + .collect()) +} + +// ── Private row types ───────────────────────────────────────────────────── + +#[derive(Debug)] +struct HeartbeatRow { + id: i64, + host_id: String, + hostname: String, + source_ip: String, + sampled_at: String, + received_at: String, + boot_id: String, + uptime_secs: i64, + sequence: i64, + collection_ms: i64, + partial: bool, + agent_version: String, + os: String, + kernel: Option, + architecture: String, + metadata_json: Option, +} + +pub fn heartbeat_host_state( + pool: &DbPool, + lookup: HeartbeatHostLookup, + since: Option<&str>, + limit: usize, +) -> Result { + let conn = pool.get()?; + let host_id = match lookup { + HeartbeatHostLookup::HostId(host_id) => host_id, + HeartbeatHostLookup::Hostname(hostname) => resolve_unique_hostname(&conn, &hostname)?, + }; + + let limit = limit.clamp(1, 100); + let fetch_limit = limit + 1; + let rows = if let Some(since) = since { + let mut stmt = conn.prepare( + "SELECT id, host_id, hostname, source_ip, sampled_at, received_at, boot_id, + uptime_secs, sequence, collection_ms, partial, agent_version, + os, kernel, architecture, metadata_json + FROM host_heartbeats + WHERE host_id = ?1 AND sampled_at >= ?2 + ORDER BY sampled_at DESC, id DESC + LIMIT ?3", + )?; + + stmt.query_map( + params![host_id, since, fetch_limit as i64], + map_heartbeat_row, + )? + .collect::>>()? + } else { + let mut stmt = conn.prepare( + "SELECT id, host_id, hostname, source_ip, sampled_at, received_at, boot_id, + uptime_secs, sequence, collection_ms, partial, agent_version, + os, kernel, architecture, metadata_json + FROM host_heartbeats + WHERE host_id = ?1 + ORDER BY sampled_at DESC, id DESC + LIMIT ?2", + )?; + + stmt.query_map(params![host_id, fetch_limit as i64], map_heartbeat_row)? + .collect::>>()? + }; + + if rows.is_empty() { + return Err(anyhow!("not_found")); + } + + let truncated = rows.len() > limit; + let mut samples = Vec::with_capacity(limit.min(rows.len())); + for row in rows.into_iter().take(limit) { + samples.push(sample_from_row(&conn, row)?); + } + let latest = samples.first().cloned(); + let host_id = samples[0].host_id.clone(); + let hostname = samples[0].hostname.clone(); + let flags = latest.as_ref().map(heartbeat_flags).unwrap_or_default(); + + Ok(HeartbeatHostState { + host_id, + hostname, + total_samples: samples.len(), + truncated, + flags, + latest, + samples, + }) +} + +/// Derive canonical domain heartbeat flags from a fully-loaded storage sample. +pub(crate) fn heartbeat_flags(sample: &HeartbeatSampleState) -> HeartbeatStateFlags { + cortex_domain::heartbeat_flags_from_sample(sample) +} + +fn resolve_unique_hostname(conn: &rusqlite::Connection, hostname: &str) -> Result { + let mut stmt = conn.prepare( + "SELECT host_id + FROM host_heartbeats + WHERE hostname = ?1 + GROUP BY host_id + ORDER BY MAX(received_at) DESC + LIMIT 2", + )?; + let host_ids = stmt + .query_map([hostname], |row| row.get::<_, String>(0))? + .collect::>>()?; + match host_ids.as_slice() { + [] => Err(anyhow!("not_found")), + [host_id] => Ok(host_id.clone()), + _ => Err(anyhow!("ambiguous_host")), + } +} + +fn map_heartbeat_row(row: &rusqlite::Row<'_>) -> rusqlite::Result { + Ok(HeartbeatRow { + id: row.get(0)?, + host_id: row.get(1)?, + hostname: row.get(2)?, + source_ip: row.get(3)?, + sampled_at: row.get(4)?, + received_at: row.get(5)?, + boot_id: row.get(6)?, + uptime_secs: row.get(7)?, + sequence: row.get(8)?, + collection_ms: row.get(9)?, + partial: row.get::<_, i64>(10)? != 0, + agent_version: row.get(11)?, + os: row.get(12)?, + kernel: row.get(13)?, + architecture: row.get(14)?, + metadata_json: row.get(15)?, + }) +} + +fn sample_from_row(conn: &rusqlite::Connection, row: HeartbeatRow) -> Result { + Ok(HeartbeatSampleState { + heartbeat_id: row.id, + host_id: row.host_id, + hostname: row.hostname, + sampled_at: row.sampled_at, + received_at: row.received_at, + source_ip: row.source_ip, + boot_id: row.boot_id, + sequence: row.sequence, + uptime_secs: row.uptime_secs, + collection_ms: row.collection_ms, + partial: row.partial, + agent_version: row.agent_version, + os: row.os, + kernel: row.kernel, + architecture: row.architecture, + metadata: row + .metadata_json + .as_deref() + .and_then(|raw| serde_json::from_str(raw).ok()), + cpu: one_json( + conn, + "SELECT json_object( + 'load1', load1, 'load5', load5, 'load15', load15, + 'usage_percent', usage_percent, 'steal_percent', steal_percent, + 'io_wait_percent', io_wait_percent + ) FROM heartbeat_cpu WHERE heartbeat_id = ?1", + row.id, + )?, + memory: one_json( + conn, + "SELECT json_object( + 'total_bytes', total_bytes, 'available_bytes', available_bytes, + 'used_percent', used_percent, 'swap_total_bytes', swap_total_bytes, + 'swap_used_bytes', swap_used_bytes + ) FROM heartbeat_memory WHERE heartbeat_id = ?1", + row.id, + )?, + disks: many_json( + conn, + "SELECT json_object( + 'mountpoint', mountpoint, 'filesystem', filesystem, + 'total_bytes', total_bytes, 'available_bytes', available_bytes, + 'used_percent', used_percent, 'read_bytes_per_sec', read_bytes_per_sec, + 'write_bytes_per_sec', write_bytes_per_sec + ) FROM heartbeat_disks WHERE heartbeat_id = ?1 ORDER BY id ASC", + row.id, + )?, + network: many_json( + conn, + "SELECT json_object( + 'interface', interface, 'rx_bytes_per_sec', rx_bytes_per_sec, + 'tx_bytes_per_sec', tx_bytes_per_sec, 'rx_errors', rx_errors, + 'tx_errors', tx_errors + ) FROM heartbeat_network WHERE heartbeat_id = ?1 ORDER BY id ASC", + row.id, + )?, + processes: one_json( + conn, + "SELECT json_object( + 'total', total, 'running', running, 'sleeping', sleeping, 'zombie', zombie, + 'top_cpu', json(top_cpu_json), 'top_memory', json(top_memory_json) + ) FROM heartbeat_processes WHERE heartbeat_id = ?1", + row.id, + )?, + containers: many_json( + conn, + "SELECT json_object( + 'runtime', runtime, 'running', running, 'stopped', stopped, + 'restarting', restarting, 'unhealthy', unhealthy, 'summary', json(summary_json) + ) FROM heartbeat_containers WHERE heartbeat_id = ?1 ORDER BY id ASC", + row.id, + )?, + }) +} + +fn one_json(conn: &rusqlite::Connection, sql: &str, heartbeat_id: i64) -> Result> { + let raw: Option = conn + .query_row(sql, [heartbeat_id], |row| row.get(0)) + .optional()?; + Ok(raw.and_then(|raw| { + serde_json::from_str(&raw) + .map_err(|error| { + warn!(heartbeat_id, error = %error, "failed to parse heartbeat JSON column"); + }) + .ok() + })) +} + +fn many_json(conn: &rusqlite::Connection, sql: &str, heartbeat_id: i64) -> Result> { + let mut stmt = conn.prepare(sql)?; + let rows = stmt + .query_map([heartbeat_id], |row| row.get::<_, String>(0))? + .collect::>>()?; + Ok(rows + .into_iter() + .filter_map(|raw| { + serde_json::from_str(&raw) + .map_err(|error| { + warn!(heartbeat_id, error = %error, "failed to parse heartbeat JSON row"); + }) + .ok() + }) + .collect()) +} + +#[cfg(test)] +#[path = "heartbeat_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/heartbeat_tests.rs b/crates/shared/cortex/storage-sqlite/src/heartbeat_tests.rs new file mode 100644 index 00000000..5002924e --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/heartbeat_tests.rs @@ -0,0 +1,484 @@ +use super::*; + +use crate::config::StorageConfig; + +fn test_pool() -> (DbPool, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let storage = StorageConfig::for_test(dir.path().join("heartbeat-state.db")); + let pool = crate::init_pool(&storage).unwrap(); + (pool, dir) +} + +fn insert_heartbeat( + pool: &DbPool, + host_id: &str, + hostname: &str, + sequence: i64, + sampled_at: &str, + partial: bool, +) -> i64 { + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO host_heartbeats ( + host_id, hostname, source_ip, sampled_at, received_at, boot_id, + uptime_secs, sequence, collection_ms, partial, agent_version, + os, architecture, metadata_json + ) VALUES (?1, ?2, '10.0.0.1:41000', ?4, ?4, 'boot-a', 60, ?3, 5, ?5, + '0.1.0', 'linux', 'x86_64', '{\"agent\":{\"interval_secs\":30}}')", + params![host_id, hostname, sequence, sampled_at, partial as i64], + ) + .unwrap(); + conn.last_insert_rowid() +} + +/// Populate `host_heartbeats_latest` as the ingest path would. +fn seed_latest( + pool: &DbPool, + host_id: &str, + heartbeat_id: i64, + hostname: &str, + sampled_at: &str, + partial: bool, +) { + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO host_heartbeats_latest + (host_id, heartbeat_id, hostname, sampled_at, received_at, + partial, agent_version, os, architecture, metadata_json) + VALUES (?1, ?2, ?3, ?4, ?4, ?5, '0.1.0', 'linux', 'x86_64', + '{\"agent\":{\"interval_secs\":30}}') + ON CONFLICT(host_id) DO UPDATE SET + heartbeat_id = excluded.heartbeat_id, + hostname = excluded.hostname, + sampled_at = excluded.sampled_at, + received_at = excluded.received_at, + partial = excluded.partial, + agent_version = excluded.agent_version, + os = excluded.os, + architecture = excluded.architecture, + metadata_json = excluded.metadata_json + WHERE excluded.sampled_at >= host_heartbeats_latest.sampled_at", + params![host_id, heartbeat_id, hostname, sampled_at, partial as i64], + ) + .unwrap(); +} + +#[test] +fn host_state_returns_latest_by_host_id() { + let (pool, _dir) = test_pool(); + let older = insert_heartbeat(&pool, "host-a", "nashost", 1, "2026-05-25T00:00:00Z", false); + let latest = insert_heartbeat(&pool, "host-a", "nashost", 2, "2026-05-25T00:01:00Z", true); + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO heartbeat_cpu (heartbeat_id, load1, load5, load15) + VALUES (?1, 0.1, 0.2, 0.3)", + [latest], + ) + .unwrap(); + drop(conn); + + let state = heartbeat_host_state( + &pool, + HeartbeatHostLookup::HostId("host-a".into()), + None, + 10, + ) + .unwrap(); + assert_eq!(state.host_id, "host-a"); + assert_eq!(state.latest.as_ref().unwrap().heartbeat_id, latest); + assert!(state.flags.collector_partial); + assert_eq!(state.samples.len(), 2); + assert!( + state + .samples + .iter() + .any(|sample| sample.heartbeat_id == older) + ); + assert!(state.latest.as_ref().unwrap().cpu.is_some()); +} + +#[test] +fn host_state_unique_hostname_fallback_and_ambiguous_hostname() { + let (pool, _dir) = test_pool(); + insert_heartbeat(&pool, "host-a", "unique", 1, "2026-05-25T00:00:00Z", false); + let state = heartbeat_host_state( + &pool, + HeartbeatHostLookup::Hostname("unique".into()), + None, + 1, + ) + .unwrap(); + assert_eq!(state.host_id, "host-a"); + + insert_heartbeat(&pool, "host-b", "shared", 1, "2026-05-25T00:00:00Z", false); + insert_heartbeat(&pool, "host-c", "shared", 1, "2026-05-25T00:01:00Z", false); + let error = heartbeat_host_state( + &pool, + HeartbeatHostLookup::Hostname("shared".into()), + None, + 1, + ) + .unwrap_err(); + assert_eq!(error.to_string(), "ambiguous_host"); +} + +#[test] +fn host_state_caps_limit_and_filters_since() { + let (pool, _dir) = test_pool(); + for sequence in 0..105 { + insert_heartbeat( + &pool, + "host-a", + "nashost", + sequence, + &format!("2026-05-25T00:{sequence:03}:00Z"), + false, + ); + } + + let capped = heartbeat_host_state( + &pool, + HeartbeatHostLookup::HostId("host-a".into()), + None, + 500, + ) + .unwrap(); + assert_eq!(capped.samples.len(), 100); + assert!(capped.truncated); + + let since = heartbeat_host_state( + &pool, + HeartbeatHostLookup::HostId("host-a".into()), + Some("2026-05-25T00:099:00Z"), + 100, + ) + .unwrap(); + assert_eq!(since.samples.len(), 6); +} + +// ── Fleet-state cache tests ─────────────────────────────────────────────── + +/// `heartbeat_latest_all` must use SCAN on the small cache table, not the +/// main `host_heartbeats` table. Verified via EXPLAIN QUERY PLAN. +#[test] +fn fleet_state_explain_does_not_scan_main_table() { + let (pool, _dir) = test_pool(); + // Seed the cache with two hosts; main table is intentionally empty for + // this test (the cache is populated by the ingest path, or migration 19). + seed_latest(&pool, "host-a", 1, "nashost", "2026-05-25T00:01:00Z", false); + seed_latest(&pool, "host-b", 2, "devhost", "2026-05-25T00:01:00Z", false); + + let conn = pool.get().unwrap(); + let plan: Vec = { + let mut stmt = conn + .prepare( + "EXPLAIN QUERY PLAN + SELECT host_id, heartbeat_id, hostname, sampled_at, received_at, + partial, metadata_json + FROM host_heartbeats_latest + ORDER BY hostname ASC", + ) + .unwrap(); + stmt.query_map([], |row| row.get::<_, String>(3)) + .unwrap() + .map(|r| r.unwrap()) + .collect() + }; + + let plan_text = plan.join("\n").to_lowercase(); + // Must see the cache table in the plan, not the main table. + assert!( + plan_text.contains("host_heartbeats_latest"), + "EXPLAIN must reference host_heartbeats_latest; got: {plan_text}" + ); + assert!( + !plan_text.contains("scan host_heartbeats\n") + && !plan_text.contains("scan host_heartbeats "), + "EXPLAIN must NOT scan host_heartbeats; got: {plan_text}" + ); +} + +#[test] +fn heartbeat_latest_all_returns_one_row_per_host_ordered_by_hostname() { + let (pool, _dir) = test_pool(); + seed_latest(&pool, "host-b", 2, "zebra", "2026-05-25T00:01:00Z", false); + seed_latest(&pool, "host-a", 1, "alpha", "2026-05-25T00:00:00Z", true); + seed_latest(&pool, "host-c", 3, "midway", "2026-05-25T00:02:00Z", false); + + let entries = heartbeat_latest_all(&pool).unwrap(); + assert_eq!(entries.len(), 3); + // Verify hostname ordering. + assert_eq!(entries[0].hostname, "alpha"); + assert_eq!(entries[1].hostname, "midway"); + assert_eq!(entries[2].hostname, "zebra"); + // Partial flag is preserved. + assert!(entries[0].partial); + assert!(!entries[1].partial); +} + +#[test] +fn cache_upsert_only_advances_on_newer_sampled_at() { + let (pool, _dir) = test_pool(); + seed_latest(&pool, "host-a", 1, "nashost", "2026-05-25T00:01:00Z", false); + // "Older" heartbeat: sampled_at is earlier, should NOT overwrite. + seed_latest(&pool, "host-a", 99, "nashost", "2026-05-24T00:00:00Z", true); + + let entries = heartbeat_latest_all(&pool).unwrap(); + assert_eq!(entries.len(), 1); + assert_eq!( + entries[0].heartbeat_id, 1, + "older heartbeat must not overwrite newer cache entry" + ); + assert!( + !entries[0].partial, + "partial flag must not be overwritten by older entry" + ); +} + +#[test] +fn heartbeat_metric_snapshot_returns_aggregates() { + let (pool, _dir) = test_pool(); + let hb_id = insert_heartbeat(&pool, "host-a", "nashost", 1, "2026-05-25T00:00:00Z", false); + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO heartbeat_cpu (heartbeat_id, load1, load5, load15, usage_percent) + VALUES (?1, 1.0, 1.5, 2.0, 91.5)", + [hb_id], + ) + .unwrap(); + conn.execute( + "INSERT INTO heartbeat_memory + (heartbeat_id, total_bytes, available_bytes, used_percent, + swap_total_bytes, swap_used_bytes) + VALUES (?1, 8000000000, 500000000, 87.5, 2000000000, 1900000000)", + [hb_id], + ) + .unwrap(); + conn.execute( + "INSERT INTO heartbeat_disks + (heartbeat_id, mountpoint, filesystem, total_bytes, available_bytes, used_percent) + VALUES (?1, '/', 'ext4', 1000000000, 50000000, 95.0)", + [hb_id], + ) + .unwrap(); + conn.execute( + "INSERT INTO heartbeat_network + (heartbeat_id, interface, rx_bytes_per_sec, tx_bytes_per_sec, rx_errors, tx_errors) + VALUES (?1, 'eth0', 1000, 500, 3, 1)", + [hb_id], + ) + .unwrap(); + conn.execute( + "INSERT INTO heartbeat_containers + (heartbeat_id, runtime, running, stopped, restarting, unhealthy) + VALUES (?1, 'docker', 5, 1, 0, 2)", + [hb_id], + ) + .unwrap(); + drop(conn); + + let snap = heartbeat_metric_snapshot(&pool, hb_id).unwrap(); + assert!( + snap.cpu_usage_percent + .is_some_and(|p| (p - 91.5).abs() < 0.01), + "cpu_usage_percent" + ); + assert!( + snap.mem_used_percent + .is_some_and(|p| (p - 87.5).abs() < 0.01), + "mem_used_percent" + ); + assert!( + snap.swap_total_bytes == Some(2_000_000_000), + "swap_total_bytes" + ); + assert!( + snap.max_disk_used_percent + .is_some_and(|p| (p - 95.0).abs() < 0.01), + "max_disk_used_percent" + ); + assert_eq!(snap.total_network_errors, Some(4), "total_network_errors"); + assert_eq!( + snap.container_unhealthy_count, + Some(2), + "container_unhealthy_count" + ); +} + +#[test] +fn heartbeat_metric_snapshot_ignores_pseudo_mounts_for_disk_pressure() { + let (pool, _dir) = test_pool(); + let hb_id = insert_heartbeat( + &pool, + "host-a", + "laptophost", + 1, + "2026-05-25T00:00:00Z", + false, + ); + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO heartbeat_disks + (heartbeat_id, mountpoint, filesystem, total_bytes, available_bytes, used_percent) + VALUES (?1, '/', 'ext4', 1000000000, 800000000, 20.0)", + [hb_id], + ) + .unwrap(); + conn.execute( + "INSERT INTO heartbeat_disks + (heartbeat_id, mountpoint, filesystem, total_bytes, available_bytes, used_percent) + VALUES (?1, '/mnt/wsl/docker-desktop/cli-tools', 'iso9660', 800000000, 0, 100.0)", + [hb_id], + ) + .unwrap(); + drop(conn); + + let snap = heartbeat_metric_snapshot(&pool, hb_id).unwrap(); + assert_eq!(snap.max_disk_used_percent, Some(20.0)); + + let batch = heartbeat_metric_snapshot_batch(&pool, &[hb_id]).unwrap(); + assert_eq!( + batch + .get(&hb_id) + .and_then(|snap| snap.max_disk_used_percent), + Some(20.0) + ); +} + +#[test] +fn heartbeat_metric_snapshot_keeps_unraid_user_share_pressure() { + let (pool, _dir) = test_pool(); + let hb_id = insert_heartbeat(&pool, "host-a", "nashost", 1, "2026-05-25T00:00:00Z", false); + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO heartbeat_disks + (heartbeat_id, mountpoint, filesystem, total_bytes, available_bytes, used_percent) + VALUES (?1, '/mnt/user/appdata/cortex', 'fuse.shfs', 1000000000, 40000000, 96.0)", + [hb_id], + ) + .unwrap(); + drop(conn); + + let snap = heartbeat_metric_snapshot(&pool, hb_id).unwrap(); + assert_eq!(snap.max_disk_used_percent, Some(96.0)); +} + +fn insert_cpu(pool: &DbPool, heartbeat_id: i64, usage_percent: f64) { + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO heartbeat_cpu (heartbeat_id, usage_percent) VALUES (?1, ?2)", + params![heartbeat_id, usage_percent], + ) + .unwrap(); +} + +fn insert_memory(pool: &DbPool, heartbeat_id: i64, available_bytes: i64) { + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO heartbeat_memory (heartbeat_id, available_bytes) VALUES (?1, ?2)", + params![heartbeat_id, available_bytes], + ) + .unwrap(); +} + +/// Regression test for the `heartbeat_window_summaries` "misuse of aggregate" +/// SQL bug: `MAX(h.id)` referenced inside a correlated subquery under GROUP BY +/// is illegal in SQLite. The query must resolve the latest heartbeat id with a +/// scalar subquery, and the returned cpu/mem must come from that latest sample. +#[test] +fn heartbeat_window_summaries_resolves_latest_sample_metrics() { + let (pool, _dir) = test_pool(); + // Two samples for host-a in the window; the second is the latest. + let id1 = insert_heartbeat(&pool, "host-a", "nashost", 1, "2026-05-25T00:01:00Z", false); + let id2 = insert_heartbeat(&pool, "host-a", "nashost", 2, "2026-05-25T00:02:00Z", true); + insert_cpu(&pool, id1, 10.0); + insert_cpu(&pool, id2, 80.0); + insert_memory(&pool, id1, 8_000); + insert_memory(&pool, id2, 2_000); + // A second host to confirm per-group resolution and ordering. + let id3 = insert_heartbeat(&pool, "host-b", "devhost", 1, "2026-05-25T00:01:30Z", false); + insert_cpu(&pool, id3, 42.0); + insert_memory(&pool, id3, 4_000); + + let from = "2026-05-25T00:00:00Z"; + let to = "2026-05-25T00:05:00Z"; + + // All-hosts path (host omitted): bounded cross-host plan. + let all = heartbeat_window_summaries(&pool, from, to, None).unwrap(); + assert_eq!(all.len(), 2, "expected one summary row per host"); + // Ordered by hostname ASC: devhost, nashost. + assert_eq!(all[0].hostname, "devhost"); + assert_eq!(all[1].hostname, "nashost"); + // nashost's metrics come from the latest sample (id2), not the first. + assert_eq!(all[1].samples, 2); + assert_eq!(all[1].partial_samples, 1); + assert_eq!(all[1].max_cpu_usage_percent, Some(80.0)); + assert_eq!(all[1].min_mem_available_bytes, Some(2_000)); + + // Single-host path. + let one = heartbeat_window_summaries(&pool, from, to, Some("host-a")).unwrap(); + assert_eq!(one.len(), 1); + assert_eq!(one[0].max_cpu_usage_percent, Some(80.0)); + assert_eq!(one[0].min_mem_available_bytes, Some(2_000)); +} + +mod stale_heartbeat_hosts_tests { + use rusqlite::Connection; + + fn conn_with_latest(entries: &[(&str, &str, i64)]) -> Connection { + let conn = Connection::open_in_memory().expect("in-memory db"); + conn.execute_batch( + "CREATE TABLE host_heartbeats_latest ( + host_id TEXT PRIMARY KEY, + heartbeat_id INTEGER NOT NULL, + hostname TEXT NOT NULL, + sampled_at TEXT NOT NULL, + received_at TEXT NOT NULL, + partial INTEGER NOT NULL DEFAULT 0, + agent_version TEXT NOT NULL DEFAULT '', + os TEXT NOT NULL DEFAULT '', + architecture TEXT NOT NULL DEFAULT '', + metadata_json TEXT + );", + ) + .expect("schema"); + for (host_id, hostname, age_secs) in entries { + conn.execute( + "INSERT INTO host_heartbeats_latest + (host_id, heartbeat_id, hostname, sampled_at, received_at) + VALUES (?1, 1, ?2, + strftime('%Y-%m-%dT%H:%M:%fZ', 'now', printf('-%d seconds', ?3)), + strftime('%Y-%m-%dT%H:%M:%fZ', 'now', printf('-%d seconds', ?3)))", + rusqlite::params![host_id, hostname, age_secs], + ) + .expect("insert"); + } + conn + } + + #[test] + fn returns_only_hosts_between_threshold_and_forget() { + let conn = conn_with_latest(&[ + ("id-fresh", "devhost", 30), // current — below threshold + ("id-stale", "backuphost", 1200), // stale — alertable + ("id-ancient", "deckhost", 700_000), // past forget horizon + ]); + let stale = crate::stale_heartbeat_hosts(&conn, 600, 604_800).expect("query"); + assert_eq!( + stale.len(), + 1, + "only the stale-but-remembered host: {stale:?}" + ); + assert_eq!(stale[0].hostname, "backuphost"); + assert_eq!(stale[0].host_id, "id-stale"); + assert!(stale[0].age_secs > 600 && stale[0].age_secs < 2000); + assert!(!stale[0].received_at.is_empty()); + } + + #[test] + fn empty_table_yields_no_hosts() { + let conn = conn_with_latest(&[]); + let stale = crate::stale_heartbeat_hosts(&conn, 600, 604_800).expect("query"); + assert!(stale.is_empty()); + } +} diff --git a/crates/shared/cortex/storage-sqlite/src/hook_events.rs b/crates/shared/cortex/storage-sqlite/src/hook_events.rs new file mode 100644 index 00000000..6a7fb085 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/hook_events.rs @@ -0,0 +1,210 @@ +//! `ai_hook_events` insert + list query layer. Table/columns are defined in +//! migration 40 (`src/db/pool.rs`). Extraction happens in +//! `crate::inputs` (Claude runtime attachments) and +//! `crate::hook_config` (Claude/Codex config-inventory collectors); this +//! module only persists and reads back already-extracted events. Mirrors +//! `src/db/skill_events.rs` one-for-one. + +use anyhow::Result; +use rusqlite::{Transaction, params}; +use serde::{Deserialize, Serialize}; + +use crate::inputs::ExtractedHookEvent; +pub use cortex_domain::HookEventEntry as AiHookEventEntry; + +use super::pool::DbPool; + +#[derive(Debug, Clone)] +pub struct HookEventInsert { + pub log_id: Option, + pub ai_tool: String, + pub ai_project: Option, + pub ai_session_id: Option, + pub hostname: String, + pub timestamp: String, + pub event: ExtractedHookEvent, +} + +/// Insert `events` inside an existing transaction with `INSERT OR IGNORE` +/// (idempotent on the `UNIQUE(ai_tool, ai_session_id, hook_event, hook_name, +/// timestamp, evidence_kind)` constraint). Returns the number of rows +/// actually inserted (excludes ignored duplicates) via SQLite `changes()` +/// summed per statement. +pub(crate) fn insert_hook_events_in_tx( + tx: &Transaction<'_>, + events: &[HookEventInsert], +) -> Result { + if events.is_empty() { + return Ok(0); + } + let mut stmt = tx.prepare_cached( + "INSERT OR IGNORE INTO ai_hook_events ( + log_id, ai_tool, ai_project, ai_session_id, hostname, timestamp, + hook_event, hook_name, hook_source, hook_command, status, + exit_code, duration_ms, stdout_preview, stderr_preview, + persisted_output_path, trusted_hash, evidence_kind, metadata_json + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19)", + )?; + let mut inserted = 0usize; + for item in events { + let changed = stmt.execute(params![ + item.log_id, + item.ai_tool, + item.ai_project, + item.ai_session_id, + item.hostname, + item.timestamp, + item.event.hook_event, + item.event.hook_name, + item.event.hook_source, + item.event.hook_command, + item.event.status.as_str(), + item.event.exit_code, + item.event.duration_ms, + item.event.stdout_preview, + item.event.stderr_preview, + item.event.persisted_output_path, + item.event.trusted_hash, + item.event.evidence_kind.as_str(), + item.event.metadata_json, + ])?; + inserted += changed; + } + Ok(inserted) +} + +/// Pool-acquiring wrapper for callers outside an existing transaction (e.g. +/// the backfill service and the config-inventory collector CLI path, both of +/// which own their own transaction boundary). +pub fn insert_hook_events(pool: &DbPool, events: &[HookEventInsert]) -> Result { + let mut conn = pool.get()?; + let _write_guard = crate::write_lock(); + let tx = conn.transaction()?; + let inserted = insert_hook_events_in_tx(&tx, events)?; + tx.commit()?; + Ok(inserted) +} + +#[derive(Debug, Clone, Default)] +pub struct AiHookEventParams { + pub hook_event: Option, + pub hook_name: Option, + pub hook_source: Option, + pub status: Option, + pub evidence_kind: Option, + pub tool: Option, + pub project: Option, + pub session_id: Option, + pub hostname: Option, + pub from: Option, + pub to: Option, + pub limit: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ListHookEventsResult { + pub total: usize, + pub truncated: bool, + pub events: Vec, +} + +const DEFAULT_LIMIT: u32 = 50; +const MAX_LIMIT: u32 = 500; + +const HOOK_EVENT_COLUMNS: &str = + "id, log_id, ai_tool, ai_project, ai_session_id, hostname, timestamp, + hook_event, hook_name, hook_source, hook_command, status, exit_code, + duration_ms, stdout_preview, stderr_preview, persisted_output_path, + trusted_hash, evidence_kind, metadata_json"; + +pub(crate) fn map_hook_event_row(row: &rusqlite::Row) -> rusqlite::Result { + Ok(AiHookEventEntry { + id: row.get(0)?, + log_id: row.get(1)?, + ai_tool: row.get(2)?, + ai_project: row.get(3)?, + ai_session_id: row.get(4)?, + hostname: row.get(5)?, + timestamp: row.get(6)?, + hook_event: row.get(7)?, + hook_name: row.get(8)?, + hook_source: row.get(9)?, + hook_command: row.get(10)?, + status: row.get(11)?, + exit_code: row.get(12)?, + duration_ms: row.get(13)?, + stdout_preview: row.get(14)?, + stderr_preview: row.get(15)?, + persisted_output_path: row.get(16)?, + trusted_hash: row.get(17)?, + evidence_kind: row.get(18)?, + metadata_json: row.get(19)?, + }) +} + +/// List `ai_hook_events` rows newest-first, applying every non-`None` +/// filter in `params` as an `AND`-ed equality/range clause. `limit` is +/// clamped to `[1, 500]`; `truncated` is `true` when more rows matched than +/// were returned (probed via `LIMIT + 1`, mirroring `list_skill_events`). +pub fn list_hook_events(pool: &DbPool, params: &AiHookEventParams) -> Result { + let conn = pool.get()?; + let limit = params.limit.unwrap_or(DEFAULT_LIMIT).clamp(1, MAX_LIMIT) as usize; + + let mut sql = format!("SELECT {HOOK_EVENT_COLUMNS} FROM ai_hook_events WHERE 1 = 1"); + let mut bindings: Vec = Vec::new(); + let mut idx = 1usize; + + macro_rules! bind_eq { + ($column:literal, $value:expr) => { + if let Some(value) = $value { + sql.push_str(&format!(" AND {} = ?{idx}", $column)); + bindings.push(rusqlite::types::Value::Text(value.clone())); + idx += 1; + } + }; + } + bind_eq!("hook_event", ¶ms.hook_event); + bind_eq!("hook_name", ¶ms.hook_name); + bind_eq!("hook_source", ¶ms.hook_source); + bind_eq!("status", ¶ms.status); + bind_eq!("evidence_kind", ¶ms.evidence_kind); + bind_eq!("ai_tool", ¶ms.tool); + bind_eq!("ai_project", ¶ms.project); + bind_eq!("ai_session_id", ¶ms.session_id); + bind_eq!("hostname", ¶ms.hostname); + if let Some(from) = ¶ms.from { + sql.push_str(&format!(" AND timestamp >= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(from.clone())); + idx += 1; + } + if let Some(to) = ¶ms.to { + sql.push_str(&format!(" AND timestamp <= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(to.clone())); + idx += 1; + } + let _ = idx; + sql.push_str(&format!( + " ORDER BY timestamp DESC, id DESC LIMIT {}", + limit + 1 + )); + + let mut stmt = conn.prepare(&sql)?; + let mut rows = stmt + .query_map( + rusqlite::params_from_iter(bindings.iter()), + map_hook_event_row, + )? + .collect::>>()?; + + let truncated = rows.len() > limit; + rows.truncate(limit); + Ok(ListHookEventsResult { + total: rows.len(), + truncated, + events: rows, + }) +} + +#[cfg(test)] +#[path = "hook_events_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/hook_events_tests.rs b/crates/shared/cortex/storage-sqlite/src/hook_events_tests.rs new file mode 100644 index 00000000..d717ee43 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/hook_events_tests.rs @@ -0,0 +1,234 @@ +use super::*; +use crate::config::StorageConfig; +use crate::inputs::{ExtractedHookEvent, HookEvidenceKind, HookStatus}; +use crate::pool::init_pool; + +fn test_pool() -> (crate::DbPool, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let pool = init_pool(&StorageConfig::for_test(db_path)).unwrap(); + (pool, dir) +} + +fn insert_log_row(pool: &crate::DbPool, hostname: &str, timestamp: &str) -> i64 { + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO logs (timestamp, hostname, severity, message, raw, source_ip) + VALUES (?1, ?2, 'info', 'msg', 'raw', 'transcript://claude_project')", + rusqlite::params![timestamp, hostname], + ) + .unwrap(); + conn.last_insert_rowid() +} + +fn sample_event(hook_name: &str) -> ExtractedHookEvent { + ExtractedHookEvent { + hook_event: "PostToolUse".to_string(), + hook_name: Some(hook_name.to_string()), + hook_source: None, + hook_command: Some("cargo fmt".to_string()), + status: HookStatus::Success, + exit_code: Some(0), + duration_ms: Some(120), + stdout_preview: Some("ok".to_string()), + stderr_preview: None, + persisted_output_path: None, + trusted_hash: None, + evidence_kind: HookEvidenceKind::RuntimeTranscript, + metadata_json: None, + } +} + +#[test] +fn insert_and_list_round_trips() { + let (pool, _dir) = test_pool(); + let log_id = insert_log_row(&pool, "devhost", "2026-06-01T00:00:00.000Z"); + let insert = HookEventInsert { + log_id: Some(log_id), + ai_tool: "claude".to_string(), + ai_project: Some("cortex".to_string()), + ai_session_id: Some("sess-1".to_string()), + hostname: "devhost".to_string(), + timestamp: "2026-06-01T00:00:00.000Z".to_string(), + event: sample_event("format-on-save"), + }; + let inserted = insert_hook_events(&pool, &[insert]).unwrap(); + assert_eq!(inserted, 1); + + let result = list_hook_events(&pool, &AiHookEventParams::default()).unwrap(); + assert_eq!(result.total, 1); + assert_eq!( + result.events[0].hook_name.as_deref(), + Some("format-on-save") + ); + assert_eq!(result.events[0].hook_event, "PostToolUse"); + assert_eq!(result.events[0].status, "success"); + assert_eq!(result.events[0].evidence_kind, "runtime_transcript"); + assert_eq!(result.events[0].log_id, Some(log_id)); +} + +#[test] +fn insert_or_ignore_is_idempotent_on_duplicate() { + let (pool, _dir) = test_pool(); + let log_id = insert_log_row(&pool, "devhost", "2026-06-01T00:00:00.000Z"); + let insert = HookEventInsert { + log_id: Some(log_id), + ai_tool: "claude".to_string(), + ai_project: None, + ai_session_id: Some("sess-1".to_string()), + hostname: "devhost".to_string(), + timestamp: "2026-06-01T00:00:00.000Z".to_string(), + event: sample_event("format-on-save"), + }; + assert_eq!( + insert_hook_events(&pool, std::slice::from_ref(&insert)).unwrap(), + 1 + ); + assert_eq!(insert_hook_events(&pool, &[insert]).unwrap(), 0); + + let result = list_hook_events(&pool, &AiHookEventParams::default()).unwrap(); + assert_eq!(result.total, 1); +} + +#[test] +fn insert_succeeds_without_log_id_for_config_inventory_rows() { + let (pool, _dir) = test_pool(); + let insert = HookEventInsert { + log_id: None, + ai_tool: "codex".to_string(), + ai_project: None, + ai_session_id: None, + hostname: "devhost".to_string(), + timestamp: "2026-06-01T00:00:00.000Z".to_string(), + event: ExtractedHookEvent { + hook_event: "PreToolUse".to_string(), + hook_name: Some("lint-check".to_string()), + hook_source: Some("~/.codex/hooks.json".to_string()), + hook_command: None, + status: HookStatus::Unknown, + exit_code: None, + duration_ms: None, + stdout_preview: None, + stderr_preview: None, + persisted_output_path: None, + trusted_hash: Some("abc123".to_string()), + evidence_kind: HookEvidenceKind::ConfigInventory, + metadata_json: None, + }, + }; + assert_eq!(insert_hook_events(&pool, &[insert]).unwrap(), 1); + let result = list_hook_events(&pool, &AiHookEventParams::default()).unwrap(); + assert_eq!(result.events[0].log_id, None); + assert_eq!(result.events[0].evidence_kind, "config_inventory"); + assert_eq!(result.events[0].trusted_hash.as_deref(), Some("abc123")); +} + +#[test] +fn list_filters_by_hook_name_project_and_tool() { + let (pool, _dir) = test_pool(); + let log_id_a = insert_log_row(&pool, "devhost", "2026-06-01T00:00:00.000Z"); + let log_id_b = insert_log_row(&pool, "nashost", "2026-06-01T01:00:00.000Z"); + insert_hook_events( + &pool, + &[ + HookEventInsert { + log_id: Some(log_id_a), + ai_tool: "claude".to_string(), + ai_project: Some("cortex".to_string()), + ai_session_id: Some("sess-a".to_string()), + hostname: "devhost".to_string(), + timestamp: "2026-06-01T00:00:00.000Z".to_string(), + event: sample_event("format-on-save"), + }, + HookEventInsert { + log_id: Some(log_id_b), + ai_tool: "codex".to_string(), + ai_project: Some("axon".to_string()), + ai_session_id: Some("sess-b".to_string()), + hostname: "nashost".to_string(), + timestamp: "2026-06-01T01:00:00.000Z".to_string(), + event: sample_event("lint-check"), + }, + ], + ) + .unwrap(); + + let result = list_hook_events( + &pool, + &AiHookEventParams { + project: Some("cortex".to_string()), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(result.total, 1); + assert_eq!( + result.events[0].hook_name.as_deref(), + Some("format-on-save") + ); + + let result = list_hook_events( + &pool, + &AiHookEventParams { + tool: Some("codex".to_string()), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(result.total, 1); + assert_eq!(result.events[0].ai_tool, "codex"); +} + +#[test] +fn list_filters_by_status_and_evidence_kind() { + let (pool, _dir) = test_pool(); + let log_id = insert_log_row(&pool, "devhost", "2026-06-01T00:00:00.000Z"); + let mut failed = sample_event("lint-check"); + failed.status = HookStatus::Failed; + failed.hook_event = "PreToolUse".to_string(); + insert_hook_events( + &pool, + &[ + HookEventInsert { + log_id: Some(log_id), + ai_tool: "claude".to_string(), + ai_project: Some("cortex".to_string()), + ai_session_id: Some("sess-a".to_string()), + hostname: "devhost".to_string(), + timestamp: "2026-06-01T00:00:00.000Z".to_string(), + event: sample_event("format-on-save"), + }, + HookEventInsert { + log_id: Some(log_id), + ai_tool: "claude".to_string(), + ai_project: Some("cortex".to_string()), + ai_session_id: Some("sess-a".to_string()), + hostname: "devhost".to_string(), + timestamp: "2026-06-01T00:00:01.000Z".to_string(), + event: failed, + }, + ], + ) + .unwrap(); + + let result = list_hook_events( + &pool, + &AiHookEventParams { + status: Some("failed".to_string()), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(result.total, 1); + assert_eq!(result.events[0].status, "failed"); + + let result = list_hook_events( + &pool, + &AiHookEventParams { + evidence_kind: Some("runtime_transcript".to_string()), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(result.total, 2); +} diff --git a/crates/shared/cortex/storage-sqlite/src/hook_incident_evidence.rs b/crates/shared/cortex/storage-sqlite/src/hook_incident_evidence.rs new file mode 100644 index 00000000..02ea4b04 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/hook_incident_evidence.rs @@ -0,0 +1,335 @@ +//! Investigation evidence-bundle layer for hook incidents. Expands a +//! `HookIncident` (grouped/scored in `src/db/hook_incidents.rs`) into a +//! bounded, truncation-flagged evidence bundle: the underlying hook events, +//! the transcript rows that triggered the `user_correction_after_hook` +//! anchor, transcript context before/after, and nearby non-AI logs split +//! into tool-call/user-correction/error subsets. Mirrors +//! `src/db/skill_incident_evidence.rs` but keyed on hook usage instead of +//! skill usage. + +use anyhow::Result; +use serde::{Deserialize, Serialize}; + +use cortex_domain::skill_signal_detectors::detect_tool_failure; + +use super::hook_events::{AiHookEventEntry, map_hook_event_row}; +use super::hook_incidents::{AiHookIncidentParams, HookIncident, search_ai_hook_incidents}; +use super::models::LogEntry; +use super::pool::DbPool; +use super::queries::map_row; + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct AiHookInvestigateParams { + pub incident_id: Option, + pub hook_event: Option, + pub hook_name: Option, + pub hook_source: Option, + pub ai_tool: Option, + pub ai_project: Option, + pub since: Option, + pub until: Option, + /// Max incidents to investigate. Default 3, clamp 1..=10. + pub limit: Option, + /// Incident grouping window minutes. Default 10, clamp 1..=120. + pub window_minutes: Option, + /// Correlation window minutes around incident. Default 5, clamp 1..=120. + pub correlation_window_minutes: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HookIncidentEvidence { + pub incident: HookIncident, + /// The `ai_hook_events` rows in this group, capped at 25. + pub hook_events: Vec, + pub hook_events_truncated: bool, + /// Transcript rows that triggered the `user_correction_after_hook` + /// anchor, capped at 50. + pub signal_anchors: Vec, + pub signal_anchors_truncated: bool, + /// Same-session transcript entries before the first hook event, capped 20. + pub transcript_before: Vec, + pub transcript_before_truncated: bool, + /// Same-session transcript entries after the last hook event, capped 20. + pub transcript_after: Vec, + pub transcript_after_truncated: bool, + /// Subset of nearby_logs matching tool-failure phrases, capped 25. + pub nearby_tool_calls: Vec, + pub nearby_tool_calls_truncated: bool, + /// Non-AI syslog/Docker logs in the correlation window, capped 50. + pub nearby_logs: Vec, + pub nearby_logs_truncated: bool, + /// Subset of nearby_logs with severity warning or above, capped 25. + pub nearby_errors: Vec, + pub nearby_errors_truncated: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiHookInvestigateResult { + pub evidence: Vec, + pub total_incidents: usize, + pub truncated: bool, +} + +pub fn investigate_ai_hook_incidents( + pool: &DbPool, + params: &AiHookInvestigateParams, +) -> Result { + const HOOK_EVENTS_CAP: usize = 25; + const SIGNAL_ANCHORS_CAP: usize = 50; + const TRANSCRIPT_CAP: usize = 20; + const NEARBY_CAP: usize = 50; + const NEARBY_SUBSET_CAP: usize = 25; + + let limit = params.limit.unwrap_or(3).clamp(1, 10) as usize; + let corr_mins = i64::from(params.correlation_window_minutes.unwrap_or(5).clamp(1, 120)); + + // `incident_id` is passed straight through to `AiHookIncidentParams`, + // which filters the full computed incident set (bounded only by + // `HOOK_INCIDENT_CANDIDATE_CAP` events, not an incident-count cap) + // before its own priority-ranked truncation. This guarantees an exact + // incident_id lookup finds its target regardless of priority rank — + // routing it through a fixed-size top-N candidate window (as a prior + // version of this code did) could silently miss incidents ranked + // below that window. + let incident_result = search_ai_hook_incidents( + pool, + &AiHookIncidentParams { + hook_event: params.hook_event.clone(), + hook_name: params.hook_name.clone(), + hook_source: params.hook_source.clone(), + ai_tool: params.ai_tool.clone(), + ai_project: params.ai_project.clone(), + ai_session_id: None, + hostname: None, + evidence_kind: None, + since: params.since.clone(), + until: params.until.clone(), + incident_id: params.incident_id.clone(), + limit: Some(limit as u32), + window_minutes: params.window_minutes, + signals: Vec::new(), + min_score: None, + }, + )?; + let total_incidents = incident_result.total_incidents; + let truncated = incident_result.truncated; + let mut incidents = incident_result.incidents; + incidents.truncate(limit); + + let conn = pool.get()?; + let mut evidence = Vec::with_capacity(incidents.len()); + + for incident in incidents { + // ── Hook events for this group ────────────────────────────────── + let (hook_events, hook_events_truncated) = if incident.hook_event_ids.is_empty() { + (Vec::new(), false) + } else { + let placeholders: Vec = (1..=incident.hook_event_ids.len()) + .map(|i| format!("?{i}")) + .collect(); + let sql = format!( + "SELECT id, log_id, ai_tool, ai_project, ai_session_id, hostname, timestamp, + hook_event, hook_name, hook_source, hook_command, status, exit_code, + duration_ms, stdout_preview, stderr_preview, persisted_output_path, + trusted_hash, evidence_kind, metadata_json + FROM ai_hook_events WHERE id IN ({}) ORDER BY timestamp ASC", + placeholders.join(",") + ); + let mut stmt = conn.prepare(&sql)?; + let rows: Vec = stmt + .query_map( + rusqlite::params_from_iter( + incident + .hook_event_ids + .iter() + .map(|id| rusqlite::types::Value::Integer(*id)), + ), + map_hook_event_row, + )? + .collect::>>()?; + let truncated = rows.len() > HOOK_EVENTS_CAP; + let mut out = rows; + out.truncate(HOOK_EVENTS_CAP); + (out, truncated) + }; + + // ── Signal anchor log rows (user_correction_after_hook) ───────── + let (signal_anchors, signal_anchors_truncated) = if incident.anchor_log_ids.is_empty() { + (Vec::new(), false) + } else { + let placeholders: Vec = (1..=incident.anchor_log_ids.len()) + .map(|i| format!("?{i}")) + .collect(); + let sql = format!( + "SELECT id, timestamp, hostname, facility, severity, app_name, + process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs WHERE id IN ({}) ORDER BY timestamp ASC", + placeholders.join(",") + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt + .query_map( + rusqlite::params_from_iter( + incident + .anchor_log_ids + .iter() + .map(|id| rusqlite::types::Value::Integer(*id)), + ), + map_row, + )? + .collect::>>()?; + let truncated = rows.len() > SIGNAL_ANCHORS_CAP; + let mut out = rows; + out.truncate(SIGNAL_ANCHORS_CAP); + (out, truncated) + }; + + // ── Transcript before/after ────────────────────────────────────── + let (transcript_before, transcript_before_truncated) = { + let mut stmt = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, app_name, + process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs + WHERE ai_session_id = ?1 AND ai_project = ?2 AND ai_tool = ?3 + AND timestamp < ?4 + ORDER BY timestamp DESC + LIMIT 21", + )?; + let rows = stmt + .query_map( + rusqlite::params![ + &incident.session_id, + &incident.project, + &incident.tool, + &incident.first_seen, + ], + map_row, + )? + .collect::>>()?; + let truncated = rows.len() > TRANSCRIPT_CAP; + let mut out = rows; + out.truncate(TRANSCRIPT_CAP); + out.reverse(); + (out, truncated) + }; + + let (transcript_after, transcript_after_truncated) = { + let mut stmt = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, app_name, + process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs + WHERE ai_session_id = ?1 AND ai_project = ?2 AND ai_tool = ?3 + AND timestamp > ?4 + ORDER BY timestamp ASC + LIMIT 21", + )?; + let rows = stmt + .query_map( + rusqlite::params![ + &incident.session_id, + &incident.project, + &incident.tool, + &incident.last_seen, + ], + map_row, + )? + .collect::>>()?; + let truncated = rows.len() > TRANSCRIPT_CAP; + let mut out = rows; + out.truncate(TRANSCRIPT_CAP); + (out, truncated) + }; + + // ── Nearby non-AI logs in the correlation window ──────────────── + let (nearby_logs, nearby_logs_truncated) = { + let win_from = chrono::DateTime::parse_from_rfc3339(&incident.first_seen) + .map(|dt| { + (dt.with_timezone(&chrono::Utc) - chrono::Duration::minutes(corr_mins)) + .format("%Y-%m-%dT%H:%M:%S%.3fZ") + .to_string() + }) + .unwrap_or_else(|_| incident.first_seen.clone()); + let win_to = chrono::DateTime::parse_from_rfc3339(&incident.last_seen) + .map(|dt| { + (dt.with_timezone(&chrono::Utc) + chrono::Duration::minutes(corr_mins)) + .format("%Y-%m-%dT%H:%M:%S%.3fZ") + .to_string() + }) + .unwrap_or_else(|_| incident.last_seen.clone()); + + let mut stmt = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, app_name, + process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs + WHERE timestamp >= ?1 AND timestamp <= ?2 AND hostname = ?3 + ORDER BY timestamp ASC + LIMIT 51", + )?; + let rows = stmt + .query_map( + rusqlite::params![win_from, win_to, &incident.hostname], + map_row, + )? + .collect::>>()?; + let truncated = rows.len() > NEARBY_CAP; + let mut out = rows; + out.truncate(NEARBY_CAP); + (out, truncated) + }; + + // ── Derived subsets: tool calls (failure phrases), errors ─────── + let mut nearby_tool_calls: Vec = nearby_logs + .iter() + .filter(|e| detect_tool_failure(&e.message)) + .cloned() + .collect(); + let nearby_tool_calls_truncated = nearby_tool_calls.len() > NEARBY_SUBSET_CAP; + nearby_tool_calls.truncate(NEARBY_SUBSET_CAP); + + let error_sevs = ["emergency", "alert", "critical", "error", "warning"]; + let mut nearby_errors: Vec = nearby_logs + .iter() + .filter(|e| error_sevs.contains(&e.severity.as_str())) + .cloned() + .collect(); + let nearby_errors_truncated = nearby_errors.len() > NEARBY_SUBSET_CAP; + nearby_errors.truncate(NEARBY_SUBSET_CAP); + + // Signal anchors already restricted to user_correction_after_hook + // rows in `search_ai_hook_incidents`; nothing else to derive here + // beyond the two subsets above. (No separate `nearby_user_corrections` + // subset like the skill-incident evidence bundle has — the anchor + // rows already *are* the correction evidence for hooks.) + evidence.push(HookIncidentEvidence { + incident, + hook_events, + hook_events_truncated, + signal_anchors, + signal_anchors_truncated, + transcript_before, + transcript_before_truncated, + transcript_after, + transcript_after_truncated, + nearby_tool_calls, + nearby_tool_calls_truncated, + nearby_logs, + nearby_logs_truncated, + nearby_errors, + nearby_errors_truncated, + }); + } + + Ok(AiHookInvestigateResult { + evidence, + total_incidents, + truncated, + }) +} + +#[cfg(test)] +#[path = "hook_incident_evidence_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/hook_incident_evidence_tests.rs b/crates/shared/cortex/storage-sqlite/src/hook_incident_evidence_tests.rs new file mode 100644 index 00000000..a596d541 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/hook_incident_evidence_tests.rs @@ -0,0 +1,372 @@ +use super::*; +use crate::config::StorageConfig; +use crate::pool::init_pool; +use crate::{DbPool, LogBatchEntry, insert_logs_batch}; + +fn test_pool() -> (DbPool, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let pool = init_pool(&StorageConfig::for_test(db_path)).unwrap(); + (pool, dir) +} + +fn make_ai_entry( + ts: &str, + host: &str, + tool: &str, + project: &str, + session_id: &str, + message: &str, +) -> LogBatchEntry { + LogBatchEntry { + timestamp: ts.to_string(), + hostname: host.to_string(), + facility: Some("local0".to_string()), + severity: "info".to_string(), + app_name: Some("ai-transcript".to_string()), + process_id: None, + message: message.to_string(), + raw: message.to_string(), + source_ip: "127.0.0.1:514".to_string(), + docker_checkpoint: None, + ai_tool: Some(tool.to_string()), + ai_project: Some(project.to_string()), + ai_session_id: Some(session_id.to_string()), + ai_transcript_path: Some(format!("{project}/{session_id}.jsonl")), + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + } +} + +#[allow(clippy::too_many_arguments)] +fn insert_hook_event_row( + pool: &DbPool, + log_id: Option, + ai_tool: &str, + ai_project: &str, + ai_session_id: &str, + hostname: &str, + timestamp: &str, + hook_event: &str, + hook_name: &str, + status: &str, + evidence_kind: &str, +) { + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO ai_hook_events + (log_id, ai_tool, ai_project, ai_session_id, hostname, timestamp, + hook_event, hook_name, status, evidence_kind) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", + rusqlite::params![ + log_id, + ai_tool, + ai_project, + ai_session_id, + hostname, + timestamp, + hook_event, + hook_name, + status, + evidence_kind, + ], + ) + .unwrap(); +} + +#[test] +fn investigate_returns_bounded_evidence_bundle_with_findings_ready_data() { + let (pool, _dir) = test_pool(); + let entries = vec![ + make_ai_entry( + "2026-01-01T00:00:00.000Z", + "devhost", + "claude", + "/home/jmagar/workspace/cortex", + "sess-1", + "starting work", + ), + make_ai_entry( + "2026-01-01T00:00:10.000Z", + "devhost", + "claude", + "/home/jmagar/workspace/cortex", + "sess-1", + "after hook context, exit code nonzero", + ), + ]; + insert_logs_batch(&pool, &entries).unwrap(); + let log_ids: Vec = { + let conn = pool.get().unwrap(); + let mut stmt = conn.prepare("SELECT id FROM logs ORDER BY id ASC").unwrap(); + stmt.query_map([], |row| row.get::<_, i64>(0)) + .unwrap() + .collect::>>() + .unwrap() + }; + + insert_hook_event_row( + &pool, + Some(log_ids[0]), + "claude", + "/home/jmagar/workspace/cortex", + "sess-1", + "devhost", + "2026-01-01T00:00:05.000Z", + "PostToolUse", + "format-on-save", + "failed", + "runtime_transcript", + ); + + let result = investigate_ai_hook_incidents( + &pool, + &AiHookInvestigateParams { + hook_name: Some("format-on-save".to_string()), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.evidence.len(), 1); + let bundle = &result.evidence[0]; + assert_eq!(bundle.incident.hook_name.as_deref(), Some("format-on-save")); + assert_eq!(bundle.hook_events.len(), 1); + assert!(!bundle.hook_events_truncated); + assert!(!bundle.transcript_before.is_empty() || !bundle.transcript_after.is_empty()); +} + +#[test] +fn investigate_by_incident_id_narrows_to_one() { + let (pool, _dir) = test_pool(); + insert_hook_event_row( + &pool, + None, + "claude", + "/home/jmagar/workspace/cortex", + "sess-a", + "devhost", + "2026-01-01T00:00:00.000Z", + "PostToolUse", + "hook-a", + "success", + "runtime_transcript", + ); + insert_hook_event_row( + &pool, + None, + "claude", + "/home/jmagar/workspace/cortex", + "sess-b", + "devhost", + "2026-01-01T01:00:00.000Z", + "PostToolUse", + "hook-b", + "failed", + "runtime_transcript", + ); + + let all = search_ai_hook_incidents(&pool, &AiHookIncidentParams::default()).unwrap(); + assert_eq!(all.incidents.len(), 2); + let target_id = all.incidents[0].incident_id.clone(); + + let result = investigate_ai_hook_incidents( + &pool, + &AiHookInvestigateParams { + incident_id: Some(target_id.clone()), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(result.evidence.len(), 1); + assert_eq!(result.evidence[0].incident.incident_id, target_id); +} + +#[test] +fn investigate_with_no_matching_hook_returns_empty_evidence() { + let (pool, _dir) = test_pool(); + let result = investigate_ai_hook_incidents( + &pool, + &AiHookInvestigateParams { + hook_name: Some("nonexistent-hook".to_string()), + ..Default::default() + }, + ) + .unwrap(); + assert!(result.evidence.is_empty()); + assert_eq!(result.total_incidents, 0); +} + +/// Regression test for a bug where an exact `incident_id` lookup routed +/// through `search_ai_hook_incidents` with `limit: Some(100)` and then +/// filtered client-side for the matching id — if the target incident ranked +/// below the top 100 by priority score, investigation silently returned +/// empty evidence for an incident that actually existed. This constructs +/// 100 higher-scored decoy incidents (failed hook status) plus one +/// lower-scored target (successful hook status) so the target provably +/// ranks outside any top-100 window, then asserts the exact lookup still +/// finds it. +#[test] +fn investigate_ai_hook_incidents_exact_incident_id_beyond_top_100_candidates() { + let (pool, _dir) = test_pool(); + + for i in 0..100 { + insert_hook_event_row( + &pool, + None, + "claude", + "/tmp/project-g", + &format!("sess-decoy-{i:03}"), + "host-a", + "2026-01-01T00:00:00.000Z", + "PostToolUse", + "format-on-save", + "failed", + "runtime_transcript", + ); + } + + // Target group: successful status, no failure signal, guaranteeing it + // ranks last among the 101 total matching incidents. + let target_session_id = "sess-target"; + insert_hook_event_row( + &pool, + None, + "claude", + "/tmp/project-g", + target_session_id, + "host-a", + "2026-01-01T00:00:00.000Z", + "PostToolUse", + "format-on-save", + "success", + "runtime_transcript", + ); + + let target_lookup = search_ai_hook_incidents( + &pool, + &AiHookIncidentParams { + hook_name: Some("format-on-save".to_string()), + ai_session_id: Some(target_session_id.to_string()), + limit: Some(1), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(target_lookup.incidents.len(), 1); + let target_id = target_lookup.incidents[0].incident_id.clone(); + + let top100 = search_ai_hook_incidents( + &pool, + &AiHookIncidentParams { + hook_name: Some("format-on-save".to_string()), + limit: Some(100), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(top100.total_incidents, 101, "100 decoys + 1 target"); + assert_eq!(top100.incidents.len(), 100); + assert!( + !top100 + .incidents + .iter() + .any(|inc| inc.incident_id == target_id), + "test setup invariant: target must rank outside the top 100" + ); + + // The regression check: an exact incident_id lookup must still find the + // target even though it ranks outside the top-100 candidate window. + let exact = investigate_ai_hook_incidents( + &pool, + &AiHookInvestigateParams { + incident_id: Some(target_id.clone()), + hook_name: Some("format-on-save".to_string()), + limit: Some(1), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!( + exact.evidence.len(), + 1, + "exact incident_id lookup must find an incident ranked outside the top 100" + ); + assert_eq!(exact.evidence[0].incident.incident_id, target_id); +} + +/// Regression test for a bug where the `nearby_logs` query only filtered by +/// timestamp range, with no hostname scope, so an incident on one host could +/// pull in unrelated log rows from a different host in the same time window. +#[test] +fn investigate_ai_hook_incidents_nearby_logs_scoped_to_incident_hostname() { + let (pool, _dir) = test_pool(); + + insert_hook_event_row( + &pool, + None, + "claude", + "/tmp/project-h", + "sess-h", + "host-a", + "2026-01-01T00:00:00.000Z", + "PostToolUse", + "format-on-save", + "success", + "runtime_transcript", + ); + + // Unrelated non-AI log on a DIFFERENT host, within the correlation window. + let other_host_log = LogBatchEntry { + timestamp: "2026-01-01T00:01:00Z".to_string(), + hostname: "host-b".to_string(), + facility: Some("local0".to_string()), + severity: "error".to_string(), + app_name: Some("nginx".to_string()), + process_id: None, + message: "connection refused".to_string(), + raw: "connection refused".to_string(), + source_ip: "10.0.0.5:514".to_string(), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + }; + insert_logs_batch(&pool, &[other_host_log]).unwrap(); + + let result = investigate_ai_hook_incidents( + &pool, + &AiHookInvestigateParams { + hook_name: Some("format-on-save".to_string()), + limit: Some(1), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.evidence.len(), 1); + let bundle = &result.evidence[0]; + assert!( + bundle.nearby_logs.iter().all(|e| e.hostname == "host-a"), + "nearby_logs leaked a cross-host row: {:?}", + bundle.nearby_logs + ); + assert!( + !bundle + .nearby_logs + .iter() + .any(|e| e.message.contains("connection refused")), + "cross-host log should not appear in nearby_logs" + ); +} diff --git a/crates/shared/cortex/storage-sqlite/src/hook_incidents.rs b/crates/shared/cortex/storage-sqlite/src/hook_incidents.rs new file mode 100644 index 00000000..2f85800c --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/hook_incidents.rs @@ -0,0 +1,432 @@ +//! Hook-usage incident grouping and scoring. Groups `ai_hook_events` rows +//! into `HookIncident`s by `(hook_event, hook_name, hook_source, ai_tool, +//! ai_project, ai_session_id, hostname, window_bucket)`, scans nearby +//! transcript logs for the `user_correction_after_hook` anchor and derives +//! the remaining anchors directly from the hook event rows themselves +//! (failure status, high duration, output-parse-error phrases, invocation +//! frequency). Mirrors `src/db/skill_incidents.rs`'s grouping query but keyed +//! on hook usage instead of skill usage. + +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +use cortex_domain::hook_signal_detectors::{ + detect_hook_invoked_too_often, detect_hook_output_parse_error, detect_user_correction, + is_hook_failure_status, is_hook_timeout, +}; +pub use cortex_domain::{HookIncident, HookSignalCounts}; + +use super::pool::DbPool; + +// --------------------------------------------------------------------------- +// Hook incident grouping +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct AiHookIncidentParams { + pub hook_event: Option, + pub hook_name: Option, + pub hook_source: Option, + pub ai_tool: Option, + pub ai_project: Option, + pub ai_session_id: Option, + pub hostname: Option, + /// Restrict candidate hook events to a single `evidence_kind` (e.g. + /// `"runtime_transcript"` to only consider proven-executed hooks). + /// `None` = no filter (all evidence kinds considered). + pub evidence_kind: Option, + pub since: Option, + pub until: Option, + /// Exact incident_id match. When set, filters the full computed incident + /// set (bounded only by `HOOK_INCIDENT_CANDIDATE_CAP`, not `limit`) + /// before the priority-ranked truncation, so a match ranked below + /// `limit` is still found. + pub incident_id: Option, + /// Max incidents to return. Default 20, clamp 1..=100. + pub limit: Option, + /// Grouping window in minutes. Default 10, clamp 1..=120. + pub window_minutes: Option, + /// Restrict to incidents containing at least one of these signal + /// categories. Empty = no filter (all incidents). + pub signals: Vec, + /// Minimum `priority_score` (inclusive). `None` = no filter. + pub min_score: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiHookIncidentResult { + pub incidents: Vec, + pub total_incidents: usize, + pub candidate_event_rows: usize, + pub candidate_cap: usize, + pub candidate_window_truncated: bool, + pub truncated: bool, +} + +const HOOK_INCIDENT_CANDIDATE_CAP: usize = 10_000; + +/// Grouping key for hook incidents: `(hook_event, hook_name, hook_source, +/// tool, project, session_id, hostname, window_bucket)`. +/// `window_bucket = unix_secs / window_secs * window_secs` (floor to window +/// boundary), mirroring `search_ai_skill_incidents`'s grouping. +pub fn search_ai_hook_incidents( + pool: &DbPool, + params: &AiHookIncidentParams, +) -> Result { + let conn = pool.get()?; + let limit = params.limit.unwrap_or(20).clamp(1, 100) as usize; + let window_secs = i64::from(params.window_minutes.unwrap_or(10).clamp(1, 120)) * 60; + + struct HookEventRow { + id: i64, + timestamp: String, + hostname: String, + tool: String, + project: String, + session_id: String, + hook_event: String, + hook_name: Option, + hook_source: Option, + status: String, + duration_ms: Option, + stdout_preview: Option, + stderr_preview: Option, + evidence_kind: String, + } + + let mut sql = String::from( + "SELECT id, timestamp, hostname, ai_tool, ai_project, ai_session_id, + hook_event, hook_name, hook_source, status, duration_ms, + stdout_preview, stderr_preview, evidence_kind + FROM ai_hook_events + WHERE 1 = 1", + ); + let mut bindings: Vec = Vec::new(); + let mut idx = 1usize; + if let Some(v) = ¶ms.hook_event { + sql.push_str(&format!(" AND hook_event = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(v.clone())); + idx += 1; + } + if let Some(v) = ¶ms.hook_name { + sql.push_str(&format!(" AND hook_name = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(v.clone())); + idx += 1; + } + if let Some(v) = ¶ms.hook_source { + sql.push_str(&format!(" AND hook_source = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(v.clone())); + idx += 1; + } + if let Some(v) = ¶ms.ai_tool { + sql.push_str(&format!(" AND ai_tool = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(v.clone())); + idx += 1; + } + if let Some(v) = ¶ms.ai_project { + sql.push_str(&format!(" AND ai_project = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(v.clone())); + idx += 1; + } + if let Some(v) = ¶ms.ai_session_id { + sql.push_str(&format!(" AND ai_session_id = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(v.clone())); + idx += 1; + } + if let Some(v) = ¶ms.hostname { + sql.push_str(&format!(" AND hostname = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(v.clone())); + idx += 1; + } + if let Some(v) = ¶ms.evidence_kind { + sql.push_str(&format!(" AND evidence_kind = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(v.clone())); + idx += 1; + } + if let Some(from) = ¶ms.since { + sql.push_str(&format!(" AND timestamp >= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(from.clone())); + idx += 1; + } + if let Some(to) = ¶ms.until { + sql.push_str(&format!(" AND timestamp <= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(to.clone())); + } + let _ = idx; + sql.push_str(&format!( + " ORDER BY timestamp ASC LIMIT {}", + HOOK_INCIDENT_CANDIDATE_CAP + 1 + )); + + let mut stmt = conn.prepare(&sql)?; + let candidate_events: Vec = stmt + .query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + Ok(HookEventRow { + id: row.get(0)?, + timestamp: row.get(1)?, + hostname: row.get(2)?, + tool: row.get(3)?, + project: row.get(4)?, + session_id: row.get(5)?, + hook_event: row.get(6)?, + hook_name: row.get(7)?, + hook_source: row.get(8)?, + status: row.get(9)?, + duration_ms: row.get(10)?, + stdout_preview: row.get(11)?, + stderr_preview: row.get(12)?, + evidence_kind: row.get(13)?, + }) + })? + .collect::>>()?; + + let candidate_window_truncated = candidate_events.len() > HOOK_INCIDENT_CANDIDATE_CAP; + let raw_candidate_count = candidate_events.len(); + + // ── Group by (hook_event, hook_name, hook_source, tool, project, + // session_id, hostname, window_bucket) ───────────────────────────────── + type GroupKey = ( + String, + Option, + Option, + String, + String, + String, + String, + i64, + ); + let mut groups: HashMap> = HashMap::new(); + + for row in candidate_events.iter().take(HOOK_INCIDENT_CANDIDATE_CAP) { + let bucket = chrono::DateTime::parse_from_rfc3339(&row.timestamp) + .map(|dt| (dt.timestamp() / window_secs) * window_secs) + .unwrap_or(0); + let key = ( + row.hook_event.clone(), + row.hook_name.clone(), + row.hook_source.clone(), + row.tool.clone(), + row.project.clone(), + row.session_id.clone(), + row.hostname.clone(), + bucket, + ); + groups.entry(key).or_default().push(row); + } + + let mut incidents: Vec = Vec::with_capacity(groups.len()); + for ( + (hook_event, hook_name, hook_source, tool, project, session_id, hostname, _bucket), + events, + ) in groups + { + let first_seen = events + .first() + .map(|e| e.timestamp.clone()) + .unwrap_or_default(); + let last_seen = events + .last() + .map(|e| e.timestamp.clone()) + .unwrap_or_default(); + let duration_secs = { + let t0 = chrono::DateTime::parse_from_rfc3339(&first_seen) + .map(|dt| dt.timestamp()) + .unwrap_or(0); + let t1 = chrono::DateTime::parse_from_rfc3339(&last_seen) + .map(|dt| dt.timestamp()) + .unwrap_or(0); + (t1 - t0).max(0) + }; + + let mut counts = HookSignalCounts::default(); + let has_runtime_evidence = events + .iter() + .all(|e| e.evidence_kind == "runtime_transcript"); + + for event in &events { + if is_hook_failure_status(&event.status) { + counts.hook_failed += 1; + } + if is_hook_timeout(&event.status, event.duration_ms) { + counts.hook_timed_out += 1; + } + let preview_hit = event + .stdout_preview + .as_deref() + .is_some_and(detect_hook_output_parse_error) + || event + .stderr_preview + .as_deref() + .is_some_and(detect_hook_output_parse_error); + if preview_hit { + counts.hook_output_parse_error += 1; + } + } + if detect_hook_invoked_too_often(events.len()) { + counts.hook_invoked_too_often = events.len(); + } + + // ── user_correction_after_hook: scan nearby transcript logs in the + // session/window following the hook events, same anchor pattern as + // skill incidents ──────────────────────────────────────────────── + let win_from = first_seen.clone(); + let win_to = chrono::DateTime::parse_from_rfc3339(&last_seen) + .map(|dt| { + (dt.with_timezone(&chrono::Utc) + chrono::Duration::seconds(window_secs)) + .format("%Y-%m-%dT%H:%M:%S%.3fZ") + .to_string() + }) + .unwrap_or_else(|_| last_seen.clone()); + + let mut anchor_stmt = conn.prepare_cached( + "SELECT id, message FROM logs + WHERE ai_session_id = ?1 AND ai_project = ?2 AND ai_tool = ?3 + AND timestamp >= ?4 AND timestamp <= ?5 + ORDER BY timestamp ASC + LIMIT 500", + )?; + let anchor_rows: Vec<(i64, String)> = anchor_stmt + .query_map( + rusqlite::params![session_id, project, tool, win_from, win_to], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)), + )? + .collect::>>()?; + + let mut anchor_log_ids: Vec = Vec::new(); + for (id, message) in &anchor_rows { + if detect_user_correction(message) { + counts.user_correction_after_hook += 1; + anchor_log_ids.push(*id); + } + } + anchor_log_ids.sort_unstable(); + anchor_log_ids.dedup(); + + let mut signals_present: Vec = Vec::new(); + if counts.hook_failed > 0 { + signals_present.push("hook_failed".to_string()); + } + if counts.hook_timed_out > 0 { + signals_present.push("hook_timed_out".to_string()); + } + if counts.hook_output_parse_error > 0 { + signals_present.push("hook_output_parse_error".to_string()); + } + if counts.hook_invoked_too_often > 0 { + signals_present.push("hook_invoked_too_often".to_string()); + } + if counts.user_correction_after_hook > 0 { + signals_present.push("user_correction_after_hook".to_string()); + } + signals_present.sort(); + + // ── Locked scoring formula (mirrors skill-incident scoring shape, + // weighted for hook-specific signal severity) ────────────────────── + let signal_variety = signals_present.len() as f64; + let priority_score = events.len() as f64 * 2.0 + + counts.hook_failed as f64 * 15.0 + + counts.hook_timed_out as f64 * 10.0 + + counts.hook_output_parse_error as f64 * 10.0 + + counts.hook_invoked_too_often as f64 * 8.0 + + counts.user_correction_after_hook as f64 * 15.0 + + signal_variety * 5.0; + + let priority_label = if priority_score < 15.0 { + "low" + } else if priority_score < 35.0 { + "medium" + } else if priority_score < 60.0 { + "high" + } else { + "critical" + } + .to_string(); + + let mut hook_event_ids: Vec = events.iter().map(|e| e.id).collect(); + hook_event_ids.sort_unstable(); + + let incident_id = { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + let mut h = DefaultHasher::new(); + hook_event.hash(&mut h); + hook_name.hash(&mut h); + hook_source.hash(&mut h); + tool.hash(&mut h); + project.hash(&mut h); + session_id.hash(&mut h); + hostname.hash(&mut h); + for id in &anchor_log_ids { + id.hash(&mut h); + } + for id in &hook_event_ids { + id.hash(&mut h); + } + format!("hook-inc-{:016x}", h.finish()) + }; + + incidents.push(HookIncident { + incident_id, + hook_event, + hook_name, + hook_source, + tool, + project, + session_id, + hostname, + first_seen, + last_seen, + duration_secs, + hook_event_count: events.len(), + hook_event_ids, + anchor_log_ids, + signal_counts: counts, + signals_present, + has_runtime_evidence, + priority_score, + priority_label, + window_minutes: (window_secs / 60) as u32, + }); + } + + if let Some(incident_id) = ¶ms.incident_id { + incidents.retain(|inc| &inc.incident_id == incident_id); + } + if !params.signals.is_empty() { + incidents.retain(|inc| { + inc.signals_present + .iter() + .any(|s| params.signals.contains(s)) + }); + } + if let Some(min_score) = params.min_score { + incidents.retain(|inc| inc.priority_score >= min_score); + } + + // Sort by priority_score descending, then last_seen descending. Uses + // total_cmp (never partial_cmp/unwrap_or(Equal)) for a total order even + // if a NaN score ever appears. + incidents.sort_by(|a, b| { + b.priority_score + .total_cmp(&a.priority_score) + .then_with(|| b.last_seen.cmp(&a.last_seen)) + }); + + let total_incidents = incidents.len(); + let truncated = total_incidents > limit || candidate_window_truncated; + incidents.truncate(limit); + + Ok(AiHookIncidentResult { + incidents, + total_incidents, + candidate_event_rows: raw_candidate_count.min(HOOK_INCIDENT_CANDIDATE_CAP), + candidate_cap: HOOK_INCIDENT_CANDIDATE_CAP, + candidate_window_truncated, + truncated, + }) +} + +#[cfg(test)] +#[path = "hook_incidents_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/hook_incidents_tests.rs b/crates/shared/cortex/storage-sqlite/src/hook_incidents_tests.rs new file mode 100644 index 00000000..6117f8e0 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/hook_incidents_tests.rs @@ -0,0 +1,324 @@ +use super::*; +use crate::config::StorageConfig; +use crate::pool::init_pool; +use crate::{DbPool, LogBatchEntry, insert_logs_batch}; + +fn test_pool() -> (DbPool, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let pool = init_pool(&StorageConfig::for_test(db_path)).unwrap(); + (pool, dir) +} + +fn make_ai_entry( + ts: &str, + host: &str, + tool: &str, + project: &str, + session_id: &str, + message: &str, +) -> LogBatchEntry { + LogBatchEntry { + timestamp: ts.to_string(), + hostname: host.to_string(), + facility: Some("local0".to_string()), + severity: "info".to_string(), + app_name: Some("ai-transcript".to_string()), + process_id: None, + message: message.to_string(), + raw: message.to_string(), + source_ip: "127.0.0.1:514".to_string(), + docker_checkpoint: None, + ai_tool: Some(tool.to_string()), + ai_project: Some(project.to_string()), + ai_session_id: Some(session_id.to_string()), + ai_transcript_path: Some(format!("{project}/{session_id}.jsonl")), + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + } +} + +#[allow(clippy::too_many_arguments)] +fn insert_hook_event_row( + pool: &DbPool, + log_id: Option, + ai_tool: &str, + ai_project: &str, + ai_session_id: &str, + hostname: &str, + timestamp: &str, + hook_event: &str, + hook_name: &str, + status: &str, + duration_ms: Option, + evidence_kind: &str, +) { + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO ai_hook_events + (log_id, ai_tool, ai_project, ai_session_id, hostname, timestamp, + hook_event, hook_name, status, duration_ms, evidence_kind) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11)", + rusqlite::params![ + log_id, + ai_tool, + ai_project, + ai_session_id, + hostname, + timestamp, + hook_event, + hook_name, + status, + duration_ms, + evidence_kind, + ], + ) + .unwrap(); +} + +#[test] +fn search_ai_hook_incidents_groups_and_scores_failures() { + let (pool, _dir) = test_pool(); + + insert_hook_event_row( + &pool, + None, + "claude", + "/home/jmagar/workspace/cortex", + "sess-hook-1", + "devhost", + "2026-01-01T00:00:00.000Z", + "PostToolUse", + "format-on-save", + "failed", + None, + "runtime_transcript", + ); + insert_hook_event_row( + &pool, + None, + "claude", + "/home/jmagar/workspace/cortex", + "sess-hook-1", + "devhost", + "2026-01-01T00:00:05.000Z", + "PostToolUse", + "format-on-save", + "failed", + None, + "runtime_transcript", + ); + + let result = search_ai_hook_incidents(&pool, &AiHookIncidentParams::default()).unwrap(); + assert_eq!(result.incidents.len(), 1); + let incident = &result.incidents[0]; + assert_eq!(incident.hook_name.as_deref(), Some("format-on-save")); + assert_eq!(incident.hook_event_count, 2); + assert_eq!(incident.signal_counts.hook_failed, 2); + assert!( + incident + .signals_present + .contains(&"hook_failed".to_string()) + ); + assert!(incident.has_runtime_evidence); + assert!(incident.priority_score > 0.0); + assert!(incident.incident_id.starts_with("hook-inc-")); +} + +#[test] +fn config_only_incident_has_runtime_evidence_false() { + let (pool, _dir) = test_pool(); + insert_hook_event_row( + &pool, + None, + "codex", + "/home/jmagar/workspace/cortex", + "sess-hook-2", + "devhost", + "2026-01-01T00:00:00.000Z", + "PreToolUse", + "lint-check", + "unknown", + None, + "config_inventory", + ); + + let result = search_ai_hook_incidents(&pool, &AiHookIncidentParams::default()).unwrap(); + assert_eq!(result.incidents.len(), 1); + assert!(!result.incidents[0].has_runtime_evidence); +} + +#[test] +fn timeout_signal_detected_from_high_duration() { + let (pool, _dir) = test_pool(); + insert_hook_event_row( + &pool, + None, + "claude", + "/home/jmagar/workspace/cortex", + "sess-hook-3", + "devhost", + "2026-01-01T00:00:00.000Z", + "PostToolUse", + "slow-hook", + "success", + Some(45_000), + "runtime_transcript", + ); + + let result = search_ai_hook_incidents(&pool, &AiHookIncidentParams::default()).unwrap(); + assert_eq!(result.incidents.len(), 1); + assert_eq!(result.incidents[0].signal_counts.hook_timed_out, 1); + assert!( + result.incidents[0] + .signals_present + .contains(&"hook_timed_out".to_string()) + ); +} + +#[test] +fn user_correction_after_hook_detected_from_nearby_transcript() { + let (pool, _dir) = test_pool(); + let entries = vec![make_ai_entry( + "2026-01-01T00:00:05.000Z", + "devhost", + "claude", + "/home/jmagar/workspace/cortex", + "sess-hook-4", + "That's not what I asked for, you shouldn't have run that hook", + )]; + insert_logs_batch(&pool, &entries).unwrap(); + let log_ids: Vec = { + let conn = pool.get().unwrap(); + let mut stmt = conn.prepare("SELECT id FROM logs ORDER BY id ASC").unwrap(); + stmt.query_map([], |row| row.get::<_, i64>(0)) + .unwrap() + .collect::>>() + .unwrap() + }; + + insert_hook_event_row( + &pool, + Some(log_ids[0]), + "claude", + "/home/jmagar/workspace/cortex", + "sess-hook-4", + "devhost", + "2026-01-01T00:00:00.000Z", + "PostToolUse", + "auto-format", + "success", + Some(100), + "runtime_transcript", + ); + + let result = search_ai_hook_incidents(&pool, &AiHookIncidentParams::default()).unwrap(); + assert_eq!(result.incidents.len(), 1); + let incident = &result.incidents[0]; + assert_eq!(incident.signal_counts.user_correction_after_hook, 1); + assert_eq!(incident.anchor_log_ids, vec![log_ids[0]]); + assert!( + incident + .signals_present + .contains(&"user_correction_after_hook".to_string()) + ); +} + +#[test] +fn filters_by_min_score_and_signals() { + let (pool, _dir) = test_pool(); + insert_hook_event_row( + &pool, + None, + "claude", + "/home/jmagar/workspace/cortex", + "sess-a", + "devhost", + "2026-01-01T00:00:00.000Z", + "PostToolUse", + "quiet-hook", + "success", + Some(50), + "runtime_transcript", + ); + insert_hook_event_row( + &pool, + None, + "claude", + "/home/jmagar/workspace/cortex", + "sess-b", + "devhost", + "2026-01-01T00:05:00.000Z", + "PostToolUse", + "loud-hook", + "failed", + None, + "runtime_transcript", + ); + + let result = search_ai_hook_incidents( + &pool, + &AiHookIncidentParams { + min_score: Some(10.0), + ..Default::default() + }, + ) + .unwrap(); + assert!(result.incidents.iter().all(|i| i.priority_score >= 10.0)); + + let result = search_ai_hook_incidents( + &pool, + &AiHookIncidentParams { + signals: vec!["hook_failed".to_string()], + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(result.incidents.len(), 1); + assert_eq!(result.incidents[0].hook_name.as_deref(), Some("loud-hook")); +} + +#[test] +fn sorted_by_priority_score_desc_using_total_cmp() { + let (pool, _dir) = test_pool(); + insert_hook_event_row( + &pool, + None, + "claude", + "/home/jmagar/workspace/cortex", + "sess-low", + "devhost", + "2026-01-01T00:00:00.000Z", + "PostToolUse", + "low-signal", + "success", + Some(10), + "runtime_transcript", + ); + insert_hook_event_row( + &pool, + None, + "claude", + "/home/jmagar/workspace/cortex", + "sess-high", + "devhost", + "2026-01-01T01:00:00.000Z", + "PostToolUse", + "high-signal", + "failed", + None, + "runtime_transcript", + ); + + let result = search_ai_hook_incidents(&pool, &AiHookIncidentParams::default()).unwrap(); + assert_eq!(result.incidents.len(), 2); + assert!(result.incidents[0].priority_score >= result.incidents[1].priority_score); + assert_eq!( + result.incidents[0].hook_name.as_deref(), + Some("high-signal") + ); +} diff --git a/crates/shared/cortex/storage-sqlite/src/ingest.rs b/crates/shared/cortex/storage-sqlite/src/ingest.rs new file mode 100644 index 00000000..25ffca97 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/ingest.rs @@ -0,0 +1,142 @@ +use std::collections::HashMap; + +use anyhow::Result; +use rusqlite::{Error as SqliteError, ErrorCode, Transaction, params}; + +use super::models::LogBatchEntry; +use super::pool::DbPool; + +/// Batch insert for higher throughput +pub fn insert_logs_batch(pool: &DbPool, entries: &[LogBatchEntry]) -> Result { + const RETRY_DELAYS_MS: &[u64] = &[25, 100, 250]; + + let mut attempt = 0usize; + loop { + match insert_logs_batch_once(pool, entries) { + Ok(inserted) => return Ok(inserted), + Err(err) if is_transient_sqlite_lock(&err) && attempt < RETRY_DELAYS_MS.len() => { + let delay_ms = RETRY_DELAYS_MS[attempt]; + tracing::warn!( + error = %err, + attempt = attempt + 1, + retry_delay_ms = delay_ms, + entry_count = entries.len(), + "Transient SQLite lock during batch insert — retrying" + ); + std::thread::sleep(std::time::Duration::from_millis(delay_ms)); + attempt += 1; + } + Err(err) => return Err(err), + } + } +} + +fn insert_logs_batch_once(pool: &DbPool, entries: &[LogBatchEntry]) -> Result { + let mut conn = pool.get()?; + let _write_guard = crate::write_lock(); + let tx = conn.transaction()?; + let _ids = insert_logs_batch_in_tx(&tx, entries)?; + tx.commit()?; + tracing::debug!( + entry_count = entries.len(), + "Committed batch insert transaction" + ); + Ok(entries.len()) +} + +pub(crate) fn insert_logs_batch_in_tx( + tx: &Transaction<'_>, + entries: &[LogBatchEntry], +) -> Result> { + let mut ids = Vec::with_capacity(entries.len()); + { + let mut stmt = tx.prepare_cached( + "INSERT INTO logs ( + timestamp, hostname, facility, severity, app_name, process_id, + message, raw, source_ip, ai_tool, ai_project, ai_session_id, ai_transcript_path, + metadata_json, http_status, auth_outcome, dns_blocked, event_action, parse_error + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19)", + )?; + + for entry in entries { + stmt.execute(params![ + entry.timestamp, + entry.hostname, + entry.facility, + entry.severity, + entry.app_name, + entry.process_id, + entry.message, + entry.raw, + entry.source_ip, + entry.ai_tool, + entry.ai_project, + entry.ai_session_id, + entry.ai_transcript_path, + entry.metadata_json, + entry.http_status, + entry.auth_outcome, + entry.dns_blocked.map(|b| b as i64), + entry.event_action, + entry.parse_error, + ])?; + ids.push(tx.last_insert_rowid()); + } + + // Batch upsert hosts — group by hostname to avoid one upsert per log entry + let mut host_counts: HashMap<&str, i64> = HashMap::new(); + for entry in entries { + *host_counts.entry(entry.hostname.as_str()).or_insert(0) += 1; + } + let mut host_stmt = tx.prepare_cached( + "INSERT INTO hosts (hostname, log_count) + VALUES (?1, ?2) + ON CONFLICT(hostname) DO UPDATE SET + last_seen = strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), + log_count = log_count + excluded.log_count", + )?; + for (hostname, count) in &host_counts { + host_stmt.execute(params![hostname, count])?; + } + let mut checkpoint_stmt = tx.prepare_cached( + "INSERT INTO docker_ingest_checkpoints (host_name, container_id, last_timestamp) + VALUES (?1, ?2, ?3) + ON CONFLICT(host_name, container_id) DO UPDATE SET + last_timestamp = excluded.last_timestamp, + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now')", + )?; + let mut checkpoint_count = 0usize; + for entry in entries { + if let Some(checkpoint) = &entry.docker_checkpoint { + checkpoint_stmt.execute(params![ + checkpoint.host_name, + checkpoint.container_id, + checkpoint.timestamp + ])?; + checkpoint_count += 1; + } + } + + tracing::debug!( + entry_count = entries.len(), + unique_hosts = host_counts.len(), + checkpoint_count, + "Prepared batch insert transaction" + ); + } + Ok(ids) +} + +fn is_transient_sqlite_lock(err: &anyhow::Error) -> bool { + err.chain().any(|cause| { + matches!( + cause.downcast_ref::(), + Some(SqliteError::SqliteFailure(sql_err, _)) + if matches!(sql_err.code, ErrorCode::DatabaseBusy | ErrorCode::DatabaseLocked) + ) + }) +} + +#[cfg(test)] +#[path = "ingest_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/ingest_health.rs b/crates/shared/cortex/storage-sqlite/src/ingest_health.rs new file mode 100644 index 00000000..7e1b2898 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/ingest_health.rs @@ -0,0 +1,65 @@ +use anyhow::Result; +use rusqlite::params; + +use super::pool::DbPool; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct IngestSourceKindHealth { + pub source_kind: String, + pub last_seen: String, + pub last_15m: i64, + pub last_1h: i64, + pub last_24h: i64, +} + +pub fn ingest_source_kind_health( + pool: &DbPool, + now: &str, + cut_15m: &str, + cut_1h: &str, + cut_24h: &str, +) -> Result> { + let conn = pool.get()?; + let mut stmt = conn.prepare( + " + WITH classified AS ( + SELECT + CASE + WHEN ai_transcript_path IS NOT NULL OR source_ip LIKE 'transcript://%' THEN 'transcript' + WHEN source_ip LIKE 'docker://%' THEN 'docker-stream' + WHEN source_ip LIKE 'docker-event://%' THEN 'docker-event' + WHEN source_ip LIKE 'agent-command://%' THEN 'agent-command' + WHEN source_ip LIKE 'shell-history://%' THEN 'shell-history' + WHEN source_ip LIKE 'file-tail://%' THEN 'file-tail' + ELSE json_extract(metadata_json, '$.source_kind') + END AS source_kind, + received_at + FROM logs + WHERE received_at >= ?4 AND received_at <= ?1 + ) + SELECT source_kind, + MAX(received_at), + SUM(CASE WHEN received_at >= ?2 THEN 1 ELSE 0 END), + SUM(CASE WHEN received_at >= ?3 THEN 1 ELSE 0 END), + COUNT(*) + FROM classified + WHERE source_kind IS NOT NULL AND source_kind != '' + GROUP BY source_kind + ORDER BY source_kind ASC + ", + )?; + let rows = stmt.query_map(params![now, cut_15m, cut_1h, cut_24h], |row| { + Ok(IngestSourceKindHealth { + source_kind: row.get(0)?, + last_seen: row.get(1)?, + last_15m: row.get::<_, Option>(2)?.unwrap_or(0), + last_1h: row.get::<_, Option>(3)?.unwrap_or(0), + last_24h: row.get(4)?, + }) + })?; + Ok(rows.collect::>>()?) +} + +#[cfg(test)] +#[path = "ingest_health_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/ingest_health_tests.rs b/crates/shared/cortex/storage-sqlite/src/ingest_health_tests.rs new file mode 100644 index 00000000..17139bd1 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/ingest_health_tests.rs @@ -0,0 +1,41 @@ +use super::*; +use crate::{StorageConfig, init_pool}; + +#[test] +fn health_groups_recent_rows_by_normalized_source_kind() { + let dir = tempfile::tempdir().unwrap(); + let config = StorageConfig { + db_path: dir.path().join("health.db"), + pool_size: 1, + wal_mode: false, + ..StorageConfig::default() + }; + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO logs(timestamp, hostname, severity, message, raw, received_at, source_ip, metadata_json) VALUES (?1,'nas','info','a','a',?1,'docker://nas/c/stdout',NULL)", + ["2026-08-18T12:00:00Z"], + ).unwrap(); + conn.execute( + "INSERT INTO logs(timestamp, hostname, severity, message, raw, received_at, source_ip, metadata_json) VALUES (?1,'nas','info','b','b',?1,'198.51.100.1:514',?2)", + rusqlite::params!["2026-08-18T11:30:00Z", r#"{"source_kind":"syslog-udp"}"#], + ).unwrap(); + drop(conn); + + let rows = ingest_source_kind_health( + &pool, + "2026-08-18T12:15:00Z", + "2026-08-18T12:00:00Z", + "2026-08-18T11:15:00Z", + "2026-08-17T12:15:00Z", + ) + .unwrap(); + assert_eq!( + rows.iter() + .map(|r| r.source_kind.as_str()) + .collect::>(), + vec!["docker-stream", "syslog-udp"] + ); + assert_eq!(rows[0].last_15m, 1); + assert_eq!(rows[1].last_1h, 1); +} diff --git a/crates/shared/cortex/storage-sqlite/src/ingest_tests.rs b/crates/shared/cortex/storage-sqlite/src/ingest_tests.rs new file mode 100644 index 00000000..4b7138cd --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/ingest_tests.rs @@ -0,0 +1,211 @@ +use super::*; +use crate::config::StorageConfig; +use crate::{DbPool, LogBatchEntry, init_pool, list_hosts, tail_logs}; + +fn test_storage_config(db_path: std::path::PathBuf) -> StorageConfig { + StorageConfig::for_test(db_path) +} + +/// Create an isolated test pool using a temp file (not :memory: — FTS5 needs file) +fn test_pool() -> (DbPool, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let config = test_storage_config(db_path); + let pool = init_pool(&config).unwrap(); + (pool, dir) // keep dir alive for test duration +} + +fn make_entry(ts: &str, host: &str, severity: &str, msg: &str) -> LogBatchEntry { + LogBatchEntry { + timestamp: ts.to_string(), + hostname: host.to_string(), + facility: None, + severity: severity.to_string(), + app_name: None, + process_id: None, + message: msg.to_string(), + raw: msg.to_string(), + source_ip: "127.0.0.1:514".to_string(), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + } +} + +#[test] +fn test_insert_and_tail() { + let (pool, _dir) = test_pool(); + let entries = vec![ + make_entry("2026-01-01T00:00:01Z", "host-a", "err", "first error"), + make_entry("2026-01-01T00:00:02Z", "host-a", "info", "second info"), + make_entry("2026-01-01T00:00:03Z", "host-b", "warning", "third warning"), + ]; + let n = insert_logs_batch(&pool, &entries).unwrap(); + assert_eq!(n, 3); + + let rows = tail_logs(&pool, None, None, None, None, 10).unwrap(); + assert_eq!(rows.len(), 3); +} + +#[test] +fn test_host_aggregation() { + let (pool, _dir) = test_pool(); + let entries = vec![ + make_entry("2026-01-01T00:00:01Z", "host-a", "info", "msg1"), + make_entry("2026-01-01T00:00:02Z", "host-a", "info", "msg2"), + make_entry("2026-01-01T00:00:03Z", "host-b", "info", "msg3"), + ]; + insert_logs_batch(&pool, &entries).unwrap(); + + let hosts = list_hosts(&pool).unwrap(); + assert_eq!(hosts.len(), 2); + // host-a should have log_count = 2 + let ha = hosts.iter().find(|h| h.hostname == "host-a").unwrap(); + assert_eq!(ha.log_count, 2); +} + +#[test] +fn test_batch_multiple_entries_same_host() { + let (pool, _dir) = test_pool(); + let entries = vec![ + make_entry("2026-01-01T00:00:01Z", "host-x", "info", "msg1"), + make_entry("2026-01-01T00:00:02Z", "host-x", "info", "msg2"), + make_entry("2026-01-01T00:00:03Z", "host-x", "err", "msg3"), + ]; + insert_logs_batch(&pool, &entries).unwrap(); + + let hosts = list_hosts(&pool).unwrap(); + assert_eq!(hosts.len(), 1); + assert_eq!(hosts[0].hostname, "host-x"); + assert_eq!(hosts[0].log_count, 3); +} + +#[test] +fn test_batch_empty() { + let (pool, _dir) = test_pool(); + let result = insert_logs_batch(&pool, &[]); + assert!(result.is_ok(), "empty batch should not error"); + assert_eq!(result.unwrap(), 0); + + let rows = tail_logs(&pool, None, None, None, None, 10).unwrap(); + assert_eq!(rows.len(), 0, "no rows should exist after empty batch"); + + let hosts = list_hosts(&pool).unwrap(); + assert_eq!(hosts.len(), 0, "no hosts should exist after empty batch"); +} + +#[test] +fn test_batch_mixed_hosts() { + let (pool, _dir) = test_pool(); + let entries = vec![ + make_entry("2026-01-01T00:00:01Z", "host-a", "info", "a msg1"), + make_entry("2026-01-01T00:00:02Z", "host-a", "info", "a msg2"), + make_entry("2026-01-01T00:00:03Z", "host-b", "info", "b msg1"), + ]; + insert_logs_batch(&pool, &entries).unwrap(); + + let hosts = list_hosts(&pool).unwrap(); + assert_eq!(hosts.len(), 2); + + let ha = hosts.iter().find(|h| h.hostname == "host-a").unwrap(); + assert_eq!(ha.log_count, 2); + + let hb = hosts.iter().find(|h| h.hostname == "host-b").unwrap(); + assert_eq!(hb.log_count, 1); +} + +#[test] +#[allow(clippy::type_complexity)] +fn insert_logs_batch_persists_enrichment_fields() { + let dir = tempfile::tempdir().unwrap(); + let config = crate::config::StorageConfig { + db_path: dir.path().join("test.db"), + wal_mode: true, + pool_size: 1, + ..Default::default() + }; + let pool = crate::pool::init_pool(&config).unwrap(); + + let entry = crate::LogBatchEntry { + timestamp: "2026-05-16T10:00:00Z".to_string(), + hostname: "test-host".to_string(), + facility: None, + severity: "info".to_string(), + app_name: Some("swag".to_string()), + process_id: None, + message: "GET / 200".to_string(), + raw: "raw line".to_string(), + source_ip: "docker://localhost/swag/stdout".to_string(), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: Some(r#"{"swag":{"method":"GET"}}"#.to_string()), + http_status: Some(200), + auth_outcome: None, + dns_blocked: None, + event_action: Some("http_request".to_string()), + parse_error: None, + }; + + super::insert_logs_batch(&pool, &[entry]).expect("insert ok"); + + let conn = pool.get().unwrap(); + let row: (Option, Option, Option, Option, Option) = conn + .query_row( + "SELECT http_status, auth_outcome, dns_blocked, event_action, parse_error FROM logs LIMIT 1", + [], + |r| Ok((r.get(0)?, r.get(1)?, r.get(2)?, r.get(3)?, r.get(4)?)), + ) + .unwrap(); + assert_eq!(row.0, Some(200)); + assert_eq!(row.1, None); + assert_eq!(row.2, None); + assert_eq!(row.3, Some("http_request".to_string())); + assert_eq!(row.4, None); +} + +#[test] +fn insert_logs_batch_in_tx_returns_ids_in_input_order() { + let (pool, _dir) = test_pool(); + let mut conn = pool.get().unwrap(); + let tx = conn.transaction().unwrap(); + + let entries = vec![ + make_entry("2026-06-01T00:00:00.000Z", "devhost", "info", "first"), + make_entry("2026-06-01T00:00:01.000Z", "devhost", "info", "second"), + ]; + + let ids = insert_logs_batch_in_tx(&tx, &entries).unwrap(); + tx.commit().unwrap(); + drop(conn); + + assert_eq!(ids.len(), 2); + assert!( + ids[1] > ids[0], + "second row's id must be greater than first" + ); + + let conn = pool.get().unwrap(); + let stored_message: String = conn + .query_row("SELECT message FROM logs WHERE id = ?1", [ids[0]], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(stored_message, "first"); + let stored_message: String = conn + .query_row("SELECT message FROM logs WHERE id = ?1", [ids[1]], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(stored_message, "second"); +} diff --git a/crates/shared/cortex/storage-sqlite/src/inputs.rs b/crates/shared/cortex/storage-sqlite/src/inputs.rs new file mode 100644 index 00000000..4b566cf2 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/inputs.rs @@ -0,0 +1,151 @@ +//! Storage-neutral input contracts for normalized AI event persistence. +//! +//! Scanner/runtime layers normalize untrusted transcript data before it reaches +//! this crate. SQLite accepts these bounded semantic values and persists them; +//! it does not own transcript parsing or scanner policy. + +/// Runtime status of a normalized hook event. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HookStatus { + Success, + Failed, + Blocked, + Error, + Unknown, + Configured, +} + +impl HookStatus { + /// Stable persisted string representation. + pub const fn as_str(self) -> &'static str { + match self { + Self::Success => "success", + Self::Failed => "failed", + Self::Blocked => "blocked", + Self::Error => "error", + Self::Unknown => "unknown", + Self::Configured => "configured", + } + } +} + +/// Provenance category for a normalized hook event. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HookEvidenceKind { + RuntimeTranscript, + ConfigInventory, + TrustedHashState, + LogCorrelation, + SideEffectInference, +} + +impl HookEvidenceKind { + /// Stable persisted string representation. + pub const fn as_str(self) -> &'static str { + match self { + Self::RuntimeTranscript => "runtime_transcript", + Self::ConfigInventory => "config_inventory", + Self::TrustedHashState => "trusted_hash_state", + Self::LogCorrelation => "log_correlation", + Self::SideEffectInference => "side_effect_inference", + } + } +} + +/// Already-normalized hook event accepted by SQLite persistence. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExtractedHookEvent { + pub hook_event: String, + pub hook_name: Option, + pub hook_source: Option, + pub hook_command: Option, + pub status: HookStatus, + pub exit_code: Option, + pub duration_ms: Option, + pub stdout_preview: Option, + pub stderr_preview: Option, + pub persisted_output_path: Option, + pub trusted_hash: Option, + pub evidence_kind: HookEvidenceKind, + pub metadata_json: Option, +} + +/// Whether a normalized MCP event represents a call or its result. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum McpEventKind { + Call, + Result, +} + +impl McpEventKind { + /// Stable persisted string representation. + pub const fn as_str(self) -> &'static str { + match self { + Self::Call => "call", + Self::Result => "result", + } + } +} + +/// Already-normalized MCP event accepted by SQLite persistence. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExtractedMcpEvent { + pub call_id: String, + pub tool_name: String, + pub mcp_server: Option, + pub mcp_tool: Option, + pub event_kind: McpEventKind, + pub turn_id: Option, + pub status: Option, + pub is_error: Option, + pub arguments_json: Option, + pub output_preview: Option, + pub error_text: Option, +} + +/// Source shape that produced a normalized skill event. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SkillEventKind { + ClaudeAttribution, + CodexSkillBlock, +} + +impl SkillEventKind { + /// Stable persisted string representation. + pub const fn as_str(self) -> &'static str { + match self { + Self::ClaudeAttribution => "claude_attribution", + Self::CodexSkillBlock => "codex_skill_block", + } + } +} + +/// Evidence source for a normalized skill event. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum SkillEvidenceKind { + StructuredJsonField, + TranscriptContent, +} + +impl SkillEvidenceKind { + /// Stable persisted string representation. + pub const fn as_str(self) -> &'static str { + match self { + Self::StructuredJsonField => "structured_json_field", + Self::TranscriptContent => "transcript_content", + } + } +} + +/// Already-normalized skill event accepted by SQLite persistence. +#[derive(Debug, Clone, PartialEq, Eq)] +pub struct ExtractedSkillEvent { + pub skill_name: String, + pub skill_plugin: Option, + pub event_kind: SkillEventKind, + pub evidence_kind: SkillEvidenceKind, +} + +#[cfg(test)] +#[path = "inputs_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/inputs_tests.rs b/crates/shared/cortex/storage-sqlite/src/inputs_tests.rs new file mode 100644 index 00000000..5a0d94f1 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/inputs_tests.rs @@ -0,0 +1,19 @@ +use super::*; + +#[test] +fn event_input_wire_values_match_donor_contract() { + assert_eq!(HookStatus::Configured.as_str(), "configured"); + assert_eq!( + HookEvidenceKind::RuntimeTranscript.as_str(), + "runtime_transcript" + ); + assert_eq!(McpEventKind::Result.as_str(), "result"); + assert_eq!( + SkillEventKind::CodexSkillBlock.as_str(), + "codex_skill_block" + ); + assert_eq!( + SkillEvidenceKind::StructuredJsonField.as_str(), + "structured_json_field" + ); +} diff --git a/crates/shared/cortex/storage-sqlite/src/lib.rs b/crates/shared/cortex/storage-sqlite/src/lib.rs new file mode 100644 index 00000000..e22b058d --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/lib.rs @@ -0,0 +1,155 @@ +//! SQLite persistence adapter extracted from Cortex. +//! +//! The adapter owns schema/migration state, connection management, transactional +//! writes, query projections, graph and observatory persistence, and bounded +//! storage maintenance. Semantic contracts come from `cortex-domain`; ingest +//! normalization comes from `cortex-ingest-core`. + +#[cfg(test)] +#[path = "agent_observatory_tests.rs"] +mod agent_observatory_tests; + +pub mod agent_observatory; +pub mod analytics; +pub mod config; +pub mod entity_resolution; +pub mod error_signatures; +pub mod graph; +pub mod graph_findings; +pub mod graph_inventory; +mod graph_resolver_projection; +mod heartbeat; +mod hook_events; +mod hook_incident_evidence; +mod hook_incidents; +mod ingest; +mod ingest_health; +pub mod inputs; +pub mod llm_invocations; +pub mod maintenance; +mod mcp_events; +mod mcp_incident_evidence; +mod mcp_incidents; +mod models; +pub mod notifications; +pub mod otlp_metrics; +pub mod otlp_traces; +mod pool; +mod queries; +pub use queries::page_agent_projection_logs; +mod queries_hosts; +mod queries_service_instances; +mod skill_events; +mod skill_incident_evidence; +mod skill_incidents; +pub mod stream_health; + +pub use config::StorageConfig; +pub use inputs::{ + ExtractedHookEvent, ExtractedMcpEvent, ExtractedSkillEvent, HookEvidenceKind, HookStatus, + McpEventKind, SkillEventKind, SkillEvidenceKind, +}; + +pub use analytics::PATTERN_SCAN_LIMIT_MAX; +pub use analytics::{ + AnomalyEntry, AppEntry, Bucket, ClockSkewEntry, ContextRef, IngestRateBuckets, + IngestRatePerHost, ListAppsParams, ListAppsResult, ListSourceIpsParams, ListSourceIpsResult, + LogEntryWithRaw, PatternEntry, RangeSummary, SilentHostEntry, SourceIpEntry, + SourceIpHostBreakdown, TimelineGroupBy, TimelinePoint, anomalies, clock_skew, context_around, + feed_logs, fetch_log_by_id, fetch_patterns, get_ai_project_context, get_ai_usage_blocks, + ingest_rate, ingest_rate_by_host, list_apps, list_source_ips, silent_hosts, summarize_range, + timeline, +}; +pub use graph::{ + ENTITY_TYPES, EVIDENCE_SOURCE_KINDS, GRAPH_WALK_MAX_DEPTH, GraphWalkEntity, + PROJECTION_STATUSES, REASON_CODES, RELATIONSHIP_TYPES, TRUST_LEVELS, graph_walk_n_hops, + is_known_entity_type, is_known_evidence_source_kind, is_known_reason_code, + is_known_relationship_type, is_known_trust_level, +}; +pub use graph_findings::{ + MountRelationshipFindingRow, PublicRouteFindingRow, list_mount_relationship_findings, + list_public_route_findings, +}; +pub use heartbeat::{ + HeartbeatHostLookup, HeartbeatHostState, HeartbeatLatestEntry, HeartbeatMetricSnapshot, + HeartbeatSampleState, HeartbeatStateFlags, HeartbeatWindowSummary, StaleHeartbeatHost, + heartbeat_host_state, heartbeat_latest_all, heartbeat_metric_snapshot_batch, + heartbeat_window_summaries, stale_heartbeat_hosts, +}; +pub use hook_events::{ + AiHookEventEntry, AiHookEventParams, HookEventInsert, ListHookEventsResult, insert_hook_events, + list_hook_events, +}; +pub use hook_incident_evidence::{ + AiHookInvestigateParams, AiHookInvestigateResult, HookIncidentEvidence, + investigate_ai_hook_incidents, +}; +pub use hook_incidents::{ + AiHookIncidentParams, AiHookIncidentResult, HookIncident, HookSignalCounts, + search_ai_hook_incidents, +}; +pub use ingest::insert_logs_batch; +pub use ingest_health::{IngestSourceKindHealth, ingest_source_kind_health}; +pub use maintenance::{ + DiskSpaceProbe, MaintenanceJob, SystemDiskSpaceProbe, checkpoint_wal_and_incremental_vacuum, + db_full_vacuum, db_incremental_vacuum, db_integrity_check, db_wal_checkpoint, + enforce_storage_budget, enforce_storage_budget_with_state, exceeds_trigger, + finish_maintenance_job, get_maintenance_job, get_storage_metrics, insert_maintenance_job, + maybe_checkpoint_wal_by_size, physical_size_bytes, purge_by_tag_window, purge_old_heartbeats, + purge_old_llm_invocations, purge_old_logs, wal_checkpoint_complete, +}; +pub use maintenance::{PragmaName, db_pragma_i64, db_pragma_string}; +pub use mcp_events::{ + AiMcpEventEntry, AiMcpEventParams, ListMcpEventsResult, McpEventInsert, insert_mcp_events, + list_mcp_events, +}; +pub use mcp_incident_evidence::{ + AiMcpInvestigateParams, AiMcpInvestigateResult, McpIncidentEvidence, + investigate_ai_mcp_incidents, +}; +pub use mcp_incidents::{ + AiMcpIncidentParams, AiMcpIncidentResult, McpIncident, McpSignalCounts, search_ai_mcp_incidents, +}; +pub use models::{ + AbuseIncident, AiAbuseMatch, AiAbuseParams, AiAbuseResult, AiCorrelateParams, AiIncidentParams, + AiIncidentResult, AiInvestigateParams, AiInvestigateResult, AiProjectContext, + AiProjectContextParams, AiProjectInventoryEntry, AiRelatedLogsForAnchor, AiRelatedLogsParams, + AiRelatedWindow, AiSessionEntry, AiToolInventoryEntry, AiUsageBlock, AiUsageBlocksParams, + AiUsageBlocksResult, AppLogCount, CorrelatedSession, DbStats, DockerCheckpoint, + ErrorSummaryEntry, GraphRelatedLogEntry, HostEntry, IncidentCluster, IncidentContextParams, + IncidentContextResult, IncidentEvidence, ListAiProjectsParams, ListAiProjectsResult, + ListAiSessionsParams, ListAiToolsParams, ListAiToolsResult, LogBatchEntry, LogEntry, + ResolvedTopicEntity, SearchAiSessionsParams, SearchAiSessionsResult, SearchParams, + SearchedAiSessionEntry, SessionGraphInputs, SeverityCount, SimilarIncidentsParams, + SimilarIncidentsResult, TopicGraphInputs, +}; +pub use models::{StorageBudgetState, StorageEnforcementOutcome, StorageMetrics, StorageRecovery}; +pub use pool::{ + DbPool, KNOWN_SCHEMA_VERSION, SchemaVersionInfo, backfill_inventory_stats, init_pool, + inventory_backfill_complete, read_schema_version_info, read_schema_version_info_conn, + reconcile_interrupted_server_work, write_lock, +}; +pub use queries::{ + AiSessionRollupStatus, RollupRefresh, SEVERITY_LEVELS, TimelineRollupStatus, + ai_session_rollup_status, correlate_session_graph, get_error_summary, get_stats, + get_stats_with_options, incident_context_summary, investigate_ai_incidents, list_ai_projects, + list_ai_sessions, list_ai_sessions_live, list_ai_tools, list_hosts, prune_timeline_rollup, + refresh_ai_session_rollup, refresh_ai_session_rollup_if_stale, refresh_timeline_rollup, + search_ai_abuse, search_ai_anchors, search_ai_incidents, search_ai_related_logs, + search_ai_sessions, search_logs, search_logs_from_graph_related_entities, severity_to_num, + similar_incidents_clusters, tail_logs, timeline_rollup_status, topic_correlate_inputs, + validate_fts_query, +}; +pub use queries_service_instances::search_logs_for_service_instances; +pub use skill_events::{ + AiSkillEventEntry, AiSkillEventParams, ListSkillEventsResult, SkillEventInsert, + insert_skill_events, list_skill_events, +}; +pub use skill_incident_evidence::{ + AiSkillInvestigateParams, AiSkillInvestigateResult, SkillIncidentEvidence, + investigate_ai_skill_incidents, +}; +pub use skill_incidents::{ + AiSkillIncidentParams, AiSkillIncidentResult, SkillIncident, SkillSignalCounts, + search_ai_skill_incidents, +}; diff --git a/crates/shared/cortex/storage-sqlite/src/llm_invocations.rs b/crates/shared/cortex/storage-sqlite/src/llm_invocations.rs new file mode 100644 index 00000000..af952980 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/llm_invocations.rs @@ -0,0 +1,196 @@ +//! Database operations for the shared LLM invocation audit table +//! (`llm_invocations`, migration 37). `LlmRunner` +//! (`src/app/llm_runner.rs`) is the only writer; the CLI/MCP/REST read +//! surfaces (`sessions llm-invocations`, MCP `llm_invocations` action, +//! `GET /api/sessions/llm-invocations`) are the readers. +//! +//! Call from inside `tokio::task::spawn_blocking`, never from async +//! context directly (same convention as `src/db/notifications.rs`). + +use rusqlite::params; + +/// Parameters for the initial (status='running' or a denial status) +/// insert. `id` is passed separately since callers generate it before +/// building the params (needed so denial paths can audit without a +/// completed spec). +pub struct LlmInvocationInsertParams { + pub caller_surface: String, + pub action: String, + pub provider: String, + pub model: Option, + pub program: Option, + pub incident_id: Option, + pub ai_tool: Option, + pub ai_project: Option, + pub ai_session_id: Option, + pub evidence_counts_json: Option, + pub prompt_bytes: Option, + pub status: String, + pub metadata_json: Option, +} + +pub fn insert_llm_invocation_running( + conn: &rusqlite::Connection, + id: &str, + p: &LlmInvocationInsertParams, +) -> rusqlite::Result<()> { + conn.execute( + "INSERT INTO llm_invocations + (id, started_at, caller_surface, action, provider, model, program, + incident_id, ai_tool, ai_project, ai_session_id, + evidence_counts_json, prompt_bytes, status, metadata_json) + VALUES (?1, strftime('%Y-%m-%dT%H:%M:%fZ','now'), ?2, ?3, ?4, ?5, ?6, + ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14)", + params![ + id, + p.caller_surface, + p.action, + p.provider, + p.model, + p.program, + p.incident_id, + p.ai_tool, + p.ai_project, + p.ai_session_id, + p.evidence_counts_json, + p.prompt_bytes, + p.status, + p.metadata_json, + ], + )?; + Ok(()) +} + +pub fn finish_llm_invocation( + conn: &rusqlite::Connection, + id: &str, + status: &str, + error: Option<&str>, + duration_ms: i64, + output_bytes: Option, +) -> rusqlite::Result<()> { + conn.execute( + "UPDATE llm_invocations + SET finished_at = strftime('%Y-%m-%dT%H:%M:%fZ','now'), + duration_ms = ?2, + status = ?3, + error = ?4, + output_bytes = COALESCE(?5, output_bytes) + WHERE id = ?1", + params![id, duration_ms, status, error, output_bytes], + )?; + Ok(()) +} + +/// A row from `llm_invocations`, as returned to CLI/MCP/REST readers. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct LlmInvocationRow { + pub id: String, + pub started_at: String, + pub finished_at: Option, + pub duration_ms: Option, + pub caller_surface: String, + pub action: String, + pub provider: String, + pub model: Option, + pub program: Option, + pub incident_id: Option, + pub ai_tool: Option, + pub ai_project: Option, + pub ai_session_id: Option, + pub evidence_counts_json: Option, + pub prompt_bytes: Option, + pub output_bytes: Option, + pub status: String, + pub error: Option, + pub metadata_json: Option, +} + +/// Fetch recent invocations, optionally filtered by `action`/`status` and +/// bounded to those started at or after `since` (ISO8601). `limit` is +/// clamped to `[1, 500]`, matching `notifications::firings_recent`. +/// +/// Eng review fix (performance-oracle + data-migration-expert, +/// independently confirmed via `EXPLAIN QUERY PLAN`): the previous +/// implementation used a single static query with a +/// `(?N IS NULL OR col = ?N)` WHERE clause per filter. That idiom is not +/// sargable — SQLite's query planner cannot use +/// `idx_llm_invocations_action_started` or +/// `idx_llm_invocations_status_started` for it under any parameter +/// combination, so it always fell back to a full scan of +/// `idx_llm_invocations_started` (or a table scan). This version builds +/// the WHERE clause dynamically, appending `AND action = ?` / `AND status +/// = ?` / `AND started_at >= ?` only for filters that are actually +/// `Some(...)`, matching the dynamic-WHERE-builder idiom already used in +/// `src/db/queries.rs` (e.g. `tail_logs_sql`'s `WHERE 1=1` + conditionally +/// appended `AND col = ?` clauses; NOT `get_error_summary_sql`, which just +/// picks between two static query templates and has no per-filter dynamic +/// WHERE clause at all). With no filters, +/// or with `since` as the only filter, `idx_llm_invocations_started` is +/// used; with `action` set, `idx_llm_invocations_action_started` is used; +/// with `status` set, `idx_llm_invocations_status_started` is used. +pub fn list_llm_invocations( + conn: &rusqlite::Connection, + limit: i64, + since: Option<&str>, + action: Option<&str>, + status: Option<&str>, +) -> rusqlite::Result> { + let clamped_limit = limit.clamp(1, 500); + + let mut sql = String::from( + "SELECT id, started_at, finished_at, duration_ms, caller_surface, action, + provider, model, program, incident_id, ai_tool, ai_project, + ai_session_id, evidence_counts_json, prompt_bytes, output_bytes, + status, error, metadata_json + FROM llm_invocations WHERE 1=1", + ); + let mut bindings: Vec = Vec::new(); + + if let Some(action) = action { + sql.push_str(" AND action = ?"); + bindings.push(rusqlite::types::Value::Text(action.to_string())); + } + if let Some(status) = status { + sql.push_str(" AND status = ?"); + bindings.push(rusqlite::types::Value::Text(status.to_string())); + } + if let Some(since) = since { + sql.push_str(" AND started_at >= ?"); + bindings.push(rusqlite::types::Value::Text(since.to_string())); + } + sql.push_str(" ORDER BY started_at DESC LIMIT ?"); + bindings.push(rusqlite::types::Value::Integer(clamped_limit)); + + let mut stmt = conn.prepare(&sql)?; + let rows = stmt + .query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + Ok(LlmInvocationRow { + id: row.get(0)?, + started_at: row.get(1)?, + finished_at: row.get(2)?, + duration_ms: row.get(3)?, + caller_surface: row.get(4)?, + action: row.get(5)?, + provider: row.get(6)?, + model: row.get(7)?, + program: row.get(8)?, + incident_id: row.get(9)?, + ai_tool: row.get(10)?, + ai_project: row.get(11)?, + ai_session_id: row.get(12)?, + evidence_counts_json: row.get(13)?, + prompt_bytes: row.get(14)?, + output_bytes: row.get(15)?, + status: row.get(16)?, + error: row.get(17)?, + metadata_json: row.get(18)?, + }) + })? + .collect::>>()?; + Ok(rows) +} + +#[cfg(test)] +#[path = "llm_invocations_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/llm_invocations_tests.rs b/crates/shared/cortex/storage-sqlite/src/llm_invocations_tests.rs new file mode 100644 index 00000000..8a5e9ffe --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/llm_invocations_tests.rs @@ -0,0 +1,321 @@ +use super::*; + +fn test_conn() -> ( + r2d2::PooledConnection, + tempfile::TempDir, +) { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let storage = crate::config::StorageConfig::for_test(db_path); + let pool = crate::init_pool(&storage).unwrap(); + let conn = pool.get().unwrap(); + (conn, dir) +} + +fn sample_params() -> LlmInvocationInsertParams { + LlmInvocationInsertParams { + caller_surface: "test".to_string(), + action: "ai_assess".to_string(), + provider: "gemini-cli".to_string(), + model: Some("gemini-3.1-flash-lite-preview".to_string()), + program: Some("gemini".to_string()), + incident_id: Some("inc-42".to_string()), + ai_tool: None, + ai_project: Some("cortex".to_string()), + ai_session_id: None, + evidence_counts_json: Some(r#"{"total_incidents":1}"#.to_string()), + prompt_bytes: Some(128), + status: "running".to_string(), + metadata_json: Some(r#"{"host":"devhost","pid":123}"#.to_string()), + } +} + +#[test] +fn insert_then_finish_round_trips() { + let (conn, _dir) = test_conn(); + insert_llm_invocation_running(&conn, "llm-test-1", &sample_params()).unwrap(); + finish_llm_invocation(&conn, "llm-test-1", "success", None, 4200, Some(512)).unwrap(); + + let rows = list_llm_invocations(&conn, 10, None, None, None).unwrap(); + assert_eq!(rows.len(), 1); + let row = &rows[0]; + assert_eq!(row.id, "llm-test-1"); + assert_eq!(row.status, "success"); + assert_eq!(row.duration_ms, Some(4200)); + assert_eq!(row.output_bytes, Some(512)); + assert_eq!(row.incident_id.as_deref(), Some("inc-42")); + assert!(row.finished_at.is_some()); +} + +#[test] +fn list_filters_by_action_and_status_and_since() { + let (conn, _dir) = test_conn(); + insert_llm_invocation_running(&conn, "llm-a", &sample_params()).unwrap(); + finish_llm_invocation(&conn, "llm-a", "success", None, 100, Some(10)).unwrap(); + + let mut other = sample_params(); + other.action = "skill_assess".to_string(); + insert_llm_invocation_running(&conn, "llm-b", &other).unwrap(); + finish_llm_invocation(&conn, "llm-b", "error", Some("boom"), 50, None).unwrap(); + + let ai_only = list_llm_invocations(&conn, 10, None, Some("ai_assess"), None).unwrap(); + assert_eq!(ai_only.len(), 1); + assert_eq!(ai_only[0].id, "llm-a"); + + let errors_only = list_llm_invocations(&conn, 10, None, None, Some("error")).unwrap(); + assert_eq!(errors_only.len(), 1); + assert_eq!(errors_only[0].id, "llm-b"); + + let future_since = + list_llm_invocations(&conn, 10, Some("2999-01-01T00:00:00Z"), None, None).unwrap(); + assert!(future_since.is_empty()); +} + +#[test] +fn list_respects_limit_and_orders_newest_first() { + let (conn, _dir) = test_conn(); + for i in 0..5 { + let id = format!("llm-{i}"); + insert_llm_invocation_running(&conn, &id, &sample_params()).unwrap(); + finish_llm_invocation(&conn, &id, "success", None, 10, Some(1)).unwrap(); + } + let rows = list_llm_invocations(&conn, 2, None, None, None).unwrap(); + assert_eq!(rows.len(), 2); +} + +// --- Eng review fix (performance-oracle + data-migration-expert): the +// dynamic WHERE-builder rewrite must preserve exact correctness across +// every filter combination — no filters, each filter alone, and all +// filters combined — while also making the composite indexes usable. +// These tests cover correctness; `explain_query_plan_uses_composite_indexes_for_filtered_queries` +// below asserts the index usage itself via `EXPLAIN QUERY PLAN`. + +#[test] +fn list_with_no_filters_returns_everything_newest_first() { + let (conn, _dir) = test_conn(); + insert_llm_invocation_running(&conn, "llm-a", &sample_params()).unwrap(); + finish_llm_invocation(&conn, "llm-a", "success", None, 10, Some(1)).unwrap(); + let mut other = sample_params(); + other.action = "skill_assess".to_string(); + insert_llm_invocation_running(&conn, "llm-b", &other).unwrap(); + finish_llm_invocation(&conn, "llm-b", "error", Some("boom"), 20, None).unwrap(); + + let rows = list_llm_invocations(&conn, 500, None, None, None).unwrap(); + assert_eq!(rows.len(), 2); +} + +#[test] +fn list_action_only_filter_returns_correct_rows() { + let (conn, _dir) = test_conn(); + insert_llm_invocation_running(&conn, "llm-a", &sample_params()).unwrap(); + finish_llm_invocation(&conn, "llm-a", "success", None, 10, Some(1)).unwrap(); + let mut other = sample_params(); + other.action = "skill_assess".to_string(); + insert_llm_invocation_running(&conn, "llm-b", &other).unwrap(); + finish_llm_invocation(&conn, "llm-b", "success", None, 10, Some(1)).unwrap(); + + let rows = list_llm_invocations(&conn, 500, None, Some("skill_assess"), None).unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].id, "llm-b"); +} + +#[test] +fn list_status_only_filter_returns_correct_rows() { + let (conn, _dir) = test_conn(); + insert_llm_invocation_running(&conn, "llm-a", &sample_params()).unwrap(); + finish_llm_invocation(&conn, "llm-a", "success", None, 10, Some(1)).unwrap(); + insert_llm_invocation_running(&conn, "llm-b", &sample_params()).unwrap(); + finish_llm_invocation(&conn, "llm-b", "error", Some("boom"), 10, None).unwrap(); + + let rows = list_llm_invocations(&conn, 500, None, None, Some("error")).unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].id, "llm-b"); +} + +#[test] +fn list_since_only_filter_returns_correct_rows() { + let (conn, _dir) = test_conn(); + insert_llm_invocation_running(&conn, "llm-a", &sample_params()).unwrap(); + finish_llm_invocation(&conn, "llm-a", "success", None, 10, Some(1)).unwrap(); + + // since in the far future: excludes everything. + let future = + list_llm_invocations(&conn, 500, Some("2999-01-01T00:00:00Z"), None, None).unwrap(); + assert!(future.is_empty()); + + // since in the far past: includes everything. + let past = list_llm_invocations(&conn, 500, Some("2000-01-01T00:00:00Z"), None, None).unwrap(); + assert_eq!(past.len(), 1); +} + +#[test] +fn list_combined_filters_intersect_correctly() { + let (conn, _dir) = test_conn(); + // Matches all three filters. + insert_llm_invocation_running(&conn, "llm-match", &sample_params()).unwrap(); + finish_llm_invocation(&conn, "llm-match", "success", None, 10, Some(1)).unwrap(); + + // Wrong action. + let mut wrong_action = sample_params(); + wrong_action.action = "skill_assess".to_string(); + insert_llm_invocation_running(&conn, "llm-wrong-action", &wrong_action).unwrap(); + finish_llm_invocation(&conn, "llm-wrong-action", "success", None, 10, Some(1)).unwrap(); + + // Wrong status. + insert_llm_invocation_running(&conn, "llm-wrong-status", &sample_params()).unwrap(); + finish_llm_invocation(&conn, "llm-wrong-status", "error", Some("boom"), 10, None).unwrap(); + + let rows = list_llm_invocations( + &conn, + 500, + Some("2000-01-01T00:00:00Z"), + Some("ai_assess"), + Some("success"), + ) + .unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].id, "llm-match"); +} + +// PR #106 reconciliation fix (pr-test-analyzer): the existing coverage above +// exercises no-filter, each single filter alone, and all three filters +// combined, but skips the three two-filter combinations. Add those so every +// pairwise intersection of `action`/`status`/`since` is proven, not just the +// full-combination and single-filter extremes. + +#[test] +fn list_action_and_status_combined_filters_intersect_correctly() { + let (conn, _dir) = test_conn(); + // Matches action=ai_assess AND status=success. + insert_llm_invocation_running(&conn, "llm-match", &sample_params()).unwrap(); + finish_llm_invocation(&conn, "llm-match", "success", None, 10, Some(1)).unwrap(); + + // Right action, wrong status. + insert_llm_invocation_running(&conn, "llm-wrong-status", &sample_params()).unwrap(); + finish_llm_invocation(&conn, "llm-wrong-status", "error", Some("boom"), 10, None).unwrap(); + + // Wrong action, right status. + let mut wrong_action = sample_params(); + wrong_action.action = "skill_assess".to_string(); + insert_llm_invocation_running(&conn, "llm-wrong-action", &wrong_action).unwrap(); + finish_llm_invocation(&conn, "llm-wrong-action", "success", None, 10, Some(1)).unwrap(); + + let rows = list_llm_invocations(&conn, 500, None, Some("ai_assess"), Some("success")).unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].id, "llm-match"); +} + +#[test] +fn list_action_and_since_combined_filters_intersect_correctly() { + let (conn, _dir) = test_conn(); + // Matches action=ai_assess and is within the since window (far past). + insert_llm_invocation_running(&conn, "llm-match", &sample_params()).unwrap(); + finish_llm_invocation(&conn, "llm-match", "success", None, 10, Some(1)).unwrap(); + + // Wrong action, but still within the since window. + let mut wrong_action = sample_params(); + wrong_action.action = "skill_assess".to_string(); + insert_llm_invocation_running(&conn, "llm-wrong-action", &wrong_action).unwrap(); + finish_llm_invocation(&conn, "llm-wrong-action", "success", None, 10, Some(1)).unwrap(); + + // Right action, but excluded by a since window in the far future. + let future_rows = list_llm_invocations( + &conn, + 500, + Some("2999-01-01T00:00:00Z"), + Some("ai_assess"), + None, + ) + .unwrap(); + assert!( + future_rows.is_empty(), + "future since must exclude even action-matching rows" + ); + + let rows = list_llm_invocations( + &conn, + 500, + Some("2000-01-01T00:00:00Z"), + Some("ai_assess"), + None, + ) + .unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].id, "llm-match"); +} + +#[test] +fn list_status_and_since_combined_filters_intersect_correctly() { + let (conn, _dir) = test_conn(); + // Matches status=error and is within the since window (far past). + insert_llm_invocation_running(&conn, "llm-match", &sample_params()).unwrap(); + finish_llm_invocation(&conn, "llm-match", "error", Some("boom"), 10, None).unwrap(); + + // Wrong status, but still within the since window. + insert_llm_invocation_running(&conn, "llm-wrong-status", &sample_params()).unwrap(); + finish_llm_invocation(&conn, "llm-wrong-status", "success", None, 10, Some(1)).unwrap(); + + // Right status, but excluded by a since window in the far future. + let future_rows = list_llm_invocations( + &conn, + 500, + Some("2999-01-01T00:00:00Z"), + None, + Some("error"), + ) + .unwrap(); + assert!( + future_rows.is_empty(), + "future since must exclude even status-matching rows" + ); + + let rows = list_llm_invocations( + &conn, + 500, + Some("2000-01-01T00:00:00Z"), + None, + Some("error"), + ) + .unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].id, "llm-match"); +} + +/// Assert (via `EXPLAIN QUERY PLAN`) that filtered queries actually use +/// the composite indexes created in migration 37, not a full scan. The +/// old `(?N IS NULL OR col = ?N)` idiom was not sargable and always fell +/// back to scanning `idx_llm_invocations_started` (or the table) +/// regardless of which filters were supplied. +#[test] +fn explain_query_plan_uses_composite_indexes_for_filtered_queries() { + let (conn, _dir) = test_conn(); + + let plan_uses_index = |sql: &str, index_name: &str| -> bool { + let explain_sql = format!("EXPLAIN QUERY PLAN {sql}"); + let mut stmt = conn.prepare(&explain_sql).unwrap(); + let details: Vec = stmt + .query_map([], |row| row.get::<_, String>(3)) + .unwrap() + .collect::>>() + .unwrap(); + details.iter().any(|d| d.contains(index_name)) + }; + + // action-only filter should use idx_llm_invocations_action_started. + assert!( + plan_uses_index( + "SELECT id FROM llm_invocations WHERE action = 'ai_assess' ORDER BY started_at DESC LIMIT 10", + "idx_llm_invocations_action_started" + ), + "action-only query must use idx_llm_invocations_action_started" + ); + + // status-only filter should use idx_llm_invocations_status_started. + assert!( + plan_uses_index( + "SELECT id FROM llm_invocations WHERE status = 'success' ORDER BY started_at DESC LIMIT 10", + "idx_llm_invocations_status_started" + ), + "status-only query must use idx_llm_invocations_status_started" + ); +} diff --git a/crates/shared/cortex/storage-sqlite/src/maintenance.rs b/crates/shared/cortex/storage-sqlite/src/maintenance.rs new file mode 100644 index 00000000..b4387baf --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/maintenance.rs @@ -0,0 +1,1383 @@ +use std::path::{Path, PathBuf}; + +use anyhow::Result; +use chrono::Utc; +use rusqlite::{OptionalExtension, params}; + +use crate::config::StorageConfig; + +use super::models::{StorageEnforcementOutcome, StorageMetrics, StorageRecovery}; +use super::pool::DbPool; + +pub trait DiskSpaceProbe { + fn free_bytes(&self, path: &Path) -> Result; +} + +pub struct SystemDiskSpaceProbe; + +impl DiskSpaceProbe for SystemDiskSpaceProbe { + fn free_bytes(&self, path: &Path) -> Result { + free_bytes_impl(path) + } +} + +#[cfg(unix)] +fn free_bytes_impl(path: &Path) -> Result { + let stats = rustix::fs::statvfs(path)?; + Ok(stats.f_bavail.saturating_mul(stats.f_bsize)) +} + +#[cfg(windows)] +fn free_bytes_impl(path: &Path) -> Result { + use std::os::windows::ffi::OsStrExt; + + // Declare GetDiskFreeSpaceExW inline — avoids adding a windows-sys dep. + unsafe extern "system" { + fn GetDiskFreeSpaceExW( + lpDirectoryName: *const u16, + lpFreeBytesAvailableToCaller: *mut u64, + lpTotalNumberOfBytes: *mut u64, + lpTotalNumberOfFreeBytes: *mut u64, + ) -> i32; + } + + let wide: Vec = path + .as_os_str() + .encode_wide() + .chain(std::iter::once(0)) + .collect(); + let mut free_bytes: u64 = 0; + let ok = unsafe { + GetDiskFreeSpaceExW( + wide.as_ptr(), + &mut free_bytes, + std::ptr::null_mut(), + std::ptr::null_mut(), + ) + }; + if ok != 0 { + return Ok(free_bytes); + } + anyhow::bail!( + "GetDiskFreeSpaceExW failed: {}", + std::io::Error::last_os_error() + ) +} + +#[cfg(not(any(unix, windows)))] +fn free_bytes_impl(path: &Path) -> Result { + let _ = path; + anyhow::bail!("free_bytes: unsupported platform") +} + +pub fn get_storage_metrics(pool: &DbPool, config: &StorageConfig) -> Result { + get_storage_metrics_with_probe(pool, config, &SystemDiskSpaceProbe) +} + +pub fn physical_size_bytes(path: &Path) -> Result { + physical_db_size_bytes(path) +} + +pub fn db_wal_checkpoint(pool: &DbPool, mode: &str) -> Result<(i64, i64, i64)> { + let mode = match mode { + "passive" => "PASSIVE", + "full" => "FULL", + "restart" => "RESTART", + "truncate" => "TRUNCATE", + other => anyhow::bail!("unsupported WAL checkpoint mode: {other}"), + }; + let sql = format!("PRAGMA wal_checkpoint({mode})"); + let conn = pool.get()?; + let result = conn.query_row(&sql, [], |row| { + Ok(( + row.get::<_, i64>(0)?, + row.get::<_, i64>(1)?, + row.get::<_, i64>(2)?, + )) + })?; + Ok(result) +} + +pub fn wal_checkpoint_complete(busy: i64, log_frames: i64, checkpointed_frames: i64) -> bool { + busy == 0 && checkpointed_frames >= log_frames +} + +pub fn db_incremental_vacuum(pool: &DbPool, pages: u32) -> Result<()> { + let conn = pool.get()?; + let _write_guard = crate::write_lock(); + conn.execute_batch(&format!("PRAGMA incremental_vacuum({pages});"))?; + Ok(()) +} + +pub fn db_full_vacuum(pool: &DbPool) -> Result<()> { + let conn = pool.get()?; + let _write_guard = crate::write_lock(); + conn.execute_batch("VACUUM;")?; + Ok(()) +} + +/// Run `PRAGMA integrity_check` (full) or `PRAGMA quick_check` (skips +/// cross-row consistency, ~10x faster on multi-GB databases). +pub fn db_integrity_check(pool: &DbPool, quick: bool) -> Result> { + let conn = pool.get()?; + let pragma = if quick { + "quick_check" + } else { + "integrity_check" + }; + let mut stmt = conn.prepare(&format!("PRAGMA {pragma}"))?; + let rows = stmt.query_map([], |row| row.get::<_, String>(0))?; + let messages = rows.collect::, _>>()?; + Ok(messages) +} + +/// A row from the `maintenance_jobs` table (bead syslog-mcp-a4pd). +#[derive(Debug, Clone)] +pub struct MaintenanceJob { + pub id: i64, + pub kind: String, + /// One of `running`, `done`, `failed`. + pub status: String, + pub started_at: String, + pub finished_at: Option, + /// JSON-encoded result payload (present once terminal), e.g. + /// `{"ok":true,"messages":["ok"]}` or `{"error":"..."}`. + pub result_json: Option, +} + +/// Insert a new `running` maintenance job and return its id. Used by the +/// background `db integrity` path to record a job before spawning the check. +pub fn insert_maintenance_job(pool: &DbPool, kind: &str) -> Result { + let conn = pool.get()?; + conn.execute( + "INSERT INTO maintenance_jobs (kind, status, started_at) + VALUES (?1, 'running', strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))", + [kind], + )?; + Ok(conn.last_insert_rowid()) +} + +/// Mark a maintenance job terminal (`done`/`failed`) with its JSON result. +pub fn finish_maintenance_job( + pool: &DbPool, + id: i64, + status: &str, + result_json: &str, +) -> Result<()> { + let conn = pool.get()?; + conn.execute( + "UPDATE maintenance_jobs + SET status = ?2, + finished_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), + result_json = ?3 + WHERE id = ?1", + rusqlite::params![id, status, result_json], + )?; + Ok(()) +} + +/// Fetch a maintenance job by id, or `None` if no such job exists. +pub fn get_maintenance_job(pool: &DbPool, id: i64) -> Result> { + let conn = pool.get()?; + let job = conn + .query_row( + "SELECT id, kind, status, started_at, finished_at, result_json + FROM maintenance_jobs WHERE id = ?1", + [id], + |r| { + Ok(MaintenanceJob { + id: r.get(0)?, + kind: r.get(1)?, + status: r.get(2)?, + started_at: r.get(3)?, + finished_at: r.get(4)?, + result_json: r.get(5)?, + }) + }, + ) + .optional()?; + Ok(job) +} + +/// Trusted SQLite PRAGMA names exposed by the storage diagnostics port. +/// +/// An enum keeps SQL identifier interpolation closed to a fixed allow-list. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum PragmaName { + PageCount, + FreelistCount, + PageSize, + AutoVacuum, + JournalMode, + CacheSize, + MmapSize, +} + +impl PragmaName { + const fn as_str(self) -> &'static str { + match self { + Self::PageCount => "page_count", + Self::FreelistCount => "freelist_count", + Self::PageSize => "page_size", + Self::AutoVacuum => "auto_vacuum", + Self::JournalMode => "journal_mode", + Self::CacheSize => "cache_size", + Self::MmapSize => "mmap_size", + } + } +} + +/// Read one trusted integer-valued SQLite PRAGMA. +pub fn db_pragma_i64(pool: &DbPool, pragma: PragmaName) -> Result { + let conn = pool.get()?; + Ok(conn.query_row(&format!("PRAGMA {}", pragma.as_str()), [], |row| row.get(0))?) +} + +/// Read one trusted string-valued SQLite PRAGMA. +pub fn db_pragma_string(pool: &DbPool, pragma: PragmaName) -> Result { + let conn = pool.get()?; + Ok(conn.query_row(&format!("PRAGMA {}", pragma.as_str()), [], |row| row.get(0))?) +} + +pub fn get_storage_metrics_with_probe( + pool: &DbPool, + config: &StorageConfig, + probe: &impl DiskSpaceProbe, +) -> Result { + let conn = pool.get()?; + let page_count: i64 = conn.query_row("PRAGMA page_count", [], |r| r.get(0))?; + let freelist_count: i64 = conn.query_row("PRAGMA freelist_count", [], |r| r.get(0))?; + let page_size: i64 = conn.query_row("PRAGMA page_size", [], |r| r.get(0))?; + drop(conn); + + let logical_db_size_bytes = ((page_count - freelist_count).max(0) * page_size).max(0) as u64; + let physical_db_size_bytes = physical_db_size_bytes(&config.db_path)?; + let free_disk_bytes = probe + .free_bytes(config.db_path.parent().unwrap_or_else(|| Path::new("."))) + .ok(); + tracing::debug!( + logical_db_size_bytes, + physical_db_size_bytes, + free_disk_bytes = ?free_disk_bytes, + db_path = %config.db_path.display(), + "Collected storage metrics" + ); + + Ok(StorageMetrics { + logical_db_size_bytes, + physical_db_size_bytes, + free_disk_bytes, + }) +} + +pub fn enforce_storage_budget( + pool: &DbPool, + config: &StorageConfig, +) -> Result { + enforce_storage_budget_with_probe(pool, config, &SystemDiskSpaceProbe) +} + +pub fn enforce_storage_budget_with_probe( + pool: &DbPool, + config: &StorageConfig, + probe: &impl DiskSpaceProbe, +) -> Result { + // No prior write-block state — used by the initial enforcement call at + // startup, before any tick has run. Hysteresis is a no-op on the first call. + enforce_storage_budget_with_state(pool, config, probe, false) +} + +/// Storage-budget enforcement with the previous tick's `write_blocked` state +/// threaded in for hysteresis on the EXTERNAL disk-pressure path. +/// +/// Two INDEPENDENT policies (syslog-mcp-w4hh): +/// - **DB-size (self-trim):** `max_db_size_mb` measures cortex's OWN logical +/// bytes. Cortex can resolve this by trimming its own oldest data, so it +/// loops `delete_oldest_*_chunk` down to `recovery_db_size_mb` — UNLESS doing +/// so would breach the err+ retention floor, in which case it stops and sets +/// `write_blocked` rather than wiping irreplaceable high-severity history. +/// - **Free-disk (external pressure):** `min_free_disk_mb` measures the WHOLE +/// filesystem (statvfs). A neighbour process filling the shared volume is not +/// something cortex can fix by deleting its own rows, so it NEVER deletes for +/// this trigger — it sets `write_blocked` and relies on ingest back-pressure +/// (receiver/writer.rs) until free disk recovers. Hysteresis: block engages at +/// `min_free_disk_mb`, clears only at `recovery_free_disk_mb`. +pub fn enforce_storage_budget_with_state( + pool: &DbPool, + config: &StorageConfig, + probe: &impl DiskSpaceProbe, + prev_write_blocked: bool, +) -> Result { + let recovery = recovery_targets(config); + let mut deleted_rows = 0usize; + let mut deleted_log_rows = 0usize; + let mut all_hosts: std::collections::HashSet = Default::default(); + + let mut metrics = get_storage_metrics_with_probe(pool, config, probe)?; + tracing::debug!( + logical_db_size_bytes = metrics.logical_db_size_bytes, + physical_db_size_bytes = metrics.physical_db_size_bytes, + free_disk_bytes = ?metrics.free_disk_bytes, + max_db_size_mb = config.max_db_size_mb, + recovery_db_size_mb = config.recovery_db_size_mb, + min_free_disk_mb = config.min_free_disk_mb, + recovery_free_disk_mb = config.recovery_free_disk_mb, + "Storage budget enforcement check started" + ); + if !storage_limits_enabled(config) { + tracing::debug!("Storage limits disabled — skipping enforcement"); + return Ok(StorageEnforcementOutcome { + metrics, + recovery, + deleted_rows, + write_blocked: false, + }); + } + + // EXTERNAL disk-pressure decision (no deletion). Evaluated with hysteresis + // from the previous tick's state so the block engages at `min_free_disk_mb` + // and clears only once free disk has climbed back to `recovery_free_disk_mb`. + // The self-trim loop below runs INDEPENDENTLY of this — both can be active in + // the same tick (DB over its cap AND the filesystem low on free space). + let mut disk_write_blocked = disk_pressure_write_blocked(&metrics, config, prev_write_blocked); + if disk_write_blocked { + tracing::warn!( + free_disk_bytes = ?metrics.free_disk_bytes, + min_free_disk_mb = config.min_free_disk_mb, + recovery_free_disk_mb = config.recovery_free_disk_mb, + "Free-disk pressure detected — blocking writes WITHOUT deleting own data \ + (external whole-filesystem condition; cortex cannot resolve it by self-trim)" + ); + } + + // SELF-TRIM loop: only the DB-size trigger drives deletion. Its recovery exit + // is `logical <= recovery_db_size_mb` (the free-disk arm never gates it). + if db_size_exceeds_trigger(&metrics, config) { + while !db_size_within_recovery(&metrics, &recovery, config) { + tracing::warn!( + logical_db_size_bytes = metrics.logical_db_size_bytes, + physical_db_size_bytes = metrics.physical_db_size_bytes, + deleted_rows, + "DB-size budget exceeded — self-trimming oldest telemetry chunk" + ); + + let deleted_orphan_children = delete_orphan_heartbeat_children(pool)?; + if deleted_orphan_children > 0 { + deleted_rows += deleted_orphan_children; + tracing::info!( + deleted_rows = deleted_orphan_children, + total_deleted_rows = deleted_rows, + "Deleted orphan heartbeat child rows for storage recovery" + ); + metrics = get_storage_metrics_with_probe(pool, config, probe)?; + continue; + } + + let deleted = match oldest_telemetry_source(pool)? { + Some(TelemetrySource::Heartbeats) => { + let deleted = delete_oldest_heartbeats_chunk(pool, config.cleanup_chunk_size)?; + DeletedTelemetryChunk { + deleted_rows: deleted, + log_hostnames: Vec::new(), + source: TelemetrySource::Heartbeats, + } + } + Some(TelemetrySource::Logs) => { + let deleted = + delete_oldest_logs_chunk(pool, config.cleanup_chunk_size, config)?; + DeletedTelemetryChunk { + deleted_rows: deleted.deleted_rows, + log_hostnames: deleted.hostnames, + source: TelemetrySource::Logs, + } + } + None => DeletedTelemetryChunk { + deleted_rows: 0, + log_hostnames: Vec::new(), + source: TelemetrySource::Logs, + }, + }; + // Floor-protection fallthrough: if the OLDEST source was logs but the + // chunk was fully err+-floor-protected (0 deleted), deletable heartbeats + // may still exist (they are simply newer than the protected logs, so + // `oldest_telemetry_source` picked logs). Trim a heartbeat chunk before + // concluding nothing is deletable — otherwise a DB-size breach would + // prematurely block writes while reclaimable heartbeat space remains. + let deleted = if deleted.deleted_rows == 0 && deleted.source == TelemetrySource::Logs { + let hb = delete_oldest_heartbeats_chunk(pool, config.cleanup_chunk_size)?; + if hb > 0 { + tracing::info!( + deleted_rows = hb, + "Oldest logs were floor-protected; trimmed heartbeat chunk instead" + ); + } + DeletedTelemetryChunk { + deleted_rows: hb, + log_hostnames: Vec::new(), + source: TelemetrySource::Heartbeats, + } + } else { + deleted + }; + + if deleted.deleted_rows == 0 { + // Could not delete any more deletable rows. This is either an empty + // DB or — the case the err+ floor exists for — every remaining row + // is floor-protected AND no heartbeats remain to trim. Either way we + // stop trimming and BLOCK writes rather than wiping protected err+ + // history to chase the DB cap. + metrics = get_storage_metrics_with_probe(pool, config, probe)?; + let still_over = db_size_exceeds_trigger(&metrics, config); + tracing::warn!( + logical_db_size_bytes = metrics.logical_db_size_bytes, + free_disk_bytes = ?metrics.free_disk_bytes, + deleted_rows, + db_size_still_over = still_over, + "Self-trim halted — no further deletable rows (err+ floor reached \ + or DB empty); blocking writes instead of deleting protected data" + ); + return Ok(StorageEnforcementOutcome { + metrics, + recovery, + deleted_rows, + // Block if EITHER the DB is still over cap with nothing left to + // safely trim, OR the external disk pressure was already latched. + write_blocked: still_over || disk_write_blocked, + }); + } + + deleted_rows += deleted.deleted_rows; + if deleted.source == TelemetrySource::Logs { + deleted_log_rows += deleted.deleted_rows; + } + tracing::info!( + deleted_rows = deleted.deleted_rows, + total_deleted_rows = deleted_rows, + source = ?deleted.source, + affected_hosts = deleted.log_hostnames.len(), + "Self-trimmed oldest telemetry chunk for storage recovery" + ); + all_hosts.extend(deleted.log_hostnames); + metrics = get_storage_metrics_with_probe(pool, config, probe)?; + } + } + + // Re-evaluate disk pressure against fresh metrics after any self-trim above + // (self-trim frees real bytes, which can lift free disk back over recovery). + disk_write_blocked = disk_pressure_write_blocked(&metrics, config, prev_write_blocked); + + if deleted_rows > 0 { + // Reconcile hosts once after all chunks — avoids N×3 SQL round-trips + // (one per chunk × 3 queries per hostname) competing with the batch writer. + let host_list: Vec = all_hosts.into_iter().collect(); + reconcile_hosts(pool, &host_list)?; + + // Incremental FTS merge — clean up phantom rows left by bulk deletes + // (DELETE trigger is intentionally absent). + // drop the connection before checkpoint_wal_and_incremental_vacuum to + // avoid pool exhaustion when pool_size = 1. + // Pass 0 here so the merge uses DEFAULT_FTS_MERGE_PAGES — storage + // enforcement is rare and the tunable page budget only matters for the + // regular retention path. + if deleted_log_rows > 0 { + fts_incremental_merge(pool, deleted_log_rows, 0); + } + + checkpoint_wal_and_incremental_vacuum(pool, config)?; + } + + tracing::debug!( + deleted_rows, + logical_db_size_bytes = metrics.logical_db_size_bytes, + physical_db_size_bytes = metrics.physical_db_size_bytes, + free_disk_bytes = ?metrics.free_disk_bytes, + write_blocked = disk_write_blocked, + "Storage budget enforcement completed" + ); + + Ok(StorageEnforcementOutcome { + metrics, + recovery, + deleted_rows, + // The DB-size path resolves by self-trim and never blocks here; the only + // reason to block on a clean completion is unresolved EXTERNAL disk pressure. + write_blocked: disk_write_blocked, + }) +} + +/// Run an incremental FTS5 merge to clean up phantom rows left by bulk DELETEs. +/// +/// Uses the only valid FTS5 incremental-merge API — the two-column form +/// `INSERT INTO logs_fts(logs_fts, rank) VALUES('merge', N)` — where `N` is a +/// page budget: the merge processes at most ~N pages of the index and then +/// returns, holding the write lock for milliseconds rather than rewriting the +/// whole index. (The older `'merge=B,M'` STRING syntax this code used does not +/// exist in modern FTS5 and errors on every call.) +/// +/// This function scales the number of merge iterations proportionally to +/// `deleted_rows` (one iteration per 5 000 rows, capped at 20) so a large bulk +/// delete reclaims more phantom space without any single call running long. +/// +/// `merge_pages` is the per-call page budget (from `CORTEX_FTS_MERGE_PAGES`). A +/// value of 0 is treated as the default `DEFAULT_FTS_MERGE_PAGES` because 0 +/// pages is a no-op in the two-arg API. +/// +/// Best-effort: errors are logged but never propagated. A merge that finds +/// nothing to do returns OK (not an error), so ordinary "no phantoms" cycles do +/// not trigger any fallback. On a genuine error (e.g. a busy/locked DB, or real +/// index corruption) we log and stop — we deliberately do NOT auto-escalate to +/// `optimize` or `rebuild`, both of which are O(index-size) under the write lock +/// and were the source of the hourly OOM. Heavy repair is left to an +/// operator-initiated path (see `db_integrity_check`). +fn fts_incremental_merge(pool: &DbPool, deleted_rows: usize, merge_pages: u32) { + // Budget one merge call per 5 000 deleted rows (rough heuristic), with a + // floor of 1 and a ceiling of 20 to bound wall-clock time. + let iterations = deleted_rows.div_ceil(5000).clamp(1, 20); + // 0 pages is a no-op in the two-arg API, so map the "unconditional" sentinel + // to a sane bounded budget. 500 matches the old block-size default. + let pages: i64 = if merge_pages == 0 { + DEFAULT_FTS_MERGE_PAGES + } else { + merge_pages as i64 + }; + + for i in 0..iterations { + match pool.get() { + Ok(conn) => { + let _write_guard = crate::write_lock(); + match conn.execute( + "INSERT INTO logs_fts(logs_fts, rank) VALUES('merge', ?1)", + [pages], + ) { + Ok(_) => { + tracing::trace!( + iteration = i + 1, + total_iterations = iterations, + pages, + "FTS incremental merge iteration" + ); + } + Err(e) => { + // A correctly-formed merge only errors on a genuine + // operational problem (busy/locked) or real corruption. + // Log and stop — never auto-escalate to optimize/rebuild, + // which rewrite the entire index under the write lock. + tracing::warn!( + error = %e, + iteration = i + 1, + "FTS incremental merge failed; stopping (no auto optimize/rebuild)" + ); + return; + } + } + } + Err(e) => { + tracing::warn!(error = %e, "FTS incremental merge: failed to get connection"); + return; + } + } + } +} + +/// Default per-call FTS5 merge page budget when `CORTEX_FTS_MERGE_PAGES` is 0. +/// 500 mirrors the historical block-size default and keeps each merge bounded. +const DEFAULT_FTS_MERGE_PAGES: i64 = 500; + +/// Purge logs older than N days. +/// +/// Uses chunked DELETEs (10 000 rows per iteration) so the WAL write lock is +/// released between chunks, letting the batch writer proceed without timing out +/// or overflowing its 1 000-entry cap. After all chunks complete, an +/// incremental FTS5 merge is issued instead of a full rebuild — a bounded +/// `VALUES('merge', N)` call processes at most N index pages per call and holds +/// the write lock for milliseconds rather than seconds. +/// +/// **High-severity exemption:** rows with `severity IN ('err','crit','alert','emerg')` +/// are excluded from time-based purge — they are never aged out by retention. +/// They CAN still be deleted by `enforce_storage_budget` under disk pressure +/// (oldest-first, no severity filter). Permanent err+ retention is therefore +/// only guaranteed if the DB never breaches `max_db_size_mb` or +/// `min_free_disk_mb`. See CLAUDE.md "Retention" for the policy interaction. +pub fn purge_old_logs(pool: &DbPool, retention_days: u32, fts_merge_pages: u32) -> Result { + if retention_days == 0 { + return Ok(0); + } + + let cutoff = Utc::now() + .checked_sub_signed(chrono::TimeDelta::days(retention_days as i64)) + .ok_or_else(|| { + anyhow::anyhow!("date arithmetic overflow for retention_days={retention_days}") + })? + .format("%Y-%m-%dT%H:%M:%SZ") + .to_string(); + + // Chunked DELETE: each iteration acquires a fresh connection from the pool + // and releases it (along with its write lock) before sleeping, giving the + // batch writer a window to acquire a connection between chunks. + // Use received_at (server clock) instead of timestamp (device clock) so that + // a device with a misconfigured clock cannot cause its logs to be purged + // immediately (future timestamp) or retained forever (past timestamp). + let mut total_deleted: usize = 0; + loop { + let conn = pool.get()?; + let _write_guard = crate::write_lock(); + let chunk = conn.execute( + "DELETE FROM logs WHERE id IN ( + SELECT id FROM logs + WHERE received_at < ?1 + AND severity NOT IN ('err', 'crit', 'alert', 'emerg') + LIMIT 10000 + )", + params![cutoff], + )?; + total_deleted += chunk; + drop(conn); // release back to pool before sleeping + if chunk == 0 { + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + + // Incremental FTS merge — much shorter write-lock duration than full rebuild. + if total_deleted > 0 { + fts_incremental_merge(pool, total_deleted, fts_merge_pages); + } + + // Passive WAL checkpoint: attempt to move WAL pages into the main DB file + // without blocking writers. Prevents unbounded WAL growth between restarts. + { + let conn = pool.get()?; + if let Err(e) = conn.execute_batch("PRAGMA wal_checkpoint(PASSIVE);") { + tracing::warn!(error = %e, "WAL checkpoint skipped (non-fatal)"); + } + } + + tracing::info!(deleted = total_deleted, cutoff = %cutoff, "Purged old logs"); + Ok(total_deleted) +} + +/// Purge heartbeat samples older than N days. +/// +/// Heartbeat tables do not rely on global SQLite foreign-key enforcement, so +/// child metric rows are deleted explicitly before their parent heartbeat rows. +/// Each chunk is its own short transaction to avoid starving log ingest. +pub fn purge_old_heartbeats( + pool: &DbPool, + retention_days: u32, + chunk_size: usize, +) -> Result { + if retention_days == 0 { + return Ok(0); + } + + let cutoff = Utc::now() + .checked_sub_signed(chrono::TimeDelta::days(retention_days as i64)) + .ok_or_else(|| { + anyhow::anyhow!("date arithmetic overflow for retention_days={retention_days}") + })? + .format("%Y-%m-%dT%H:%M:%SZ") + .to_string(); + + let mut total_deleted = 0usize; + let orphan_children = delete_orphan_heartbeat_children(pool)?; + let orphan_latest = delete_orphan_heartbeat_latest(pool)?; + if orphan_children > 0 { + tracing::warn!( + deleted_rows = orphan_children, + "Purged orphan heartbeat child rows" + ); + total_deleted += orphan_children; + } + if orphan_latest > 0 { + tracing::warn!( + deleted_rows = orphan_latest, + "Purged orphan heartbeat latest cache rows" + ); + total_deleted += orphan_latest; + } + loop { + let deleted = delete_heartbeat_chunk_before(pool, &cutoff, chunk_size)?; + total_deleted += deleted; + if deleted == 0 { + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + + tracing::info!( + deleted = total_deleted, + cutoff = %cutoff, + "Purged old heartbeats" + ); + Ok(total_deleted) +} + +/// Delete rows for a single `app_name` older than `max_days`. +/// +/// Same chunked-DELETE pattern as [`purge_old_logs`] (10 000 rows per +/// iteration with the WAL lock released between chunks). Rows with +/// `severity IN ('err','crit','alert','emerg')` are excluded — high-severity +/// log entries are protected from time-based purge regardless of source. +/// +/// Designed for short-retention tags (e.g. `adguard-allowed` at 7 days) +/// running on the new composite index `(app_name, received_at)` introduced +/// by Migration 3. **MUST run before [`purge_old_logs`]** in the maintenance +/// task to avoid SQLite write-lock contention from concurrent chunked +/// DELETEs over the same table. +/// +/// Uses the private `fts_incremental_merge` helper after the loop because FTS5 DELETE triggers +/// were intentionally dropped in Migration 1 — phantoms otherwise accumulate. +pub fn purge_by_tag_window( + pool: &DbPool, + app_name: &str, + max_days: u32, + fts_merge_pages: u32, +) -> Result { + if max_days == 0 { + return Ok(0); + } + + let cutoff = Utc::now() + .checked_sub_signed(chrono::TimeDelta::days(max_days as i64)) + .ok_or_else(|| anyhow::anyhow!("date arithmetic overflow for max_days={max_days}"))? + .format("%Y-%m-%dT%H:%M:%SZ") + .to_string(); + + let mut total_deleted: usize = 0; + loop { + let conn = pool.get()?; + let _write_guard = crate::write_lock(); + let chunk = conn.execute( + "DELETE FROM logs WHERE id IN ( + SELECT id FROM logs + WHERE app_name = ?1 + AND received_at < ?2 + AND severity NOT IN ('err', 'crit', 'alert', 'emerg') + LIMIT 10000 + )", + params![app_name, cutoff], + )?; + total_deleted += chunk; + drop(conn); + if chunk == 0 { + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + + if total_deleted > 0 { + fts_incremental_merge(pool, total_deleted, fts_merge_pages); + } + + tracing::info!( + app_name, + max_days, + deleted = total_deleted, + cutoff = %cutoff, + "Purged tag window" + ); + Ok(total_deleted) +} + +/// Purge `llm_invocations` audit rows older than N days. +/// +/// `llm_invocations` (migration 37, `src/db/llm_invocations.rs`) is the +/// shared audit trail for every LLM-backed assessment call (`ai_assess` +/// today; `skill_assess`/`mcp_assess`/`hook_assess` in later phases). Unlike +/// `logs`, rows here carry no severity concept, so there is no err+-style +/// exemption — retention applies uniformly to every row. Reuses the global +/// `CORTEX_RETENTION_DAYS` setting (0 disables, same as [`purge_old_logs`]) +/// rather than a dedicated hardcoded cap like the AdGuard tags or heartbeats: +/// those exist to override the global policy because their volume would +/// otherwise dominate it, but invocation volume is bounded by `LlmRunner`'s +/// own per-minute/per-hour caps and is far lower than logs/heartbeats. +/// +/// Same chunked-DELETE pattern as [`purge_old_logs`] (`chunk_size` rows per +/// iteration, WAL write lock released between chunks) keyed on `started_at` +/// (set from the server clock at invocation start, not a caller-supplied +/// value) rather than a client-controlled field. No FTS5 merge is needed — +/// `llm_invocations` is not indexed by `logs_fts`. +pub fn purge_old_llm_invocations( + pool: &DbPool, + retention_days: u32, + chunk_size: usize, +) -> Result { + if retention_days == 0 { + return Ok(0); + } + + // `llm_invocations.started_at` is written by SQLite as + // `strftime('%Y-%m-%dT%H:%M:%fZ','now')`, which includes millisecond + // fractional seconds (e.g. `2026-07-01T20:09:50.850Z`). The cutoff must + // carry the same precision: since the comparison is a lexicographic + // string compare (`started_at < cutoff`), a cutoff formatted without + // fractional seconds (e.g. `...:50Z`) does NOT sort the same as it would + // as a real timestamp comparison — `"...:50Z"` can compare greater OR + // less than `"...:50.850Z"` depending on the next byte (`Z` (0x5A) vs + // `.` (0x2E)), misordering rows within the same second. Use + // `SecondsFormat::Millis` (matches `%f`'s 3-digit precision) so the + // comparison is safe. + let cutoff = Utc::now() + .checked_sub_signed(chrono::TimeDelta::days(retention_days as i64)) + .ok_or_else(|| { + anyhow::anyhow!("date arithmetic overflow for retention_days={retention_days}") + })? + .to_rfc3339_opts(chrono::SecondsFormat::Millis, true); + + let mut total_deleted: usize = 0; + loop { + let conn = pool.get()?; + let _write_guard = crate::write_lock(); + let chunk = conn.execute( + "DELETE FROM llm_invocations WHERE id IN ( + SELECT id FROM llm_invocations + WHERE started_at < ?1 + LIMIT ?2 + )", + params![cutoff, chunk_size as i64], + )?; + total_deleted += chunk; + drop(conn); // release back to pool before sleeping + if chunk == 0 { + break; + } + std::thread::sleep(std::time::Duration::from_millis(50)); + } + + tracing::info!( + deleted = total_deleted, + cutoff = %cutoff, + "Purged old llm_invocations rows" + ); + Ok(total_deleted) +} + +#[derive(Debug)] +struct DeletedChunk { + deleted_rows: usize, + hostnames: Vec, +} + +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +enum TelemetrySource { + Logs, + Heartbeats, +} + +#[derive(Debug)] +struct DeletedTelemetryChunk { + deleted_rows: usize, + log_hostnames: Vec, + source: TelemetrySource, +} + +fn oldest_telemetry_source(pool: &DbPool) -> Result> { + let conn = pool.get()?; + let oldest_log: Option = + conn.query_row("SELECT MIN(received_at) FROM logs", [], |row| row.get(0))?; + let oldest_heartbeat: Option = + conn.query_row("SELECT MIN(received_at) FROM host_heartbeats", [], |row| { + row.get(0) + })?; + + Ok(match (oldest_log, oldest_heartbeat) { + (Some(log), Some(heartbeat)) if heartbeat <= log => Some(TelemetrySource::Heartbeats), + (Some(_), Some(_)) | (Some(_), None) => Some(TelemetrySource::Logs), + (None, Some(_)) => Some(TelemetrySource::Heartbeats), + (None, None) => None, + }) +} + +/// Delete the oldest chunk of log rows for DB-size self-trim, honouring the +/// err+ retention FLOOR (syslog-mcp-w4hh). +/// +/// The floor protects, per source IP, the most-recent `err_floor_per_source_cap` +/// rows whose `severity IN ('err','crit','alert','emerg')` received within the +/// last `err_floor_window_hours`. Those rows are EXCLUDED from the deletable set, +/// so self-trim destroys low-value telemetry first and never wipes recent, +/// per-source-bounded high-severity history to chase the DB-size cap. +/// +/// Two security bounds (W1) make this safe against unauthenticated syslog: +/// - **time window** — only recent err+ is protected, so a hostile source +/// cannot pin the floor indefinitely with old severity=err spam; +/// - **per-source cap** — keyed on `source_ip` (the socket peer, which the +/// sender cannot freely vary per packet), NOT the payload `hostname` (which +/// is attacker-controlled), so no single source can monopolise the floor. +/// +/// Returning `deleted_rows == 0` while the DB is still over cap is the signal to +/// the caller that the floor (or an empty deletable set) has been reached; the +/// caller converts that to `write_blocked` instead of deleting protected rows. +fn delete_oldest_logs_chunk( + pool: &DbPool, + chunk_size: usize, + config: &StorageConfig, +) -> Result { + let conn = pool.get()?; + + // Build the protected-id CTE + the deletable selection. When the floor is + // disabled (window or cap == 0) we fall back to the original unfiltered + // oldest-first selection. + let floor_enabled = config.err_floor_window_hours > 0 && config.err_floor_per_source_cap > 0; + + // Window start as an RFC3339 string comparable to `received_at`. + // + // `received_at` is stored with MILLISECOND precision and a `Z` suffix (see + // `app::time::rfc3339_z`, the syslog/docker/OTLP ingest paths). We MUST format + // `window_start` the same way: a second-precision string like + // "...:27Z" sorts AFTER "...:27.680Z" lexicographically (because 'Z'=0x5A > + // '.'=0x2E), so a coarser format would silently protect the wrong rows. + // + // Overflow handling: `err_floor_window_hours` is a u64 and `TimeDelta` is + // i64-hours-bounded. A pathological value would overflow the conversion or the + // subtraction. The old code mapped that to `None`, which downstream collapsed + // to `""` — and `received_at >= ""` is always true, so EVERY err+ row would be + // protected, defeating the trim entirely. Fail fast instead. + let window_start = if floor_enabled { + let hours = i64::try_from(config.err_floor_window_hours).map_err(|_| { + anyhow::anyhow!( + "err_floor_window_hours ({}) is too large to represent as a time delta", + config.err_floor_window_hours + ) + })?; + let delta = chrono::TimeDelta::try_hours(hours).ok_or_else(|| { + anyhow::anyhow!( + "err_floor_window_hours ({hours}) overflows the supported time-delta range" + ) + })?; + let start = Utc::now().checked_sub_signed(delta).ok_or_else(|| { + anyhow::anyhow!( + "err_floor_window_hours ({hours}) underflows the representable timestamp range" + ) + })?; + Some(start.to_rfc3339_opts(chrono::SecondsFormat::Millis, true)) + } else { + None + }; + + // Materialize the protected-id set ONCE per chunk into a TEMP table. + // Previously the window-function subquery (a full err+ window scan) was + // embedded in BOTH the hostnames SELECT and the DELETE, executing twice + // per 1000-row chunk inside the recovery loop while holding the write + // lock (full-review PM3). The DELETE now collects hostnames via + // RETURNING, so the whole chunk costs one window-fn pass and one + // delete-select pass. + // + // `source_ip` is stored as `ip:port`; we PARTITION on the IP portion only + // (strip the ephemeral port) so all packets from one peer share a single + // per-source budget. Window functions require SQLite >= 3.25 and + // RETURNING requires >= 3.35 (rusqlite `bundled` ships 3.4x). + if floor_enabled { + conn.execute_batch( + "CREATE TEMP TABLE IF NOT EXISTS _err_floor_protected (id INTEGER PRIMARY KEY); + DELETE FROM _err_floor_protected;", + )?; + conn.prepare_cached( + "INSERT INTO _err_floor_protected (id) + SELECT id FROM ( + SELECT id, ROW_NUMBER() OVER ( + PARTITION BY substr(source_ip, 1, + CASE WHEN instr(source_ip, ':') > 0 + THEN instr(source_ip, ':') - 1 + ELSE length(source_ip) END) + ORDER BY received_at DESC, id DESC + ) AS rn + FROM logs + WHERE severity IN ('err','crit','alert','emerg') + AND received_at >= :window_start + ) WHERE rn <= :cap", + )? + .execute(rusqlite::named_params! { + ":window_start": window_start.as_deref().unwrap_or(""), + ":cap": config.err_floor_per_source_cap as i64, + })?; + } + + // Delete the deletable chunk, collecting hostnames from the deleted rows + // via RETURNING. When the floor is active, protected err+ rows are never + // in this set — so this path never overrides the err+ exemption. + // Serialize the DELETE behind the process-wide write lock (v1.1.3) so it + // never races other writers against SQLite's single write lock. + let delete_sql = if floor_enabled { + "DELETE FROM logs WHERE id IN ( + SELECT id FROM logs + WHERE id NOT IN (SELECT id FROM _err_floor_protected) + ORDER BY received_at ASC, id ASC LIMIT :chunk) + RETURNING hostname" + } else { + "DELETE FROM logs WHERE id IN ( + SELECT id FROM logs ORDER BY received_at ASC, id ASC LIMIT :chunk) + RETURNING hostname" + }; + let _write_guard = crate::write_lock(); + let deleted_hostnames: Vec = conn + .prepare_cached(delete_sql)? + .query_map( + rusqlite::named_params! { ":chunk": chunk_size as i64 }, + |row| row.get(0), + )? + .collect::>>()?; + let deleted_rows = deleted_hostnames.len(); + let hostnames: Vec = deleted_hostnames + .into_iter() + .collect::>() + .into_iter() + .collect(); + + tracing::debug!( + deleted_rows, + affected_hosts = hostnames.len(), + chunk_size, + floor_enabled, + "Deleted oldest deletable logs chunk (err+ floor honoured)" + ); + + Ok(DeletedChunk { + deleted_rows, + hostnames, + }) +} + +fn delete_oldest_heartbeats_chunk(pool: &DbPool, chunk_size: usize) -> Result { + delete_heartbeat_chunk_where(pool, "", &[], chunk_size) +} + +fn delete_heartbeat_chunk_before(pool: &DbPool, cutoff: &str, chunk_size: usize) -> Result { + delete_heartbeat_chunk_where(pool, "WHERE received_at < ?1", &[cutoff], chunk_size) +} + +fn delete_heartbeat_chunk_where( + pool: &DbPool, + where_clause: &str, + params: &[&str], + chunk_size: usize, +) -> Result { + let mut conn = pool.get()?; + let _write_guard = crate::write_lock(); + let tx = conn.transaction()?; + tx.execute_batch( + "CREATE TEMP TABLE IF NOT EXISTS temp_heartbeat_delete_ids ( + id INTEGER PRIMARY KEY + ); + DELETE FROM temp_heartbeat_delete_ids;", + )?; + + let insert_sql = format!( + "INSERT INTO temp_heartbeat_delete_ids (id) + SELECT id FROM host_heartbeats + {where_clause} + ORDER BY received_at ASC, id ASC + LIMIT ?{}", + params.len() + 1 + ); + let mut values: Vec<&dyn rusqlite::ToSql> = params + .iter() + .map(|value| value as &dyn rusqlite::ToSql) + .collect(); + let chunk_limit = chunk_size as i64; + values.push(&chunk_limit); + tx.execute(&insert_sql, rusqlite::params_from_iter(values))?; + + let selected: usize = tx.query_row( + "SELECT COUNT(*) FROM temp_heartbeat_delete_ids", + [], + |row| row.get::<_, i64>(0), + )? as usize; + if selected == 0 { + tx.execute_batch("DELETE FROM temp_heartbeat_delete_ids;")?; + tx.commit()?; + return Ok(0); + } + + for table in HEARTBEAT_CHILD_TABLES { + tx.execute( + &format!( + "DELETE FROM {table} + WHERE heartbeat_id IN (SELECT id FROM temp_heartbeat_delete_ids)" + ), + [], + )?; + } + let deleted = tx.execute( + "DELETE FROM host_heartbeats + WHERE id IN (SELECT id FROM temp_heartbeat_delete_ids)", + [], + )?; + tx.execute_batch("DELETE FROM temp_heartbeat_delete_ids;")?; + tx.commit()?; + + tracing::debug!( + deleted_rows = deleted, + child_tables = HEARTBEAT_CHILD_TABLES.len(), + chunk_size, + "Deleted heartbeat chunk" + ); + Ok(deleted) +} + +fn delete_orphan_heartbeat_children(pool: &DbPool) -> Result { + let conn = pool.get()?; + let mut total_deleted = 0usize; + for table in HEARTBEAT_CHILD_TABLES { + let _write_guard = crate::write_lock(); + let deleted = conn.execute( + &format!( + "DELETE FROM {table} + WHERE NOT EXISTS ( + SELECT 1 FROM host_heartbeats + WHERE host_heartbeats.id = {table}.heartbeat_id + )" + ), + [], + )?; + total_deleted += deleted; + } + Ok(total_deleted) +} + +pub fn delete_orphan_heartbeat_latest(pool: &DbPool) -> Result { + let conn = pool.get()?; + let _write_guard = crate::write_lock(); + let deleted = conn.execute( + "DELETE FROM host_heartbeats_latest + WHERE NOT EXISTS ( + SELECT 1 FROM host_heartbeats + WHERE host_heartbeats.id = host_heartbeats_latest.heartbeat_id + )", + [], + )?; + Ok(deleted) +} + +const HEARTBEAT_CHILD_TABLES: &[&str] = &[ + "heartbeat_cpu", + "heartbeat_memory", + "heartbeat_disks", + "heartbeat_network", + "heartbeat_processes", + "heartbeat_containers", +]; + +fn reconcile_hosts(pool: &DbPool, hostnames: &[String]) -> Result<()> { + if hostnames.is_empty() { + return Ok(()); + } + + let mut conn = pool.get()?; + let _write_guard = crate::write_lock(); + let tx = conn.transaction()?; + for hostname in hostnames { + // One query: count + timestamp bounds in a single pass over the index. + // MIN/MAX return NULL when count=0, so timestamps are Option. + let (count, first_seen, last_seen): (i64, Option, Option) = tx.query_row( + "SELECT COUNT(*), MIN(received_at), MAX(received_at) + FROM logs WHERE hostname = ?1", + [hostname], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + )?; + + match (count, first_seen, last_seen) { + (0, _, _) | (_, None, _) | (_, _, None) => { + tx.execute("DELETE FROM hosts WHERE hostname = ?1", [hostname])?; + } + (count, Some(first_seen), Some(last_seen)) => { + tx.execute( + "UPDATE hosts + SET first_seen = ?2, last_seen = ?3, log_count = ?4 + WHERE hostname = ?1", + params![hostname, first_seen, last_seen, count], + )?; + } + } + } + tx.commit()?; + tracing::debug!( + host_count = hostnames.len(), + "Reconciled host aggregates after log deletion" + ); + Ok(()) +} + +pub fn maybe_checkpoint_wal_by_size( + pool: &DbPool, + db_path: &Path, + threshold_bytes: u64, +) -> Result> { + if threshold_bytes == 0 { + return Ok(None); + } + let wal_path = sqlite_sidecar_path(db_path, "wal"); + let wal_size = match std::fs::metadata(&wal_path) { + Ok(metadata) => metadata.len(), + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(None), + Err(error) => return Err(error.into()), + }; + if wal_size < threshold_bytes { + return Ok(None); + } + let passive = db_wal_checkpoint(pool, "passive")?; + if wal_checkpoint_complete(passive.0, passive.1, passive.2) { + // PASSIVE copies frames into the main database but deliberately keeps + // the WAL file at its high-water size. Once every frame is checkpointed, + // a bounded TRUNCATE checkpoint can safely release that disk space. + let truncate = db_wal_checkpoint(pool, "truncate")?; + if !wal_checkpoint_complete(truncate.0, truncate.1, truncate.2) { + tracing::warn!( + busy = truncate.0, + log_frames = truncate.1, + checkpointed_frames = truncate.2, + "WAL truncate checkpoint incomplete" + ); + } + } + Ok(Some(passive)) +} + +pub fn checkpoint_wal_and_incremental_vacuum(pool: &DbPool, config: &StorageConfig) -> Result<()> { + match maybe_checkpoint_wal_by_size( + pool, + &config.db_path, + config.wal_checkpoint_threshold_bytes(), + ) { + Ok(Some((busy, log_frames, checkpointed_frames))) => { + if busy != 0 || checkpointed_frames < log_frames { + tracing::warn!( + busy, + log_frames, + checkpointed_frames, + "WAL threshold checkpoint incomplete" + ); + } else { + tracing::debug!( + busy, + log_frames, + checkpointed_frames, + "WAL threshold checkpoint completed" + ); + } + } + Ok(None) => tracing::debug!("WAL threshold checkpoint skipped"), + Err(error) => { + tracing::warn!(error = %error, "WAL threshold checkpoint skipped (non-fatal)"); + } + } + let conn = pool.get()?; + let _write_guard = crate::write_lock(); + match conn.execute_batch("PRAGMA incremental_vacuum(1000);") { + Err(e) => { + tracing::warn!(error = %e, "incremental vacuum skipped (non-fatal)"); + } + _ => { + tracing::debug!("Incremental vacuum completed"); + } + } + Ok(()) +} + +fn storage_limits_enabled(config: &StorageConfig) -> bool { + config.max_db_size_mb > 0 || config.min_free_disk_mb > 0 +} + +fn recovery_targets(config: &StorageConfig) -> StorageRecovery { + StorageRecovery { + logical_db_size_bytes: mb_to_bytes(config.recovery_db_size_mb), + free_disk_bytes: (config.min_free_disk_mb > 0) + .then(|| mb_to_bytes(config.recovery_free_disk_mb)), + } +} + +/// Combined trigger: true if EITHER the DB-size cap or the free-disk floor is +/// breached. Retained for the read-only health/stats surfaces (queries.rs, +/// service.rs) that report whether writes are currently constrained — they want +/// the OR of both conditions. Enforcement itself uses the split helpers below so +/// the two pressures get distinct remediation. +pub fn exceeds_trigger(metrics: &StorageMetrics, config: &StorageConfig) -> bool { + db_size_exceeds_trigger(metrics, config) || disk_free_below_trigger(metrics, config) +} + +/// DB-size trigger: cortex's OWN logical bytes exceed `max_db_size_mb`. +/// Resolvable by self-trim. +fn db_size_exceeds_trigger(metrics: &StorageMetrics, config: &StorageConfig) -> bool { + config.max_db_size_mb > 0 && metrics.logical_db_size_bytes > mb_to_bytes(config.max_db_size_mb) +} + +/// Free-disk trigger: whole-filesystem free space is below `min_free_disk_mb`. +/// EXTERNAL — never resolved by deleting cortex's own data. +fn disk_free_below_trigger(metrics: &StorageMetrics, config: &StorageConfig) -> bool { + // FAIL-CLOSED: when the free-disk guardrail is enabled (`min_free_disk_mb > 0`) + // but the statvfs probe failed (`free_disk_bytes == None`), treat free space as + // 0 (unknown == worst case) so the guardrail engages conservatively instead of + // silently disabling itself. With the guardrail disabled the function short- + // circuits on `> 0` and never inspects the probe at all. + config.min_free_disk_mb > 0 + && metrics.free_disk_bytes.unwrap_or(0) < mb_to_bytes(config.min_free_disk_mb) +} + +/// Self-trim recovery exit: the DB-size loop stops once logical size is at or +/// below `recovery_db_size_mb`. Deliberately ignores the free-disk arm so the +/// self-trim loop is NOT gated by an external condition it cannot fix. +fn db_size_within_recovery( + metrics: &StorageMetrics, + recovery: &StorageRecovery, + config: &StorageConfig, +) -> bool { + config.max_db_size_mb == 0 || metrics.logical_db_size_bytes <= recovery.logical_db_size_bytes +} + +/// Hysteresis decision for the external free-disk write-block. +/// +/// - Below `min_free_disk_mb` → engage the block. +/// - At/above `recovery_free_disk_mb` → clear the block. +/// - In the (min, recovery) hysteresis band → keep whatever the previous tick +/// decided (`prev`). This needs prior state: the answer in the band is not a +/// pure function of current metrics, which is exactly why the block latches +/// instead of flapping at the trigger threshold. +fn disk_pressure_write_blocked( + metrics: &StorageMetrics, + config: &StorageConfig, + prev: bool, +) -> bool { + if config.min_free_disk_mb == 0 { + return false; + } + // FAIL-CLOSED: the guardrail is enabled here, so a failed statvfs probe + // (`None`) is treated as 0 free bytes (worst case) — the block engages rather + // than fails open. Mirrors `disk_free_below_trigger`. + let free = metrics.free_disk_bytes.unwrap_or(0); + if free < mb_to_bytes(config.min_free_disk_mb) { + true + } else if free >= mb_to_bytes(config.recovery_free_disk_mb) { + false + } else { + prev + } +} + +fn mb_to_bytes(mb: u64) -> u64 { + mb.saturating_mul(1_048_576) +} + +fn physical_db_size_bytes(db_path: &Path) -> Result { + let mut total = file_size_if_exists(db_path)?; + total += file_size_if_exists(&sqlite_sidecar_path(db_path, "wal"))?; + total += file_size_if_exists(&sqlite_sidecar_path(db_path, "shm"))?; + Ok(total) +} + +pub(crate) fn sqlite_sidecar_path(db_path: &Path, suffix: &str) -> PathBuf { + let mut path = db_path.as_os_str().to_os_string(); + path.push("-"); + path.push(suffix); + PathBuf::from(path) +} + +fn file_size_if_exists(path: &Path) -> Result { + match std::fs::metadata(path) { + Ok(metadata) => Ok(metadata.len()), + Err(err) if err.kind() == std::io::ErrorKind::NotFound => Ok(0), + Err(err) => Err(err.into()), + } +} + +#[cfg(test)] +#[path = "maintenance_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/maintenance_tests.rs b/crates/shared/cortex/storage-sqlite/src/maintenance_tests.rs new file mode 100644 index 00000000..c5af8d23 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/maintenance_tests.rs @@ -0,0 +1,1341 @@ +use super::*; +use crate::config::StorageConfig; +use crate::{DbPool, LogBatchEntry, init_pool, insert_logs_batch, list_hosts, tail_logs}; +use anyhow::Result; +use rusqlite::params; +use std::path::Path; + +fn test_storage_config(db_path: std::path::PathBuf) -> StorageConfig { + StorageConfig::for_test(db_path) +} + +/// Create an isolated test pool using a temp file (not :memory: — FTS5 needs file) +fn test_pool() -> (DbPool, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let config = test_storage_config(db_path); + let pool = init_pool(&config).unwrap(); + (pool, dir) // keep dir alive for test duration +} + +fn make_entry(ts: &str, host: &str, severity: &str, msg: &str) -> LogBatchEntry { + LogBatchEntry { + timestamp: ts.to_string(), + hostname: host.to_string(), + facility: None, + severity: severity.to_string(), + app_name: None, + process_id: None, + message: msg.to_string(), + raw: msg.to_string(), + source_ip: "127.0.0.1:514".to_string(), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + } +} + +fn update_received_at(pool: &DbPool, message: &str, received_at: &str) { + let conn = pool.get().unwrap(); + conn.execute( + "UPDATE logs SET received_at = ?1 WHERE message = ?2", + params![received_at, message], + ) + .unwrap(); +} + +fn insert_heartbeat(pool: &DbPool, hostname: &str, received_at: &str) -> i64 { + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO host_heartbeats ( + host_id, hostname, source_ip, sampled_at, received_at, boot_id, uptime_secs, + sequence, collection_ms, agent_version, os, architecture + ) VALUES ( + ?1, ?2, '127.0.0.1', ?3, ?3, ?4, 1, 1, 1, 'test', 'linux', 'x86_64' + )", + params![hostname, hostname, received_at, format!("boot-{hostname}")], + ) + .unwrap(); + conn.last_insert_rowid() +} + +#[test] +fn test_storage_metrics_report_logical_size() { + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("metrics.db")); + let pool = init_pool(&config).unwrap(); + insert_logs_batch( + &pool, + &[make_entry( + "2026-01-01T00:00:01Z", + "host-a", + "info", + "hello", + )], + ) + .unwrap(); + + let metrics = get_storage_metrics(&pool, &config).unwrap(); + assert!(metrics.logical_db_size_bytes > 0); + assert!(metrics.physical_db_size_bytes >= metrics.logical_db_size_bytes); + assert!(metrics.free_disk_bytes.is_some()); +} + +#[test] +fn physical_db_size_counts_extensionless_sqlite_sidecars() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("cortex"); + std::fs::write(&db_path, b"db").unwrap(); + std::fs::write(sqlite_sidecar_path(&db_path, "wal"), b"wal").unwrap(); + std::fs::write(sqlite_sidecar_path(&db_path, "shm"), b"shm").unwrap(); + + let total = physical_db_size_bytes(&db_path).unwrap(); + assert_eq!(total, 2 + 3 + 3); +} + +#[test] +fn maybe_checkpoint_wal_by_size_skips_missing_disabled_and_below_threshold_wal() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("missing.db"); + let config = test_storage_config(db_path.clone()); + let pool = init_pool(&config).unwrap(); + + assert!( + maybe_checkpoint_wal_by_size(&pool, &db_path, 0) + .unwrap() + .is_none() + ); + assert!( + maybe_checkpoint_wal_by_size(&pool, &db_path, u64::MAX) + .unwrap() + .is_none() + ); +} + +#[test] +fn maybe_checkpoint_wal_by_size_runs_when_wal_exceeds_threshold() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("wal-threshold.db"); + let mut config = test_storage_config(db_path.clone()); + config.pool_size = 2; + config.wal_mode = true; + let pool = init_pool(&config).unwrap(); + let held_conn = pool.get().unwrap(); + held_conn + .execute_batch( + "CREATE TABLE wal_threshold_probe(id INTEGER PRIMARY KEY, value TEXT); + INSERT INTO wal_threshold_probe(value) VALUES ('x');", + ) + .unwrap(); + assert!( + sqlite_sidecar_path(&db_path, "wal").exists(), + "test setup should create a WAL sidecar" + ); + + let checkpoint = maybe_checkpoint_wal_by_size(&pool, &db_path, 1) + .unwrap() + .expect("WAL above tiny threshold should checkpoint"); + assert!(checkpoint.1 >= checkpoint.2); + drop(held_conn); +} + +#[test] +fn threshold_maintenance_truncates_a_fully_checkpointed_wal() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("wal-truncate.db"); + let mut config = test_storage_config(db_path.clone()); + config.pool_size = 2; + config.wal_mode = true; + config.wal_checkpoint_mb = 1; + let pool = init_pool(&config).unwrap(); + { + let conn = pool.get().unwrap(); + conn.execute_batch("CREATE TABLE wal_truncate_probe(id INTEGER PRIMARY KEY, value BLOB);") + .unwrap(); + let payload = vec![b'x'; 64 * 1024]; + for _ in 0..32 { + conn.execute( + "INSERT INTO wal_truncate_probe(value) VALUES (?1)", + [&payload], + ) + .unwrap(); + } + } + let wal_path = sqlite_sidecar_path(&db_path, "wal"); + let before = std::fs::metadata(&wal_path).unwrap().len(); + assert!(before > config.wal_checkpoint_threshold_bytes()); + + checkpoint_wal_and_incremental_vacuum(&pool, &config).unwrap(); + + let after = std::fs::metadata(&wal_path).map(|m| m.len()).unwrap_or(0); + assert!( + after < before, + "fully checkpointed WAL should be truncated below its high-water size: before={before} after={after}" + ); +} + +#[test] +fn test_purge_old_logs_removes_old() { + let (pool, _dir) = test_pool(); + let entries = vec![ + make_entry("2020-01-01T00:00:00Z", "host-a", "info", "old message"), + make_entry("2099-01-01T00:00:00Z", "host-a", "info", "future message"), + ]; + insert_logs_batch(&pool, &entries).unwrap(); + + // Purge uses received_at (server clock), not timestamp (device clock). + // Backdate the first entry's received_at so it falls outside retention. + let conn = pool.get().unwrap(); + conn.execute( + "UPDATE logs SET received_at = '2020-01-01T00:00:00Z' WHERE message = 'old message'", + [], + ) + .unwrap(); + drop(conn); + + let deleted = purge_old_logs(&pool, 90, 0).unwrap(); + assert_eq!(deleted, 1, "should have deleted exactly the old entry"); + + let remaining = tail_logs(&pool, None, None, None, None, 10).unwrap(); + assert_eq!(remaining.len(), 1); + assert_eq!(remaining[0].message, "future message"); +} + +#[test] +fn test_purge_zero_retention_noop() { + let (pool, _dir) = test_pool(); + let entries = vec![make_entry("2020-01-01T00:00:00Z", "host-a", "info", "old")]; + insert_logs_batch(&pool, &entries).unwrap(); + + let deleted = purge_old_logs(&pool, 0, 0).unwrap(); + assert_eq!(deleted, 0, "retention_days=0 should be a no-op"); +} + +#[test] +fn test_purge_old_heartbeats_removes_children_before_parent() { + let (pool, _dir) = test_pool(); + let old_id = insert_heartbeat(&pool, "host-old", "2020-01-01T00:00:00Z"); + let new_id = insert_heartbeat(&pool, "host-new", "2099-01-01T00:00:00Z"); + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO heartbeat_cpu (heartbeat_id, usage_percent) VALUES (?1, 10.0)", + [old_id], + ) + .unwrap(); + conn.execute( + "INSERT INTO heartbeat_cpu (heartbeat_id, usage_percent) VALUES (?1, 20.0)", + [new_id], + ) + .unwrap(); + drop(conn); + + let deleted = purge_old_heartbeats(&pool, 90, 100).unwrap(); + assert_eq!(deleted, 1); + + let conn = pool.get().unwrap(); + let old_rows: i64 = conn + .query_row( + "SELECT COUNT(*) FROM host_heartbeats WHERE hostname = 'host-old'", + [], + |row| row.get(0), + ) + .unwrap(); + let old_child_rows: i64 = conn + .query_row( + "SELECT COUNT(*) FROM heartbeat_cpu WHERE heartbeat_id = ?1", + [old_id], + |row| row.get(0), + ) + .unwrap(); + let new_rows: i64 = conn + .query_row( + "SELECT COUNT(*) FROM host_heartbeats WHERE hostname = 'host-new'", + [], + |row| row.get(0), + ) + .unwrap(); + let new_child_rows: i64 = conn + .query_row( + "SELECT COUNT(*) FROM heartbeat_cpu WHERE heartbeat_id = ?1", + [new_id], + |row| row.get(0), + ) + .unwrap(); + + assert_eq!(old_rows, 0); + assert_eq!(old_child_rows, 0); + assert_eq!(new_rows, 1); + assert_eq!(new_child_rows, 1); +} + +#[test] +fn test_heartbeat_cleanup_removes_all_child_tables_and_orphans() { + let (pool, _dir) = test_pool(); + let heartbeat_id = insert_heartbeat(&pool, "host-old", "2020-01-01T00:00:00Z"); + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO heartbeat_cpu (heartbeat_id, usage_percent) VALUES (?1, 10.0)", + [heartbeat_id], + ) + .unwrap(); + conn.execute( + "INSERT INTO heartbeat_memory (heartbeat_id, total_bytes) VALUES (?1, 1024)", + [heartbeat_id], + ) + .unwrap(); + conn.execute( + "INSERT INTO heartbeat_disks (heartbeat_id, mountpoint) VALUES (?1, '/dev/sda')", + [heartbeat_id], + ) + .unwrap(); + conn.execute( + "INSERT INTO heartbeat_network (heartbeat_id, interface) VALUES (?1, 'eth0')", + [heartbeat_id], + ) + .unwrap(); + conn.execute( + "INSERT INTO heartbeat_processes (heartbeat_id, total) VALUES (?1, 10)", + [heartbeat_id], + ) + .unwrap(); + conn.execute( + "INSERT INTO heartbeat_containers (heartbeat_id, running) VALUES (?1, 1)", + [heartbeat_id], + ) + .unwrap(); + conn.execute( + "INSERT INTO heartbeat_cpu (heartbeat_id, usage_percent) VALUES (999999, 99.0)", + [], + ) + .unwrap(); + drop(conn); + + let deleted = purge_old_heartbeats(&pool, 90, 100).unwrap(); + assert_eq!(deleted, 2); + + let conn = pool.get().unwrap(); + for table in HEARTBEAT_CHILD_TABLES { + let remaining: i64 = conn + .query_row(&format!("SELECT COUNT(*) FROM {table}"), [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(remaining, 0, "{table} should be empty after cleanup"); + } +} + +#[test] +fn test_enforce_storage_budget_keeps_recent_heartbeats_when_logs_are_older() { + let (pool, dir) = test_pool(); + let large_old = "old-log-".repeat(350_000); + let entries = vec![make_entry( + "2026-01-01T00:00:01Z", + "deleted-host", + "info", + &large_old, + )]; + insert_logs_batch(&pool, &entries).unwrap(); + update_received_at(&pool, &large_old, "2020-01-01T00:00:00Z"); + let heartbeat_id = insert_heartbeat(&pool, "recent-heartbeat", "2099-01-01T00:00:00Z"); + + let mut config = test_storage_config(dir.path().join("test.db")); + config.max_db_size_mb = 3; + config.recovery_db_size_mb = 2; + + let outcome = enforce_storage_budget(&pool, &config).unwrap(); + assert!(outcome.deleted_rows > 0); + + let conn = pool.get().unwrap(); + let heartbeats: i64 = conn + .query_row( + "SELECT COUNT(*) FROM host_heartbeats WHERE id = ?1", + [heartbeat_id], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(heartbeats, 1); + drop(conn); + + let logs = tail_logs(&pool, None, None, None, None, 10).unwrap(); + assert!(logs.is_empty()); +} + +#[test] +fn test_enforce_storage_budget_deletes_by_received_at_until_recovery_target() { + let (pool, dir) = test_pool(); + let large_old = "oldest-".repeat(350_000); + let large_new = "newest-".repeat(30_000); + let entries = vec![ + make_entry("2026-01-01T00:00:01Z", "deleted-host", "info", &large_old), + make_entry("2026-01-01T00:00:02Z", "surviving-host", "info", &large_new), + ]; + insert_logs_batch(&pool, &entries).unwrap(); + update_received_at(&pool, &large_old, "2026-01-01T00:00:00Z"); + update_received_at(&pool, &large_new, "2026-01-02T00:00:00Z"); + + let mut config = test_storage_config(dir.path().join("test.db")); + config.max_db_size_mb = 3; + config.recovery_db_size_mb = 2; + + let outcome = enforce_storage_budget(&pool, &config).unwrap(); + assert!(outcome.deleted_rows > 0); + assert!(outcome.metrics.logical_db_size_bytes <= outcome.recovery.logical_db_size_bytes); + + let rows = tail_logs(&pool, None, None, None, None, 10).unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].message, large_new); +} + +#[test] +fn test_enforce_storage_budget_reconciles_hosts_after_deletes() { + let (pool, dir) = test_pool(); + let large_oldest = "delete-me-1-".repeat(150_000); + let large_older = "delete-me-2-".repeat(150_000); + // Deliberately small relative to `recovery_db_size_mb` below: after both + // `deleted-host` rows are trimmed, this row alone (plus fixed schema/ + // index overhead, which grows slowly as the schema gains tables/indexes + // over time) must land comfortably under the 2MB recovery target with + // real headroom — not within a single SQLite page (4096 bytes) of it. + // A prior version of this fixture sized `large_keep` right at that + // boundary, so an unrelated schema change (e.g. a new index) could tip + // post-delete size a few KB over the target and cause the loop to trim + // one extra (wrongly "surviving") row. See PR2 (GH #94) investigation. + let large_keep = "keep-me-".repeat(5_000); + let entries = vec![ + make_entry( + "2026-01-01T00:00:01Z", + "deleted-host", + "info", + &large_oldest, + ), + make_entry("2026-01-01T00:00:02Z", "deleted-host", "info", &large_older), + make_entry( + "2026-01-01T00:00:03Z", + "surviving-host", + "info", + &large_keep, + ), + ]; + insert_logs_batch(&pool, &entries).unwrap(); + update_received_at(&pool, &large_oldest, "2026-01-01T00:00:00Z"); + update_received_at(&pool, &large_older, "2026-01-01T00:00:01Z"); + update_received_at(&pool, &large_keep, "2026-01-02T00:00:00Z"); + + let mut config = test_storage_config(dir.path().join("test.db")); + config.max_db_size_mb = 3; + config.recovery_db_size_mb = 2; + + enforce_storage_budget(&pool, &config).unwrap(); + + let hosts = list_hosts(&pool).unwrap(); + assert!(hosts.iter().all(|host| host.hostname != "deleted-host")); + let surviving = hosts + .iter() + .find(|host| host.hostname == "surviving-host") + .unwrap(); + assert_eq!(surviving.log_count, 1); +} + +#[test] +fn delete_orphan_heartbeat_latest_removes_cache_rows_without_parent_sample() { + let (pool, _dir) = test_pool(); + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO host_heartbeats_latest + (host_id, heartbeat_id, hostname, sampled_at, received_at, + partial, agent_version, os, architecture, metadata_json) + VALUES ('host-a', 12345, 'devhost', '2026-07-16T00:00:00Z', + '2026-07-16T00:00:00Z', 0, '0.1.0', 'linux', 'x86_64', '{}')", + [], + ) + .unwrap(); + drop(conn); + + let deleted = delete_orphan_heartbeat_latest(&pool).unwrap(); + + assert_eq!(deleted, 1); + let remaining: i64 = pool + .get() + .unwrap() + .query_row("SELECT COUNT(*) FROM host_heartbeats_latest", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(remaining, 0); +} + +#[derive(Clone)] +struct FakeDiskSpaceProbe { + values: std::sync::Arc>>, +} + +impl FakeDiskSpaceProbe { + fn new(values: Vec) -> Self { + Self { + values: std::sync::Arc::new(std::sync::Mutex::new(values)), + } + } +} + +impl DiskSpaceProbe for FakeDiskSpaceProbe { + fn free_bytes(&self, _path: &Path) -> Result { + let mut values = self.values.lock().unwrap(); + let value = if values.len() > 1 { + values.remove(0) + } else { + *values.first().unwrap_or(&0) + }; + Ok(value) + } +} + +/// syslog-mcp-w4hh: low whole-filesystem free space is an EXTERNAL condition. +/// Cortex must NOT delete its own data to chase it — it blocks writes instead. +#[test] +fn external_disk_pressure_does_not_delete() { + let (pool, dir) = test_pool(); + let entries = vec![ + make_entry("2026-01-01T00:00:01Z", "host-a", "info", "older"), + make_entry("2026-01-01T00:00:02Z", "host-b", "info", "newer"), + ]; + insert_logs_batch(&pool, &entries).unwrap(); + update_received_at(&pool, "older", "2026-01-01T00:00:00Z"); + update_received_at(&pool, "newer", "2026-01-02T00:00:00Z"); + + // DB-size limit disabled; only the free-disk floor is active. The DB itself + // is tiny, so the disk pressure is genuinely external. + let mut config = test_storage_config(dir.path().join("test.db")); + config.max_db_size_mb = 0; + config.recovery_db_size_mb = 0; + config.min_free_disk_mb = 512; + config.recovery_free_disk_mb = 768; + + // Probe reports a persistently low free-disk value (well below the trigger). + let probe = FakeDiskSpaceProbe::new(vec![64 * 1_048_576]); + let outcome = enforce_storage_budget_with_probe(&pool, &config, &probe).unwrap(); + + assert_eq!( + outcome.deleted_rows, 0, + "must NOT delete own data under external disk pressure" + ); + assert!( + outcome.write_blocked, + "must block writes while free disk is below the floor" + ); + + let rows = tail_logs(&pool, None, None, None, None, 10).unwrap(); + assert_eq!(rows.len(), 2, "both rows must survive — nothing deleted"); + + // Notification policy consumes this storage outcome in the application layer. + // This crate proves only the persistence invariant: external pressure blocks + // writes without deleting retained data. +} + +/// syslog-mcp-w4hh: when the DB grows past max_db_size_mb but the only remaining +/// rows are floor-protected err+ (recent window + within per-source cap), self-trim +/// must STOP at the floor and convert to write_blocked rather than wiping err+. +#[test] +fn self_trim_respects_err_floor() { + let (pool, dir) = test_pool(); + let now = chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(); + // One large, deletable info row (oldest) + several recent err+ rows that the + // floor protects. The err rows are big enough that, even after the info row is + // trimmed, the DB stays over the recovery target. + let big_info = "info-junk-".repeat(120_000); + let big_err1 = "err-keep-1-".repeat(120_000); + let big_err2 = "err-keep-2-".repeat(120_000); + insert_logs_batch( + &pool, + &[ + make_entry("2026-01-01T00:00:01Z", "host-a", "info", &big_info), + make_entry("2026-01-01T00:00:02Z", "host-a", "err", &big_err1), + make_entry("2026-01-01T00:00:03Z", "host-a", "crit", &big_err2), + ], + ) + .unwrap(); + // info row is oldest; err rows are received "now" so they are inside the window. + update_received_at(&pool, &big_info, "2026-01-01T00:00:00Z"); + update_received_at(&pool, &big_err1, &now); + update_received_at(&pool, &big_err2, &now); + + let mut config = test_storage_config(dir.path().join("test.db")); + config.max_db_size_mb = 2; + config.recovery_db_size_mb = 1; // recovery target the err rows alone exceed + config.min_free_disk_mb = 0; + config.recovery_free_disk_mb = 0; + config.cleanup_chunk_size = 1; + config.err_floor_window_hours = 24; + config.err_floor_per_source_cap = 10_000; + + let outcome = enforce_storage_budget(&pool, &config).unwrap(); + + // err+ rows must survive — the floor protected them. + let rows = tail_logs(&pool, None, None, None, None, 10).unwrap(); + let messages: Vec<&str> = rows.iter().map(|r| r.message.as_str()).collect(); + assert!( + messages.contains(&big_err1.as_str()), + "err row must survive the floor" + ); + assert!( + messages.contains(&big_err2.as_str()), + "crit row must survive the floor" + ); + assert!( + !messages.contains(&big_info.as_str()), + "the deletable info row should have been trimmed" + ); + // DB is still over cap (err+ retained), so writes must be blocked rather than + // the err+ history wiped. + assert!( + outcome.write_blocked, + "must block writes once trim reaches the err+ floor while still over cap" + ); +} + +/// Helper: insert a log with an explicit source_ip (the socket peer), used by the +/// W1 bound tests below to exercise the per-source partition. +fn make_entry_from( + ts: &str, + host: &str, + severity: &str, + source_ip: &str, + msg: &str, +) -> LogBatchEntry { + let mut e = make_entry(ts, host, severity, msg); + e.source_ip = source_ip.to_string(); + e +} + +/// syslog-mcp-w4hh W1 (monopolization defense): the per-source cap BOUNDS how +/// much err+ a single source IP can keep in the protected set. With cap=2, only +/// the 2 newest err+ rows from one source survive self-trim; the rest are +/// deletable even though they are inside the time window and high severity. +#[test] +fn err_floor_per_source_cap_evicts_excess() { + let (pool, dir) = test_pool(); + let now = chrono::Utc::now(); + // Five large err rows from the SAME source IP, all recent, staggered by second. + let mut msgs = Vec::new(); + for i in 0..5 { + let ts = (now - chrono::TimeDelta::seconds(10 - i)) + .format("%Y-%m-%dT%H:%M:%SZ") + .to_string(); + let msg = format!("err-{i}-{}", "z".repeat(700_000)); + insert_logs_batch( + &pool, + &[make_entry_from(&ts, "host-a", "err", "10.0.0.5:5000", &msg)], + ) + .unwrap(); + update_received_at(&pool, &msg, &ts); + msgs.push(msg); + } + + let mut config = test_storage_config(dir.path().join("test.db")); + config.max_db_size_mb = 2; + config.recovery_db_size_mb = 1; + config.min_free_disk_mb = 0; + config.recovery_free_disk_mb = 0; + config.cleanup_chunk_size = 1; + config.err_floor_window_hours = 24; + config.err_floor_per_source_cap = 2; // only 2 protected per source IP + + enforce_storage_budget(&pool, &config).unwrap(); + + let rows = tail_logs(&pool, None, None, None, None, 10).unwrap(); + let surviving: Vec<&str> = rows.iter().map(|r| r.message.as_str()).collect(); + // The 2 NEWEST err rows (indices 3,4) must survive; older ones evicted. + assert!( + surviving.contains(&msgs[4].as_str()) && surviving.contains(&msgs[3].as_str()), + "the 2 newest err rows from the source must be protected" + ); + assert!( + surviving.len() <= 2, + "per-source cap=2 must bound the source's protected err+; got {} survivors", + surviving.len() + ); + assert!( + !surviving.contains(&msgs[0].as_str()), + "the oldest err row must be evicted beyond the cap (monopolization bound)" + ); +} + +/// syslog-mcp-w4hh W1 (unbounded-pin defense): the time window BOUNDS how far +/// back the floor protects. err+ rows received OUTSIDE the window are deletable +/// by self-trim, so a hostile source cannot pin the DB at max with stale err spam. +#[test] +fn err_floor_window_evicts_stale_err() { + let (pool, dir) = test_pool(); + let big_stale_err = "stale-err-".repeat(120_000); + let big_recent_err = "recent-err-".repeat(120_000); + insert_logs_batch( + &pool, + &[ + make_entry_from( + "2026-01-01T00:00:01Z", + "host-a", + "err", + "10.0.0.9:6000", + &big_stale_err, + ), + make_entry_from( + "2026-01-01T00:00:02Z", + "host-a", + "err", + "10.0.0.9:6000", + &big_recent_err, + ), + ], + ) + .unwrap(); + let now = chrono::Utc::now(); + // Stale err: 48h ago, well outside a 1h window → NOT protected → deletable. + let stale_ts = (now - chrono::TimeDelta::hours(48)) + .format("%Y-%m-%dT%H:%M:%SZ") + .to_string(); + let recent_ts = now.format("%Y-%m-%dT%H:%M:%SZ").to_string(); + update_received_at(&pool, &big_stale_err, &stale_ts); + update_received_at(&pool, &big_recent_err, &recent_ts); + + let mut config = test_storage_config(dir.path().join("test.db")); + config.max_db_size_mb = 2; + config.recovery_db_size_mb = 1; + config.min_free_disk_mb = 0; + config.recovery_free_disk_mb = 0; + config.cleanup_chunk_size = 1; + config.err_floor_window_hours = 1; // 1h window — stale err falls outside + config.err_floor_per_source_cap = 10_000; + + enforce_storage_budget(&pool, &config).unwrap(); + + let rows = tail_logs(&pool, None, None, None, None, 10).unwrap(); + let surviving: Vec<&str> = rows.iter().map(|r| r.message.as_str()).collect(); + assert!( + !surviving.contains(&big_stale_err.as_str()), + "stale err+ outside the window must be deletable (unbounded-pin bound)" + ); + assert!( + surviving.contains(&big_recent_err.as_str()), + "recent err+ inside the window must still be protected" + ); +} + +/// syslog-mcp-w4hh: hysteresis. The external disk-pressure block engages at +/// min_free_disk_mb and clears only at recovery_free_disk_mb — in the band between +/// them the prior state is carried forward (latch, no flap). +#[test] +fn disk_pressure_write_block_uses_hysteresis() { + let (pool, dir) = test_pool(); + insert_logs_batch( + &pool, + &[make_entry("2026-01-01T00:00:01Z", "host-a", "info", "x")], + ) + .unwrap(); + + let mut config = test_storage_config(dir.path().join("test.db")); + config.max_db_size_mb = 0; + config.recovery_db_size_mb = 0; + config.min_free_disk_mb = 512; + config.recovery_free_disk_mb = 768; + + // Below trigger (512MB): engages regardless of prior state. + let low = FakeDiskSpaceProbe::new(vec![100 * 1_048_576]); + let blocked = enforce_storage_budget_with_state(&pool, &config, &low, false).unwrap(); + assert!(blocked.write_blocked, "below min: must block"); + assert_eq!(blocked.deleted_rows, 0, "must not delete on disk pressure"); + + // In the hysteresis band (600MB, between 512 and 768): keep prior state. + let band = FakeDiskSpaceProbe::new(vec![600 * 1_048_576]); + let still_blocked = enforce_storage_budget_with_state(&pool, &config, &band, true).unwrap(); + assert!( + still_blocked.write_blocked, + "in band with prev=true: stay blocked (latch)" + ); + let stays_clear = enforce_storage_budget_with_state(&pool, &config, &band, false).unwrap(); + assert!( + !stays_clear.write_blocked, + "in band with prev=false: stay clear (no premature engage)" + ); + + // At/above recovery (800MB): clear regardless of prior state. + let high = FakeDiskSpaceProbe::new(vec![800 * 1_048_576]); + let cleared = enforce_storage_budget_with_state(&pool, &config, &high, true).unwrap(); + assert!( + !cleared.write_blocked, + "at recovery threshold: must clear even if prev=true" + ); +} + +#[test] +fn test_enforce_storage_budget_is_noop_when_limits_disabled() { + let (pool, dir) = test_pool(); + let config = test_storage_config(dir.path().join("test.db")); + let mut disabled = config.clone(); + disabled.max_db_size_mb = 0; + disabled.recovery_db_size_mb = 0; + disabled.min_free_disk_mb = 0; + disabled.recovery_free_disk_mb = 0; + + let outcome = enforce_storage_budget(&pool, &disabled).unwrap(); + assert_eq!(outcome.deleted_rows, 0); + assert!(!outcome.write_blocked); +} + +// ---- purge_by_tag_window ---- + +fn make_tagged(ts: &str, host: &str, severity: &str, app: &str, msg: &str) -> LogBatchEntry { + LogBatchEntry { + timestamp: ts.to_string(), + hostname: host.to_string(), + facility: None, + severity: severity.to_string(), + app_name: Some(app.to_string()), + process_id: None, + message: msg.to_string(), + raw: msg.to_string(), + source_ip: "127.0.0.1:514".to_string(), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + } +} + +#[test] +fn test_purge_by_tag_window_zero_days_is_noop() { + let (pool, _dir) = test_pool(); + let entries = vec![make_tagged( + "2020-01-01T00:00:00Z", + "h", + "info", + "adguard-allowed", + "old", + )]; + insert_logs_batch(&pool, &entries).unwrap(); + + let deleted = super::purge_by_tag_window(&pool, "adguard-allowed", 0, 0).unwrap(); + assert_eq!(deleted, 0, "max_days=0 must be a no-op"); + + let remaining = tail_logs(&pool, None, None, None, None, 10).unwrap(); + assert_eq!(remaining.len(), 1, "row must still be present"); +} + +#[test] +fn test_purge_by_tag_window_only_targets_named_tag() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[ + make_tagged( + "2020-01-01T00:00:00Z", + "h", + "info", + "adguard-allowed", + "old-allowed", + ), + make_tagged("2020-01-01T00:00:00Z", "h", "info", "nginx", "old-nginx"), + make_tagged("2020-01-01T00:00:00Z", "h", "info", "kernel", "old-kernel"), + ], + ) + .unwrap(); + // Backdate received_at past the 7-day window + update_received_at(&pool, "old-allowed", "2020-01-01T00:00:00Z"); + update_received_at(&pool, "old-nginx", "2020-01-01T00:00:00Z"); + update_received_at(&pool, "old-kernel", "2020-01-01T00:00:00Z"); + + let deleted = super::purge_by_tag_window(&pool, "adguard-allowed", 7, 0).unwrap(); + assert_eq!(deleted, 1, "only the adguard-allowed row must be deleted"); + + let remaining = tail_logs(&pool, None, None, None, None, 10).unwrap(); + let messages: Vec<&str> = remaining.iter().map(|r| r.message.as_str()).collect(); + assert!(messages.contains(&"old-nginx"), "nginx must survive"); + assert!(messages.contains(&"old-kernel"), "kernel must survive"); + assert!( + !messages.contains(&"old-allowed"), + "adguard-allowed must be gone" + ); +} + +#[test] +fn test_purge_by_tag_window_excludes_high_severity_rows() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[ + make_tagged( + "2020-01-01T00:00:00Z", + "h", + "info", + "adguard-allowed", + "info-old", + ), + make_tagged( + "2020-01-01T00:00:00Z", + "h", + "err", + "adguard-allowed", + "err-old", + ), + make_tagged( + "2020-01-01T00:00:00Z", + "h", + "crit", + "adguard-allowed", + "crit-old", + ), + ], + ) + .unwrap(); + update_received_at(&pool, "info-old", "2020-01-01T00:00:00Z"); + update_received_at(&pool, "err-old", "2020-01-01T00:00:00Z"); + update_received_at(&pool, "crit-old", "2020-01-01T00:00:00Z"); + + let deleted = super::purge_by_tag_window(&pool, "adguard-allowed", 7, 0).unwrap(); + assert_eq!(deleted, 1, "only the info row should be purged"); + + let remaining = tail_logs(&pool, None, None, None, None, 10).unwrap(); + let messages: Vec<&str> = remaining.iter().map(|r| r.message.as_str()).collect(); + assert!( + messages.contains(&"err-old"), + "err must be exempt from time-based purge" + ); + assert!( + messages.contains(&"crit-old"), + "crit must be exempt from time-based purge" + ); +} + +#[test] +fn test_purge_by_tag_window_respects_cutoff_boundary() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[ + make_tagged( + "2020-01-01T00:00:00Z", + "h", + "info", + "adguard-allowed", + "old", + ), + make_tagged( + "2020-01-01T00:00:00Z", + "h", + "info", + "adguard-allowed", + "fresh", + ), + ], + ) + .unwrap(); + // 'old' is past the 7-day window, 'fresh' is well inside it + update_received_at(&pool, "old", "2020-01-01T00:00:00Z"); + update_received_at( + &pool, + "fresh", + &chrono::Utc::now().format("%Y-%m-%dT%H:%M:%SZ").to_string(), + ); + + let deleted = super::purge_by_tag_window(&pool, "adguard-allowed", 7, 0).unwrap(); + assert_eq!(deleted, 1, "only the old row should be deleted"); + + let remaining = tail_logs(&pool, None, None, None, None, 10).unwrap(); + let messages: Vec<&str> = remaining.iter().map(|r| r.message.as_str()).collect(); + assert!(messages.contains(&"fresh"), "fresh row must survive"); +} + +/// A disk-space probe that always fails (simulates a statvfs/ENOENT error). +/// `get_storage_metrics_with_probe` maps the `Err` to `free_disk_bytes == None`. +#[derive(Clone)] +struct FailingDiskSpaceProbe; + +impl DiskSpaceProbe for FailingDiskSpaceProbe { + fn free_bytes(&self, _path: &Path) -> Result { + anyhow::bail!("simulated statvfs probe failure") + } +} + +/// syslog-mcp-w4hh (review bug #1 — FAIL-CLOSED): when the free-disk guardrail is +/// ENABLED but the disk-space probe fails (`free_disk_bytes == None`), the guardrail +/// must engage conservatively (treat unknown free space as the worst case) instead +/// of failing open. Previously `unwrap_or(u64::MAX)` made a probe failure look like +/// infinite free space, so the block NEVER engaged — defeating the safety behavior. +#[test] +fn probe_failure_engages_write_block_does_not_fail_open() { + let (pool, dir) = test_pool(); + let entries = vec![ + make_entry("2026-01-01T00:00:01Z", "host-a", "info", "older"), + make_entry("2026-01-01T00:00:02Z", "host-b", "info", "newer"), + ]; + insert_logs_batch(&pool, &entries).unwrap(); + + // DB-size limit disabled; only the free-disk guardrail is active and ENABLED. + let mut config = test_storage_config(dir.path().join("test.db")); + config.max_db_size_mb = 0; + config.recovery_db_size_mb = 0; + config.min_free_disk_mb = 512; + config.recovery_free_disk_mb = 768; + + // The probe fails on every call → free_disk_bytes is None. + let probe = FailingDiskSpaceProbe; + let outcome = enforce_storage_budget_with_probe(&pool, &config, &probe).unwrap(); + + assert!( + outcome.metrics.free_disk_bytes.is_none(), + "probe failure must surface as None, not a fabricated value" + ); + assert!( + outcome.write_blocked, + "FAIL-CLOSED: an enabled free-disk guardrail must block writes when the \ + probe fails (unknown == worst case), not fail open" + ); + // External pressure must never delete cortex's own data. + assert_eq!( + outcome.deleted_rows, 0, + "probe-failure pressure is external — must not delete own data" + ); + let rows = tail_logs(&pool, None, None, None, None, 10).unwrap(); + assert_eq!(rows.len(), 2, "both rows must survive — nothing deleted"); +} + +/// syslog-mcp-w4hh (review bug #1, unit-level): with the guardrail DISABLED +/// (`min_free_disk_mb == 0`), a probe failure must NOT engage the block — the +/// fail-closed behavior is scoped to the case where the operator asked for the +/// guardrail. This pins both halves of the trigger/write-block decision. +#[test] +fn probe_failure_with_guardrail_disabled_does_not_block() { + let metrics = StorageMetrics { + logical_db_size_bytes: 0, + physical_db_size_bytes: 0, + free_disk_bytes: None, // probe failed + }; + let mut config = StorageConfig::for_test(std::path::PathBuf::from("/tmp/x.db")); + config.min_free_disk_mb = 0; + config.recovery_free_disk_mb = 0; + + assert!( + !super::disk_free_below_trigger(&metrics, &config), + "disabled guardrail must not trigger on a failed probe" + ); + assert!( + !super::disk_pressure_write_blocked(&metrics, &config, false), + "disabled guardrail must not write-block on a failed probe" + ); + + // And with the guardrail ENABLED, the same None must engage both. + config.min_free_disk_mb = 512; + config.recovery_free_disk_mb = 768; + assert!( + super::disk_free_below_trigger(&metrics, &config), + "enabled guardrail must treat a failed probe as below the floor" + ); + assert!( + super::disk_pressure_write_blocked(&metrics, &config, false), + "enabled guardrail must write-block on a failed probe" + ); +} + +/// syslog-mcp-w4hh (review bug #2 — TIMESTAMP FORMAT): `received_at` is stored with +/// MILLISECOND precision (e.g. "...:27.680Z"). The err+ floor `window_start` must be +/// formatted the same way. A second-precision cutoff like "...:27Z" sorts AFTER +/// "...:27.680Z" lexicographically ('Z'=0x5A > '.'=0x2E), so a row that is genuinely +/// inside the window would be judged outside it and lose protection. This test pins +/// a recent err row whose received_at carries fractional seconds and verifies it is +/// protected by the floor (deleted_rows comes only from the deletable info row). +#[test] +fn err_floor_window_matches_fractional_second_received_at() { + let window_hours = 1i64; + let big_info = "info-junk-".repeat(120_000); + let big_err = "err-keep-".repeat(120_000); + + // Place the protected err row's received_at in the SAME WHOLE SECOND as the + // floor cutoff, with a `.999` fractional part. The cutoff is computed inside + // the function as `Utc::now() - window_hours`; we mirror that here. This is + // the only arrangement that exercises bug #2: a row a few seconds inside the + // window never reaches the fractional position (the date/second fields + // differ), so it passes under BOTH formats and proves nothing. + // + // Discrimination at the boundary second "HH:MM:SS": + // - Correct (Millis) cutoff "HH:MM:SS.mmmZ" with mmm < 999 → row ".999Z" >= + // cutoff → PROTECTED (this is the fix). + // - Buggy (second) cutoff "HH:MM:SSZ" → comparing "...SS.999Z" vs "...SSZ", + // the char after "SS" is '.'(0x2E) < 'Z'(0x5A), so row < cutoff → NOT + // protected → deleted. + // + // Determinism: the function captures its OWN `Utc::now()` between our two + // bracket reads (`now_before` … `enforce_storage_budget` … `now_after`). + // We only assert when BOTH brackets floor `(now - window)` to the SAME whole + // second — which, since the function's `now` lies between them and floor is + // monotonic, proves the function shared that boundary second. If a + // whole-second boundary was crossed mid-op (possible under heavy parallel + // load), the arrangement is invalid, so we retry with fresh data instead of + // flaking — no wall-clock-headroom guessing. + let floor_minus_window = |t: chrono::DateTime| { + (t - chrono::TimeDelta::hours(window_hours)) + .format("%Y-%m-%dT%H:%M:%S") + .to_string() + }; + + for _ in 0..200 { + let (pool, dir) = test_pool(); + insert_logs_batch( + &pool, + &[ + make_entry("2026-01-01T00:00:01Z", "host-a", "info", &big_info), + make_entry("2026-01-01T00:00:02Z", "host-a", "err", &big_err), + ], + ) + .unwrap(); + // Oldest, deletable info row. + update_received_at(&pool, &big_info, "2026-01-01T00:00:00Z"); + + let now_before = chrono::Utc::now(); + let boundary_second = floor_minus_window(now_before); + let recent_ts = format!("{boundary_second}.999Z"); + update_received_at(&pool, &big_err, &recent_ts); + + let mut config = test_storage_config(dir.path().join("test.db")); + config.max_db_size_mb = 2; + config.recovery_db_size_mb = 1; + config.min_free_disk_mb = 0; + config.recovery_free_disk_mb = 0; + config.cleanup_chunk_size = 1; + config.err_floor_window_hours = window_hours as u64; + config.err_floor_per_source_cap = 10_000; + + let outcome = enforce_storage_budget(&pool, &config).unwrap(); + let now_after = chrono::Utc::now(); + + // Boundary crossed during the op → the function's cutoff second may not + // match our fixture. Discard this attempt and rebuild. + if floor_minus_window(now_after) != boundary_second { + continue; + } + + let rows = tail_logs(&pool, None, None, None, None, 10).unwrap(); + let messages: Vec<&str> = rows.iter().map(|r| r.message.as_str()).collect(); + assert!( + messages.contains(&big_err.as_str()), + "recent err+ with fractional-second received_at must be protected by the floor" + ); + assert!( + !messages.contains(&big_info.as_str()), + "the deletable info row should have been trimmed" + ); + assert!( + outcome.write_blocked, + "still over cap after floor protected the err row → writes blocked" + ); + return; + } + + panic!( + "could not align the cutoff whole-second within 200 attempts (excessive scheduling jitter)" + ); +} + +/// syslog-mcp-w4hh (review bug #3 — heartbeat fallthrough): during a DB-SIZE breach, +/// when the OLDEST telemetry is logs but that chunk is fully err+-floor-protected +/// (delete returns 0), deletable heartbeats may still remain (newer than the +/// protected logs). The self-trim loop must fall through to trimming heartbeats +/// before declaring write_blocked, rather than blocking prematurely. +#[test] +fn self_trim_falls_through_to_heartbeats_when_logs_floor_protected() { + let (pool, dir) = test_pool(); + // A large, recent, floor-protected err log is the OLDEST telemetry. Heartbeats + // are NEWER, so oldest_telemetry_source picks logs first — and that chunk is + // fully protected (0 deleted). Deletable heartbeats remain. + let big_err = "err-protected-".repeat(120_000); + let now = chrono::Utc::now(); + let err_ts = + (now - chrono::TimeDelta::minutes(10)).to_rfc3339_opts(chrono::SecondsFormat::Millis, true); + let hb_ts = now.to_rfc3339_opts(chrono::SecondsFormat::Millis, true); + + insert_logs_batch(&pool, &[make_entry(&err_ts, "host-a", "err", &big_err)]).unwrap(); + update_received_at(&pool, &big_err, &err_ts); + // Insert several heartbeats (newer than the err row) — these are deletable. + for i in 0..5 { + insert_heartbeat(&pool, &format!("hb-host-{i}"), &hb_ts); + } + + let mut config = test_storage_config(dir.path().join("test.db")); + config.max_db_size_mb = 1; + config.recovery_db_size_mb = 1; + config.min_free_disk_mb = 0; + config.recovery_free_disk_mb = 0; + config.cleanup_chunk_size = 1; + config.err_floor_window_hours = 24; // protects the err row + config.err_floor_per_source_cap = 10_000; + + let hb_before: i64 = { + let conn = pool.get().unwrap(); + conn.query_row("SELECT COUNT(*) FROM host_heartbeats", [], |r| r.get(0)) + .unwrap() + }; + assert_eq!(hb_before, 5); + + let outcome = enforce_storage_budget(&pool, &config).unwrap(); + + // The protected err log must survive. + let rows = tail_logs(&pool, None, None, None, None, 10).unwrap(); + let messages: Vec<&str> = rows.iter().map(|r| r.message.as_str()).collect(); + assert!( + messages.contains(&big_err.as_str()), + "floor-protected err row must survive" + ); + // At least one heartbeat must have been trimmed (the fallthrough engaged) + // rather than the loop blocking immediately on the 0-deleted log chunk. + let hb_after: i64 = { + let conn = pool.get().unwrap(); + conn.query_row("SELECT COUNT(*) FROM host_heartbeats", [], |r| r.get(0)) + .unwrap() + }; + assert!( + hb_after < hb_before, + "deletable heartbeats must be trimmed via fallthrough (before: {hb_before}, after: {hb_after})" + ); + assert!( + outcome.deleted_rows > 0, + "fallthrough must report the heartbeat rows it trimmed" + ); +} + +fn insert_llm_invocation(pool: &DbPool, id: &str, started_at: &str) { + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO llm_invocations (id, started_at, caller_surface, action, provider, status) + VALUES (?1, ?2, 'cli', 'ai_assess', 'gemini-cli', 'completed')", + params![id, started_at], + ) + .unwrap(); +} + +fn count_llm_invocations(pool: &DbPool) -> i64 { + let conn = pool.get().unwrap(); + conn.query_row("SELECT COUNT(*) FROM llm_invocations", [], |row| row.get(0)) + .unwrap() +} + +#[test] +fn test_purge_old_llm_invocations_removes_old() { + let (pool, _dir) = test_pool(); + insert_llm_invocation(&pool, "old-1", "2020-01-01T00:00:00.000Z"); + insert_llm_invocation(&pool, "new-1", "2099-01-01T00:00:00.000Z"); + + let deleted = purge_old_llm_invocations(&pool, 90, 1000).unwrap(); + assert_eq!(deleted, 1, "should delete exactly the old row"); + assert_eq!(count_llm_invocations(&pool), 1); +} + +#[test] +fn test_purge_old_llm_invocations_zero_retention_noop() { + let (pool, _dir) = test_pool(); + insert_llm_invocation(&pool, "old-1", "2020-01-01T00:00:00.000Z"); + + let deleted = purge_old_llm_invocations(&pool, 0, 1000).unwrap(); + assert_eq!(deleted, 0, "retention_days=0 should be a no-op"); + assert_eq!(count_llm_invocations(&pool), 1); +} + +#[test] +fn test_purge_old_llm_invocations_chunked() { + let (pool, _dir) = test_pool(); + for i in 0..5 { + insert_llm_invocation(&pool, &format!("old-{i}"), "2020-01-01T00:00:00.000Z"); + } + insert_llm_invocation(&pool, "new-1", "2099-01-01T00:00:00.000Z"); + + // chunk_size smaller than the deletable set exercises the loop-until-empty path. + let deleted = purge_old_llm_invocations(&pool, 90, 2).unwrap(); + assert_eq!(deleted, 5); + assert_eq!(count_llm_invocations(&pool), 1); +} + +/// Regression test for the lexicographic-cutoff bug: a cutoff formatted +/// without fractional seconds (e.g. `...:50Z`) does not sort consistently +/// against `started_at` values that DO carry fractional seconds (SQLite +/// writes `started_at` via `strftime('%Y-%m-%dT%H:%M:%fZ','now')`). Seed two +/// rows within the same wall-clock second that straddle a +/// millisecond-precision cutoff, and confirm the purge honors true +/// chronological order rather than string comparison. +#[test] +fn test_purge_old_llm_invocations_millisecond_precision_ordering() { + let (pool, _dir) = test_pool(); + + // Two rows within the same wall-clock second, one just after and one + // just before a cutoff that sits between them. + insert_llm_invocation(&pool, "just-before-cutoff", "2020-01-01T00:00:50.000001Z"); + insert_llm_invocation(&pool, "just-after-cutoff", "2020-01-01T00:00:50.999999Z"); + + // Directly exercise the cutoff-comparison SQL with a hand-built cutoff + // straddling the two rows, to pin down true chronological behavior + // independent of `Utc::now()` in the production function. + let cutoff = "2020-01-01T00:00:50.500000Z"; + let conn = pool.get().unwrap(); + let deleted = conn + .execute( + "DELETE FROM llm_invocations WHERE id IN ( + SELECT id FROM llm_invocations + WHERE started_at < ?1 + LIMIT 1000 + )", + params![cutoff], + ) + .unwrap(); + drop(conn); + + assert_eq!( + deleted, 1, + "exactly the row before the cutoff must be deleted" + ); + + let conn = pool.get().unwrap(); + let remaining_id: String = conn + .query_row("SELECT id FROM llm_invocations", [], |row| row.get(0)) + .unwrap(); + assert_eq!( + remaining_id, "just-after-cutoff", + "the row chronologically after the cutoff must survive" + ); +} + +/// End-to-end regression test through the real production function: the +/// cutoff it formats must retain millisecond precision so that a row +/// inserted a fraction of a second before `Utc::now() - retention_days` is +/// correctly purged instead of surviving due to a lexicographic-comparison +/// mismatch against a coarser cutoff string. +#[test] +fn test_purge_old_llm_invocations_cutoff_has_fractional_seconds() { + let (pool, _dir) = test_pool(); + + // A row just past the retention boundary, 1s before now-90days (a 1ms + // margin would be too tight here: both the stored timestamp and the + // cutoff are formatted at millisecond precision, so rounding/truncation + // could collapse a 1ms delta to zero and make this test flaky). + let boundary = + (chrono::Utc::now() - chrono::TimeDelta::days(90) - chrono::TimeDelta::seconds(1)) + .to_rfc3339_opts(chrono::SecondsFormat::Millis, true); + insert_llm_invocation(&pool, "at-boundary", &boundary); + + let deleted = purge_old_llm_invocations(&pool, 90, 1000).unwrap(); + assert_eq!( + deleted, 1, + "row just past the retention boundary must be purged; a cutoff \ + missing fractional-second precision could misorder it" + ); + assert_eq!(count_llm_invocations(&pool), 0); +} diff --git a/crates/shared/cortex/storage-sqlite/src/mcp_events.rs b/crates/shared/cortex/storage-sqlite/src/mcp_events.rs new file mode 100644 index 00000000..525db7c6 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/mcp_events.rs @@ -0,0 +1,281 @@ +//! `ai_mcp_events` insert + list query layer. Table/columns are defined in +//! migration 39 (`src/db/pool.rs`). Extraction happens in +//! `crate::inputs`; this module only persists and reads back +//! already-extracted events. +//! +//! Unlike `ai_skill_events` (one `ChunkSkillSource` per log row, at most one +//! event emitted), a single transcript line can carry multiple `tool_use` +//! blocks, and a call's paired result frequently lands on a LATER log row +//! (or a different chunk/transaction entirely — Claude's `tool_result` +//! rows are separate transcript lines from their `tool_use` call). So +//! `insert_mcp_events_in_tx` intentionally does not try to join call/result +//! rows itself: each extracted event (call or result) is inserted as its +//! own row keyed by `(ai_tool, ai_session_id, call_id, event_kind)`, and +//! `mcp_tool`/`mcp_server`/`tool_name` on a bare result row are populated +//! via a best-effort backfill-join against the earlier call row in the same +//! statement batch (see `resolve_result_tool_name_in_tx`) — a result event +//! extracted with an empty `tool_name` (see +//! `scanner::mcp_events::extract_claude_mcp_events`) looks up its sibling +//! call row by `(ai_tool, ai_session_id, call_id)` and copies +//! `tool_name`/`mcp_server`/`mcp_tool` forward so incident grouping (keyed +//! on `mcp_server`/`mcp_tool`) still works for result-only anchors. + +use anyhow::Result; +use rusqlite::{OptionalExtension, Transaction, params}; +use serde::{Deserialize, Serialize}; + +use crate::inputs::ExtractedMcpEvent; +pub use cortex_domain::McpEventEntry as AiMcpEventEntry; + +use super::pool::DbPool; + +#[derive(Debug, Clone)] +pub struct McpEventInsert { + pub log_id: i64, + pub ai_tool: String, + pub ai_project: Option, + pub ai_session_id: Option, + pub hostname: String, + pub timestamp: String, + pub event: ExtractedMcpEvent, +} + +/// Resolved `(tool_name, mcp_server, mcp_tool)` for a result event copied +/// forward from its paired call row. +type ResolvedCallIdentity = (String, Option, Option); + +/// Look up `tool_name`/`mcp_server`/`mcp_tool` from the paired call row for +/// a result event that didn't carry its own tool name (Claude's +/// `tool_result` shape never repeats the tool name — only Codex's +/// `function_call_output` could in principle, and it doesn't either). Falls +/// back to empty/`NULL` when no matching call row has been inserted yet +/// (e.g. backfill processing results before their call, or the call row +/// fell outside the backfill's scan window) — the row is still inserted so +/// incident detection on `is_error`/`status` isn't lost, it's just +/// unclassified until a later re-ingest or backfill pass fills it in. +fn resolve_result_tool_name_in_tx( + tx: &Transaction<'_>, + ai_tool: &str, + ai_session_id: Option<&str>, + call_id: &str, +) -> Result> { + // Filters on COALESCE(ai_session_id, '') rather than `ai_session_id IS + // ?2` so this lookup can use idx_ai_mcp_events_dedupe (built on the + // same COALESCE expression) instead of falling back to the wider + // idx_ai_mcp_events_session_time index. prepare_cached avoids + // re-parsing this statement on every result-row insert (roughly half + // of all ai_mcp_events rows, since calls and results are paired 1:1). + let row: Option = tx + .prepare_cached( + "SELECT tool_name, mcp_server, mcp_tool FROM ai_mcp_events + WHERE ai_tool = ?1 + AND COALESCE(ai_session_id, '') = COALESCE(?2, '') + AND call_id = ?3 AND event_kind = 'call' + LIMIT 1", + )? + .query_row(params![ai_tool, ai_session_id, call_id], |row| { + Ok((row.get(0)?, row.get(1)?, row.get(2)?)) + }) + .optional()?; + Ok(row) +} + +/// Insert `events` inside an existing transaction with `INSERT OR IGNORE` +/// (idempotent on the `UNIQUE(ai_tool, ai_session_id, call_id, event_kind)` +/// constraint). Returns the number of rows actually inserted (excludes +/// ignored duplicates). +pub(crate) fn insert_mcp_events_in_tx( + tx: &Transaction<'_>, + events: &[McpEventInsert], +) -> Result { + if events.is_empty() { + return Ok(0); + } + let mut inserted = 0usize; + for item in events { + let (tool_name, mcp_server, mcp_tool) = if item.event.event_kind + == crate::inputs::McpEventKind::Result + && item.event.tool_name.is_empty() + { + resolve_result_tool_name_in_tx( + tx, + &item.ai_tool, + item.ai_session_id.as_deref(), + &item.event.call_id, + )? + .unwrap_or((String::new(), None, None)) + } else { + ( + item.event.tool_name.clone(), + item.event.mcp_server.clone(), + item.event.mcp_tool.clone(), + ) + }; + let changed = tx.prepare_cached( + "INSERT OR IGNORE INTO ai_mcp_events ( + call_log_id, result_log_id, ai_tool, ai_project, ai_session_id, hostname, + timestamp, turn_id, call_id, tool_name, mcp_server, mcp_tool, event_kind, + status, duration_ms, is_error, arguments_json, output_preview, error_text + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, ?11, ?12, ?13, ?14, ?15, ?16, ?17, ?18, ?19)", + )?.execute(params![ + (item.event.event_kind == crate::inputs::McpEventKind::Call) + .then_some(item.log_id), + (item.event.event_kind == crate::inputs::McpEventKind::Result) + .then_some(item.log_id), + item.ai_tool, + item.ai_project, + item.ai_session_id, + item.hostname, + item.timestamp, + item.event.turn_id, + item.event.call_id, + tool_name, + mcp_server, + mcp_tool, + item.event.event_kind.as_str(), + item.event.status, + Option::::None, + item.event.is_error.map(i64::from), + item.event.arguments_json, + item.event.output_preview, + item.event.error_text, + ])?; + inserted += changed; + } + Ok(inserted) +} + +/// Pool-acquiring wrapper for callers outside an existing transaction (e.g. +/// the backfill service, which owns its own chunked transaction boundary). +pub fn insert_mcp_events(pool: &DbPool, events: &[McpEventInsert]) -> Result { + let mut conn = pool.get()?; + let _write_guard = crate::write_lock(); + let tx = conn.transaction()?; + let inserted = insert_mcp_events_in_tx(&tx, events)?; + tx.commit()?; + Ok(inserted) +} + +#[derive(Debug, Clone, Default)] +pub struct AiMcpEventParams { + pub tool_name: Option, + pub mcp_server: Option, + pub mcp_tool: Option, + pub ai_tool: Option, + pub ai_project: Option, + pub ai_session_id: Option, + pub hostname: Option, + pub is_error: Option, + pub from: Option, + pub to: Option, + pub limit: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ListMcpEventsResult { + pub total: usize, + pub truncated: bool, + pub events: Vec, +} + +const DEFAULT_LIMIT: u32 = 50; +const MAX_LIMIT: u32 = 500; + +fn map_mcp_event_row(row: &rusqlite::Row) -> rusqlite::Result { + Ok(AiMcpEventEntry { + id: row.get(0)?, + call_log_id: row.get(1)?, + result_log_id: row.get(2)?, + ai_tool: row.get(3)?, + ai_project: row.get(4)?, + ai_session_id: row.get(5)?, + hostname: row.get(6)?, + timestamp: row.get(7)?, + turn_id: row.get(8)?, + call_id: row.get(9)?, + tool_name: row.get(10)?, + mcp_server: row.get(11)?, + mcp_tool: row.get(12)?, + event_kind: row.get(13)?, + status: row.get(14)?, + duration_ms: row.get(15)?, + is_error: row.get::<_, Option>(16)?.map(|v| v != 0), + arguments_json: row.get(17)?, + output_preview: row.get(18)?, + error_text: row.get(19)?, + }) +} + +const MCP_EVENT_COLUMNS: &str = "id, call_log_id, result_log_id, ai_tool, ai_project, \ + ai_session_id, hostname, timestamp, turn_id, call_id, tool_name, mcp_server, mcp_tool, \ + event_kind, status, duration_ms, is_error, arguments_json, output_preview, error_text"; + +/// List `ai_mcp_events` rows newest-first, applying every non-`None` filter +/// in `params` as an `AND`-ed equality/range clause. `limit` is clamped to +/// `[1, 500]`; `truncated` is `true` when more rows matched than were +/// returned (probed via `LIMIT + 1`, mirroring `list_skill_events`). +pub fn list_mcp_events(pool: &DbPool, params: &AiMcpEventParams) -> Result { + let conn = pool.get()?; + let limit = params.limit.unwrap_or(DEFAULT_LIMIT).clamp(1, MAX_LIMIT) as usize; + + let mut sql = format!("SELECT {MCP_EVENT_COLUMNS} FROM ai_mcp_events WHERE 1 = 1"); + let mut bindings: Vec = Vec::new(); + let mut idx = 1usize; + + macro_rules! bind_eq { + ($column:literal, $value:expr) => { + if let Some(value) = $value { + sql.push_str(&format!(" AND {} = ?{idx}", $column)); + bindings.push(rusqlite::types::Value::Text(value.clone())); + idx += 1; + } + }; + } + bind_eq!("tool_name", ¶ms.tool_name); + bind_eq!("mcp_server", ¶ms.mcp_server); + bind_eq!("mcp_tool", ¶ms.mcp_tool); + bind_eq!("ai_tool", ¶ms.ai_tool); + bind_eq!("ai_project", ¶ms.ai_project); + bind_eq!("ai_session_id", ¶ms.ai_session_id); + bind_eq!("hostname", ¶ms.hostname); + if let Some(is_error) = params.is_error { + sql.push_str(&format!(" AND is_error = ?{idx}")); + bindings.push(rusqlite::types::Value::Integer(i64::from(is_error))); + idx += 1; + } + if let Some(from) = ¶ms.from { + sql.push_str(&format!(" AND timestamp >= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(from.clone())); + idx += 1; + } + if let Some(to) = ¶ms.to { + sql.push_str(&format!(" AND timestamp <= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(to.clone())); + idx += 1; + } + let _ = idx; + sql.push_str(&format!( + " ORDER BY timestamp DESC, id DESC LIMIT {}", + limit + 1 + )); + + let mut stmt = conn.prepare(&sql)?; + let mut rows = stmt + .query_map( + rusqlite::params_from_iter(bindings.iter()), + map_mcp_event_row, + )? + .collect::>>()?; + + let truncated = rows.len() > limit; + rows.truncate(limit); + Ok(ListMcpEventsResult { + total: rows.len(), + truncated, + events: rows, + }) +} + +#[cfg(test)] +#[path = "mcp_events_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/mcp_events_tests.rs b/crates/shared/cortex/storage-sqlite/src/mcp_events_tests.rs new file mode 100644 index 00000000..eebde89c --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/mcp_events_tests.rs @@ -0,0 +1,255 @@ +use super::*; +use crate::config::StorageConfig; +use crate::inputs::McpEventKind; +use crate::pool::init_pool; + +fn test_pool() -> (crate::DbPool, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let pool = init_pool(&StorageConfig::for_test(db_path)).unwrap(); + (pool, dir) +} + +fn insert_log_row(pool: &crate::DbPool, hostname: &str, timestamp: &str) -> i64 { + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO logs (timestamp, hostname, severity, message, raw, source_ip) + VALUES (?1, ?2, 'info', 'msg', 'raw', 'transcript://claude_project')", + rusqlite::params![timestamp, hostname], + ) + .unwrap(); + conn.last_insert_rowid() +} + +fn call_event( + call_id: &str, + tool_name: &str, + mcp_server: Option<&str>, + mcp_tool: Option<&str>, +) -> ExtractedMcpEvent { + ExtractedMcpEvent { + call_id: call_id.to_string(), + tool_name: tool_name.to_string(), + mcp_server: mcp_server.map(str::to_string), + mcp_tool: mcp_tool.map(str::to_string), + event_kind: McpEventKind::Call, + turn_id: None, + status: None, + is_error: None, + arguments_json: Some("{}".to_string()), + output_preview: None, + error_text: None, + } +} + +fn result_event(call_id: &str, is_error: bool) -> ExtractedMcpEvent { + ExtractedMcpEvent { + call_id: call_id.to_string(), + tool_name: String::new(), + mcp_server: None, + mcp_tool: None, + event_kind: McpEventKind::Result, + turn_id: None, + status: Some(if is_error { "error" } else { "ok" }.to_string()), + is_error: Some(is_error), + arguments_json: None, + output_preview: (!is_error).then(|| "ok output".to_string()), + error_text: is_error.then(|| "boom".to_string()), + } +} + +#[test] +fn insert_and_list_round_trips_a_call_event() { + let (pool, _dir) = test_pool(); + let log_id = insert_log_row(&pool, "devhost", "2026-06-01T00:00:00.000Z"); + let insert = McpEventInsert { + log_id, + ai_tool: "claude".to_string(), + ai_project: Some("cortex".to_string()), + ai_session_id: Some("sess-1".to_string()), + hostname: "devhost".to_string(), + timestamp: "2026-06-01T00:00:00.000Z".to_string(), + event: call_event( + "toolu_1", + "mcp__labby__search", + Some("labby"), + Some("search"), + ), + }; + let inserted = insert_mcp_events(&pool, &[insert]).unwrap(); + assert_eq!(inserted, 1); + + let result = list_mcp_events(&pool, &AiMcpEventParams::default()).unwrap(); + assert_eq!(result.total, 1); + assert_eq!(result.events[0].call_id, "toolu_1"); + assert_eq!(result.events[0].tool_name, "mcp__labby__search"); + assert_eq!(result.events[0].mcp_server.as_deref(), Some("labby")); + assert_eq!(result.events[0].mcp_tool.as_deref(), Some("search")); + assert_eq!(result.events[0].event_kind, "call"); + assert_eq!(result.events[0].call_log_id, Some(log_id)); + assert_eq!(result.events[0].result_log_id, None); +} + +#[test] +fn insert_or_ignore_is_idempotent_on_duplicate() { + let (pool, _dir) = test_pool(); + let log_id = insert_log_row(&pool, "devhost", "2026-06-01T00:00:00.000Z"); + let insert = McpEventInsert { + log_id, + ai_tool: "claude".to_string(), + ai_project: None, + ai_session_id: None, + hostname: "devhost".to_string(), + timestamp: "2026-06-01T00:00:00.000Z".to_string(), + event: call_event("toolu_dup", "Bash", None, None), + }; + assert_eq!( + insert_mcp_events(&pool, std::slice::from_ref(&insert)).unwrap(), + 1 + ); + assert_eq!(insert_mcp_events(&pool, &[insert]).unwrap(), 0); + + let result = list_mcp_events(&pool, &AiMcpEventParams::default()).unwrap(); + assert_eq!(result.total, 1); +} + +#[test] +fn result_event_resolves_tool_name_from_paired_call_row() { + let (pool, _dir) = test_pool(); + let call_log_id = insert_log_row(&pool, "devhost", "2026-06-01T00:00:00.000Z"); + let result_log_id = insert_log_row(&pool, "devhost", "2026-06-01T00:00:01.000Z"); + + insert_mcp_events( + &pool, + &[ + McpEventInsert { + log_id: call_log_id, + ai_tool: "claude".to_string(), + ai_project: Some("cortex".to_string()), + ai_session_id: Some("sess-1".to_string()), + hostname: "devhost".to_string(), + timestamp: "2026-06-01T00:00:00.000Z".to_string(), + event: call_event( + "toolu_paired", + "mcp__gh__search", + Some("gh"), + Some("search"), + ), + }, + McpEventInsert { + log_id: result_log_id, + ai_tool: "claude".to_string(), + ai_project: Some("cortex".to_string()), + ai_session_id: Some("sess-1".to_string()), + hostname: "devhost".to_string(), + timestamp: "2026-06-01T00:00:01.000Z".to_string(), + event: result_event("toolu_paired", false), + }, + ], + ) + .unwrap(); + + let result = list_mcp_events( + &pool, + &AiMcpEventParams { + mcp_server: Some("gh".to_string()), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(result.total, 2); + let result_row = result + .events + .iter() + .find(|e| e.event_kind == "result") + .expect("result row present"); + assert_eq!(result_row.tool_name, "mcp__gh__search"); + assert_eq!(result_row.mcp_server.as_deref(), Some("gh")); + assert_eq!(result_row.mcp_tool.as_deref(), Some("search")); + assert_eq!(result_row.result_log_id, Some(result_log_id)); + assert_eq!(result_row.is_error, Some(false)); +} + +#[test] +fn result_event_without_paired_call_still_inserts_unclassified() { + let (pool, _dir) = test_pool(); + let log_id = insert_log_row(&pool, "devhost", "2026-06-01T00:00:00.000Z"); + let insert = McpEventInsert { + log_id, + ai_tool: "claude".to_string(), + ai_project: None, + ai_session_id: None, + hostname: "devhost".to_string(), + timestamp: "2026-06-01T00:00:00.000Z".to_string(), + event: result_event("toolu_orphan", true), + }; + let inserted = insert_mcp_events(&pool, &[insert]).unwrap(); + assert_eq!(inserted, 1); + let result = list_mcp_events(&pool, &AiMcpEventParams::default()).unwrap(); + assert_eq!(result.events[0].tool_name, ""); + assert_eq!(result.events[0].mcp_server, None); + assert_eq!(result.events[0].is_error, Some(true)); +} + +#[test] +fn list_filters_by_mcp_server_project_and_is_error() { + let (pool, _dir) = test_pool(); + let log_id_a = insert_log_row(&pool, "devhost", "2026-06-01T00:00:00.000Z"); + let log_id_b = insert_log_row(&pool, "nashost", "2026-06-01T01:00:00.000Z"); + insert_mcp_events( + &pool, + &[ + McpEventInsert { + log_id: log_id_a, + ai_tool: "claude".to_string(), + ai_project: Some("cortex".to_string()), + ai_session_id: Some("sess-a".to_string()), + hostname: "devhost".to_string(), + timestamp: "2026-06-01T00:00:00.000Z".to_string(), + event: call_event( + "toolu_a", + "mcp__labby__search", + Some("labby"), + Some("search"), + ), + }, + McpEventInsert { + log_id: log_id_b, + ai_tool: "codex".to_string(), + ai_project: Some("axon".to_string()), + ai_session_id: Some("sess-b".to_string()), + hostname: "nashost".to_string(), + timestamp: "2026-06-01T01:00:00.000Z".to_string(), + event: { + let mut e = + call_event("toolu_b", "mcp__gh__search", Some("gh"), Some("search")); + e.is_error = Some(true); + e + }, + }, + ], + ) + .unwrap(); + + let result = list_mcp_events( + &pool, + &AiMcpEventParams { + mcp_server: Some("labby".to_string()), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(result.total, 1); + assert_eq!(result.events[0].call_id, "toolu_a"); + + let result = list_mcp_events( + &pool, + &AiMcpEventParams { + is_error: Some(true), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(result.total, 1); + assert_eq!(result.events[0].call_id, "toolu_b"); +} diff --git a/crates/shared/cortex/storage-sqlite/src/mcp_incident_evidence.rs b/crates/shared/cortex/storage-sqlite/src/mcp_incident_evidence.rs new file mode 100644 index 00000000..2a75c706 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/mcp_incident_evidence.rs @@ -0,0 +1,353 @@ +//! Investigation evidence-bundle layer for MCP incidents. Expands an +//! `McpIncident` (grouped/scored in `src/db/mcp_incidents.rs`) into a +//! bounded, truncation-flagged evidence bundle: the underlying MCP events, +//! the transcript rows that triggered anchor signals, transcript context +//! before/after, and nearby non-AI logs split into error/user-correction +//! subsets. Mirrors `investigate_ai_skill_incidents` in +//! `src/db/skill_incident_evidence.rs` but keyed on MCP tool-call usage +//! instead of skill-attribution events. + +use anyhow::Result; +use serde::{Deserialize, Serialize}; + +use cortex_domain::mcp_signal_detectors::detect_user_correction_after_tool_call; + +use super::mcp_events::AiMcpEventEntry; +use super::mcp_incidents::{AiMcpIncidentParams, McpIncident, search_ai_mcp_incidents}; +use super::models::LogEntry; +use super::pool::DbPool; +use super::queries::map_row; + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct AiMcpInvestigateParams { + pub incident_id: Option, + pub mcp_server: Option, + pub mcp_tool: Option, + pub tool_name: Option, + pub ai_tool: Option, + pub ai_project: Option, + pub since: Option, + pub until: Option, + /// Max incidents to investigate. Default 3, clamp 1..=10. + pub limit: Option, + /// Incident grouping window minutes. Default 10, clamp 1..=120. + pub window_minutes: Option, + /// Correlation window minutes around incident. Default 5, clamp 1..=120. + pub correlation_window_minutes: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct McpIncidentEvidence { + pub incident: McpIncident, + /// The `ai_mcp_events` rows in this group, capped at 25. + pub mcp_events: Vec, + pub mcp_events_truncated: bool, + /// Transcript rows that triggered an anchor signal, capped at 50. + pub signal_anchors: Vec, + pub signal_anchors_truncated: bool, + /// Same-session transcript entries before the first event, capped 20. + pub transcript_before: Vec, + pub transcript_before_truncated: bool, + /// Same-session transcript entries after the last event, capped 20. + pub transcript_after: Vec, + pub transcript_after_truncated: bool, + /// Subset of nearby_logs matching user-correction phrases, capped 25. + pub nearby_user_corrections: Vec, + pub nearby_user_corrections_truncated: bool, + /// Non-AI syslog/Docker logs in the correlation window, capped 50. + pub nearby_logs: Vec, + pub nearby_logs_truncated: bool, + /// Subset of nearby_logs with severity warning or above, capped 25. + pub nearby_errors: Vec, + pub nearby_errors_truncated: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiMcpInvestigateResult { + pub evidence: Vec, + pub total_incidents: usize, + pub truncated: bool, +} + +fn map_mcp_event_row(row: &rusqlite::Row) -> rusqlite::Result { + Ok(AiMcpEventEntry { + id: row.get(0)?, + call_log_id: row.get(1)?, + result_log_id: row.get(2)?, + ai_tool: row.get(3)?, + ai_project: row.get(4)?, + ai_session_id: row.get(5)?, + hostname: row.get(6)?, + timestamp: row.get(7)?, + turn_id: row.get(8)?, + call_id: row.get(9)?, + tool_name: row.get(10)?, + mcp_server: row.get(11)?, + mcp_tool: row.get(12)?, + event_kind: row.get(13)?, + status: row.get(14)?, + duration_ms: row.get(15)?, + is_error: row.get::<_, Option>(16)?.map(|v| v != 0), + arguments_json: row.get(17)?, + output_preview: row.get(18)?, + error_text: row.get(19)?, + }) +} + +const MCP_EVENT_COLUMNS: &str = "id, call_log_id, result_log_id, ai_tool, ai_project, \ + ai_session_id, hostname, timestamp, turn_id, call_id, tool_name, mcp_server, mcp_tool, \ + event_kind, status, duration_ms, is_error, arguments_json, output_preview, error_text"; + +pub fn investigate_ai_mcp_incidents( + pool: &DbPool, + params: &AiMcpInvestigateParams, +) -> Result { + const MCP_EVENTS_CAP: usize = 25; + const SIGNAL_ANCHORS_CAP: usize = 50; + const TRANSCRIPT_CAP: usize = 20; + const NEARBY_CAP: usize = 50; + const NEARBY_SUBSET_CAP: usize = 25; + + let limit = params.limit.unwrap_or(3).clamp(1, 10) as usize; + let corr_mins = i64::from(params.correlation_window_minutes.unwrap_or(5).clamp(1, 120)); + + // `incident_id` is passed straight through to `AiMcpIncidentParams`, + // which filters the full computed incident set (bounded only by + // `MCP_INCIDENT_CANDIDATE_CAP` events, not an incident-count cap) before + // its own priority-ranked truncation. This guarantees an exact + // incident_id lookup finds its target regardless of priority rank — + // routing it through a fixed-size top-N candidate window (as a prior + // version of this code did) could silently miss incidents ranked below + // that window. + let incident_result = search_ai_mcp_incidents( + pool, + &AiMcpIncidentParams { + mcp_server: params.mcp_server.clone(), + mcp_tool: params.mcp_tool.clone(), + tool_name: params.tool_name.clone(), + ai_tool: params.ai_tool.clone(), + ai_project: params.ai_project.clone(), + ai_session_id: None, + hostname: None, + since: params.since.clone(), + until: params.until.clone(), + incident_id: params.incident_id.clone(), + limit: Some(limit as u32), + window_minutes: params.window_minutes, + signals: Vec::new(), + min_score: None, + }, + )?; + let total_incidents = incident_result.total_incidents; + let truncated = incident_result.truncated; + let mut incidents = incident_result.incidents; + incidents.truncate(limit); + + let conn = pool.get()?; + let mut evidence = Vec::with_capacity(incidents.len()); + + for incident in incidents { + // ── MCP events for this group ──────────────────────────────────── + let (mcp_events, mcp_events_truncated) = if incident.mcp_event_ids.is_empty() { + (Vec::new(), false) + } else { + let placeholders: Vec = (1..=incident.mcp_event_ids.len()) + .map(|i| format!("?{i}")) + .collect(); + let sql = format!( + "SELECT {MCP_EVENT_COLUMNS} FROM ai_mcp_events WHERE id IN ({}) ORDER BY timestamp ASC", + placeholders.join(",") + ); + let mut stmt = conn.prepare(&sql)?; + let rows: Vec = stmt + .query_map( + rusqlite::params_from_iter( + incident + .mcp_event_ids + .iter() + .map(|id| rusqlite::types::Value::Integer(*id)), + ), + map_mcp_event_row, + )? + .collect::>>()?; + let truncated = rows.len() > MCP_EVENTS_CAP; + let mut out = rows; + out.truncate(MCP_EVENTS_CAP); + (out, truncated) + }; + + // ── Signal anchor log rows ────────────────────────────────────── + let (signal_anchors, signal_anchors_truncated) = if incident.anchor_log_ids.is_empty() { + (Vec::new(), false) + } else { + let placeholders: Vec = (1..=incident.anchor_log_ids.len()) + .map(|i| format!("?{i}")) + .collect(); + let sql = format!( + "SELECT id, timestamp, hostname, facility, severity, app_name, + process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs WHERE id IN ({}) ORDER BY timestamp ASC", + placeholders.join(",") + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt + .query_map( + rusqlite::params_from_iter( + incident + .anchor_log_ids + .iter() + .map(|id| rusqlite::types::Value::Integer(*id)), + ), + map_row, + )? + .collect::>>()?; + let truncated = rows.len() > SIGNAL_ANCHORS_CAP; + let mut out = rows; + out.truncate(SIGNAL_ANCHORS_CAP); + (out, truncated) + }; + + // ── Transcript before/after ────────────────────────────────────── + let (transcript_before, transcript_before_truncated) = { + let mut stmt = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, app_name, + process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs + WHERE ai_session_id = ?1 AND ai_project = ?2 AND ai_tool = ?3 + AND timestamp < ?4 + ORDER BY timestamp DESC + LIMIT 21", + )?; + let rows = stmt + .query_map( + rusqlite::params![ + &incident.session_id, + &incident.project, + &incident.tool, + &incident.first_seen, + ], + map_row, + )? + .collect::>>()?; + let truncated = rows.len() > TRANSCRIPT_CAP; + let mut out = rows; + out.truncate(TRANSCRIPT_CAP); + out.reverse(); + (out, truncated) + }; + + let (transcript_after, transcript_after_truncated) = { + let mut stmt = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, app_name, + process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs + WHERE ai_session_id = ?1 AND ai_project = ?2 AND ai_tool = ?3 + AND timestamp > ?4 + ORDER BY timestamp ASC + LIMIT 21", + )?; + let rows = stmt + .query_map( + rusqlite::params![ + &incident.session_id, + &incident.project, + &incident.tool, + &incident.last_seen, + ], + map_row, + )? + .collect::>>()?; + let truncated = rows.len() > TRANSCRIPT_CAP; + let mut out = rows; + out.truncate(TRANSCRIPT_CAP); + (out, truncated) + }; + + // ── Nearby non-AI logs in the correlation window ──────────────── + let (nearby_logs, nearby_logs_truncated) = { + let win_from = chrono::DateTime::parse_from_rfc3339(&incident.first_seen) + .map(|dt| { + (dt.with_timezone(&chrono::Utc) - chrono::Duration::minutes(corr_mins)) + .format("%Y-%m-%dT%H:%M:%S%.3fZ") + .to_string() + }) + .unwrap_or_else(|_| incident.first_seen.clone()); + let win_to = chrono::DateTime::parse_from_rfc3339(&incident.last_seen) + .map(|dt| { + (dt.with_timezone(&chrono::Utc) + chrono::Duration::minutes(corr_mins)) + .format("%Y-%m-%dT%H:%M:%S%.3fZ") + .to_string() + }) + .unwrap_or_else(|_| incident.last_seen.clone()); + + let mut stmt = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, app_name, + process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs + WHERE timestamp >= ?1 AND timestamp <= ?2 AND hostname = ?3 + ORDER BY timestamp ASC + LIMIT 51", + )?; + let rows = stmt + .query_map( + rusqlite::params![win_from, win_to, &incident.hostname], + map_row, + )? + .collect::>>()?; + let truncated = rows.len() > NEARBY_CAP; + let mut out = rows; + out.truncate(NEARBY_CAP); + (out, truncated) + }; + + // ── Derived subsets: user corrections, errors ──────────────────── + let mut nearby_user_corrections: Vec = nearby_logs + .iter() + .filter(|e| detect_user_correction_after_tool_call(&e.message)) + .cloned() + .collect(); + let nearby_user_corrections_truncated = nearby_user_corrections.len() > NEARBY_SUBSET_CAP; + nearby_user_corrections.truncate(NEARBY_SUBSET_CAP); + + let error_sevs = ["emergency", "alert", "critical", "error", "warning"]; + let mut nearby_errors: Vec = nearby_logs + .iter() + .filter(|e| error_sevs.contains(&e.severity.as_str())) + .cloned() + .collect(); + let nearby_errors_truncated = nearby_errors.len() > NEARBY_SUBSET_CAP; + nearby_errors.truncate(NEARBY_SUBSET_CAP); + + evidence.push(McpIncidentEvidence { + incident, + mcp_events, + mcp_events_truncated, + signal_anchors, + signal_anchors_truncated, + transcript_before, + transcript_before_truncated, + transcript_after, + transcript_after_truncated, + nearby_user_corrections, + nearby_user_corrections_truncated, + nearby_logs, + nearby_logs_truncated, + nearby_errors, + nearby_errors_truncated, + }); + } + + Ok(AiMcpInvestigateResult { + evidence, + total_incidents, + truncated, + }) +} + +#[cfg(test)] +#[path = "mcp_incident_evidence_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/mcp_incident_evidence_tests.rs b/crates/shared/cortex/storage-sqlite/src/mcp_incident_evidence_tests.rs new file mode 100644 index 00000000..127e489c --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/mcp_incident_evidence_tests.rs @@ -0,0 +1,442 @@ +use super::*; +use crate::config::StorageConfig; +use crate::mcp_incidents::AiMcpIncidentParams; +use crate::pool::init_pool; +use crate::{DbPool, LogBatchEntry, insert_logs_batch}; + +fn test_pool() -> (DbPool, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let pool = init_pool(&StorageConfig::for_test(db_path)).unwrap(); + (pool, dir) +} + +fn make_ai_entry( + ts: &str, + host: &str, + tool: &str, + project: &str, + session_id: &str, + message: &str, +) -> LogBatchEntry { + LogBatchEntry { + timestamp: ts.to_string(), + hostname: host.to_string(), + facility: Some("local0".to_string()), + severity: "info".to_string(), + app_name: Some("ai-transcript".to_string()), + process_id: None, + message: message.to_string(), + raw: message.to_string(), + source_ip: "127.0.0.1:514".to_string(), + docker_checkpoint: None, + ai_tool: Some(tool.to_string()), + ai_project: Some(project.to_string()), + ai_session_id: Some(session_id.to_string()), + ai_transcript_path: Some(format!("{project}/{session_id}.jsonl")), + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + } +} + +#[allow(clippy::too_many_arguments)] +fn insert_mcp_event( + pool: &DbPool, + call_log_id: i64, + ai_tool: &str, + ai_project: &str, + ai_session_id: &str, + hostname: &str, + timestamp: &str, + call_id: &str, + tool_name: &str, + mcp_server: Option<&str>, + mcp_tool: Option<&str>, + is_error: Option, +) { + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO ai_mcp_events + (call_log_id, ai_tool, ai_project, ai_session_id, hostname, timestamp, + call_id, tool_name, mcp_server, mcp_tool, event_kind, is_error, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, 'call', ?11, ?6)", + rusqlite::params![ + call_log_id, + ai_tool, + ai_project, + ai_session_id, + hostname, + timestamp, + call_id, + tool_name, + mcp_server, + mcp_tool, + is_error.map(i64::from), + ], + ) + .unwrap(); +} + +#[test] +fn investigate_ai_mcp_incidents_bundle_has_bounded_collections_and_truncation_flags() { + let (pool, _dir) = test_pool(); + + let call_log = make_ai_entry( + "2026-01-01T00:00:00Z", + "devhost", + "codex", + "/tmp/project-d", + "sess-d", + "called mcp__labby__search", + ); + let correction_log = make_ai_entry( + "2026-01-01T00:01:00Z", + "devhost", + "codex", + "/tmp/project-d", + "sess-d", + "no, that's the wrong tool", + ); + insert_logs_batch(&pool, &[call_log, correction_log]).unwrap(); + let log_ids: Vec = { + let conn = pool.get().unwrap(); + let mut stmt = conn.prepare("SELECT id FROM logs ORDER BY id ASC").unwrap(); + stmt.query_map([], |row| row.get::<_, i64>(0)) + .unwrap() + .collect::>>() + .unwrap() + }; + insert_mcp_event( + &pool, + log_ids[0], + "codex", + "/tmp/project-d", + "sess-d", + "devhost", + "2026-01-01T00:00:00Z", + "call_d", + "mcp__labby__search", + Some("labby"), + Some("search"), + Some(false), + ); + + let result = investigate_ai_mcp_incidents( + &pool, + &AiMcpInvestigateParams { + mcp_server: Some("labby".into()), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.evidence.len(), 1); + let bundle = &result.evidence[0]; + assert_eq!(bundle.incident.mcp_server, "labby"); + assert_eq!(bundle.mcp_events.len(), 1); + assert!(!bundle.mcp_events_truncated); + assert!( + bundle + .signal_anchors + .iter() + .any(|e| e.message.contains("wrong tool")) + ); +} + +#[test] +fn investigate_ai_mcp_incidents_filters_by_incident_id() { + let (pool, _dir) = test_pool(); + let call_log = make_ai_entry( + "2026-01-01T00:00:00Z", + "devhost", + "claude", + "/tmp/project-f", + "sess-f", + "called mcp__gh__search", + ); + insert_logs_batch(&pool, &[call_log]).unwrap(); + let log_id: i64 = { + let conn = pool.get().unwrap(); + conn.query_row("SELECT id FROM logs LIMIT 1", [], |row| row.get(0)) + .unwrap() + }; + insert_mcp_event( + &pool, + log_id, + "claude", + "/tmp/project-f", + "sess-f", + "devhost", + "2026-01-01T00:00:00Z", + "call_f", + "mcp__gh__search", + Some("gh"), + Some("search"), + Some(false), + ); + + let all = search_ai_mcp_incidents( + &pool, + &AiMcpIncidentParams { + mcp_server: Some("gh".into()), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(all.incidents.len(), 1); + let incident_id = all.incidents[0].incident_id.clone(); + + let result = investigate_ai_mcp_incidents( + &pool, + &AiMcpInvestigateParams { + incident_id: Some(incident_id.clone()), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(result.evidence.len(), 1); + assert_eq!(result.evidence[0].incident.incident_id, incident_id); + + let none_result = investigate_ai_mcp_incidents( + &pool, + &AiMcpInvestigateParams { + incident_id: Some("mcp-inc-doesnotexist".into()), + ..Default::default() + }, + ) + .unwrap(); + assert!(none_result.evidence.is_empty()); +} + +/// Regression test for a bug where an exact `incident_id` lookup routed +/// through `search_ai_mcp_incidents` with `limit: Some(100)` and then +/// filtered client-side for the matching id — if the target incident ranked +/// below the top 100 by priority score, investigation silently returned +/// empty evidence for an incident that actually existed. This constructs +/// 100 higher-scored decoy incidents plus one lower-scored target so the +/// target provably ranks outside any top-100 window, then asserts the exact +/// lookup still finds it. +#[test] +fn investigate_ai_mcp_incidents_exact_incident_id_beyond_top_100_candidates() { + let (pool, _dir) = test_pool(); + + fn log_ids_for_session(pool: &DbPool, session_id: &str) -> Vec { + let conn = pool.get().unwrap(); + let mut stmt = conn + .prepare("SELECT id FROM logs WHERE ai_session_id = ?1 ORDER BY timestamp ASC, id ASC") + .unwrap(); + stmt.query_map([session_id], |row| row.get::<_, i64>(0)) + .unwrap() + .collect::>>() + .unwrap() + } + + // 100 decoy groups, each scored higher than baseline via a + // user_correction_after_tool_call anchor, so every decoy outranks the + // target. + for i in 0..100 { + let session_id = format!("sess-decoy-{i:03}"); + let call_log = make_ai_entry( + "2026-01-01T00:00:00Z", + "host-a", + "codex", + "/tmp/project-g", + &session_id, + "called mcp__labby__search", + ); + let correction_log = make_ai_entry( + "2026-01-01T00:00:30Z", + "host-a", + "codex", + "/tmp/project-g", + &session_id, + "no, that's the wrong tool", + ); + insert_logs_batch(&pool, &[call_log, correction_log]).unwrap(); + let ids = log_ids_for_session(&pool, &session_id); + insert_mcp_event( + &pool, + ids[0], + "codex", + "/tmp/project-g", + &session_id, + "host-a", + "2026-01-01T00:00:00Z", + &format!("call-decoy-{i:03}"), + "mcp__labby__search", + Some("labby"), + Some("search"), + Some(false), + ); + } + + // Target group: baseline score only (no anchor signal), guaranteeing it + // ranks last among the 101 total matching incidents. + let target_session_id = "sess-target".to_string(); + let target_call_log = make_ai_entry( + "2026-01-01T00:00:00Z", + "host-a", + "codex", + "/tmp/project-g", + &target_session_id, + "called mcp__labby__search", + ); + insert_logs_batch(&pool, &[target_call_log]).unwrap(); + let target_log_ids = log_ids_for_session(&pool, &target_session_id); + insert_mcp_event( + &pool, + target_log_ids[0], + "codex", + "/tmp/project-g", + &target_session_id, + "host-a", + "2026-01-01T00:00:00Z", + "call-target", + "mcp__labby__search", + Some("labby"), + Some("search"), + Some(false), + ); + + let target_lookup = search_ai_mcp_incidents( + &pool, + &AiMcpIncidentParams { + mcp_server: Some("labby".into()), + ai_session_id: Some(target_session_id.clone()), + limit: Some(1), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(target_lookup.incidents.len(), 1); + let target_id = target_lookup.incidents[0].incident_id.clone(); + + let top100 = search_ai_mcp_incidents( + &pool, + &AiMcpIncidentParams { + mcp_server: Some("labby".into()), + limit: Some(100), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(top100.total_incidents, 101, "100 decoys + 1 target"); + assert_eq!(top100.incidents.len(), 100); + assert!( + !top100 + .incidents + .iter() + .any(|inc| inc.incident_id == target_id), + "test setup invariant: target must rank outside the top 100" + ); + + // The regression check: an exact incident_id lookup must still find the + // target even though it ranks outside the top-100 candidate window. + let exact = investigate_ai_mcp_incidents( + &pool, + &AiMcpInvestigateParams { + incident_id: Some(target_id.clone()), + mcp_server: Some("labby".into()), + limit: Some(1), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!( + exact.evidence.len(), + 1, + "exact incident_id lookup must find an incident ranked outside the top 100" + ); + assert_eq!(exact.evidence[0].incident.incident_id, target_id); +} + +/// Regression test for a bug where the `nearby_logs` query only filtered by +/// timestamp range, with no hostname scope, so an incident on one host could +/// pull in unrelated log rows from a different host in the same time window. +#[test] +fn investigate_ai_mcp_incidents_nearby_logs_scoped_to_incident_hostname() { + let (pool, _dir) = test_pool(); + + let call_log = make_ai_entry( + "2026-01-01T00:00:00Z", + "host-a", + "codex", + "/tmp/project-h", + "sess-h", + "called mcp__labby__search", + ); + insert_logs_batch(&pool, &[call_log]).unwrap(); + let log_id: i64 = { + let conn = pool.get().unwrap(); + conn.query_row("SELECT id FROM logs LIMIT 1", [], |row| row.get(0)) + .unwrap() + }; + insert_mcp_event( + &pool, + log_id, + "codex", + "/tmp/project-h", + "sess-h", + "host-a", + "2026-01-01T00:00:00Z", + "call-h", + "mcp__labby__search", + Some("labby"), + Some("search"), + Some(false), + ); + + // Unrelated non-AI log on a DIFFERENT host, within the correlation window. + let other_host_log = LogBatchEntry { + timestamp: "2026-01-01T00:01:00Z".to_string(), + hostname: "host-b".to_string(), + facility: Some("local0".to_string()), + severity: "error".to_string(), + app_name: Some("nginx".to_string()), + process_id: None, + message: "connection refused".to_string(), + raw: "connection refused".to_string(), + source_ip: "10.0.0.5:514".to_string(), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + }; + insert_logs_batch(&pool, &[other_host_log]).unwrap(); + + let result = investigate_ai_mcp_incidents( + &pool, + &AiMcpInvestigateParams { + mcp_server: Some("labby".into()), + limit: Some(1), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.evidence.len(), 1); + let bundle = &result.evidence[0]; + assert!( + bundle.nearby_logs.iter().all(|e| e.hostname == "host-a"), + "nearby_logs leaked a cross-host row: {:?}", + bundle.nearby_logs + ); + assert!( + !bundle + .nearby_logs + .iter() + .any(|e| e.message.contains("connection refused")), + "cross-host log should not appear in nearby_logs" + ); +} diff --git a/crates/shared/cortex/storage-sqlite/src/mcp_incidents.rs b/crates/shared/cortex/storage-sqlite/src/mcp_incidents.rs new file mode 100644 index 00000000..a3aa0e38 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/mcp_incidents.rs @@ -0,0 +1,401 @@ +//! MCP-incident grouping and scoring. Groups `ai_mcp_events` rows into +//! `McpIncident`s by `(mcp_server, mcp_tool, ai_tool, ai_project, +//! ai_session_id, hostname, window_bucket)` per GH #94's "MCP grouping key" +//! section, scans nearby transcript logs for the six deterministic anchor +//! signals in `cortex_domain::mcp_signal_detectors`, and scores/sorts the +//! resulting groups. Mirrors `search_ai_skill_incidents` in +//! `src/db/skill_incidents.rs` but keyed on MCP tool-call usage instead of +//! skill-attribution events. Only MCP-classified rows (`mcp_server IS NOT +//! NULL`) participate in grouping — general/builtin tool calls are excluded +//! per the schema note in GH #94 ("`cortex assess mcp` filters to +//! MCP-classified rows"). + +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +use cortex_domain::mcp_signal_detectors::{ + detect_auth_or_permission_failure, detect_repeated_call_failure, + detect_schema_or_validation_error, detect_timeout_or_rate_limit, detect_unknown_tool_or_server, + detect_user_correction_after_tool_call, +}; +pub use cortex_domain::{McpIncident, McpSignalCounts}; + +use super::pool::DbPool; + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct AiMcpIncidentParams { + pub mcp_server: Option, + pub mcp_tool: Option, + pub tool_name: Option, + pub ai_tool: Option, + pub ai_project: Option, + pub ai_session_id: Option, + pub hostname: Option, + pub since: Option, + pub until: Option, + /// Exact incident_id match. When set, filters the full computed incident + /// set (bounded only by `MCP_INCIDENT_CANDIDATE_CAP`, not `limit`) before + /// the priority-ranked truncation, so a match ranked below `limit` is + /// still found. + pub incident_id: Option, + /// Max incidents to return. Default 20, clamp 1..=100. + pub limit: Option, + /// Grouping window in minutes. Default 10, clamp 1..=120. + pub window_minutes: Option, + /// Restrict to incidents containing at least one of these signal + /// categories. Empty = no filter (all incidents). + pub signals: Vec, + /// Minimum `priority_score` (inclusive). `None` = no filter. + pub min_score: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiMcpIncidentResult { + pub incidents: Vec, + pub total_incidents: usize, + pub candidate_event_rows: usize, + pub candidate_cap: usize, + pub candidate_window_truncated: bool, + pub truncated: bool, +} + +const MCP_INCIDENT_CANDIDATE_CAP: usize = 10_000; + +/// Grouping key for MCP incidents: `(mcp_server, mcp_tool, ai_tool, +/// ai_project, ai_session_id, hostname, window_bucket)`. +/// `window_bucket = unix_secs / window_secs * window_secs` (floor to window +/// boundary), mirroring `search_ai_skill_incidents`'s grouping. +pub fn search_ai_mcp_incidents( + pool: &DbPool, + params: &AiMcpIncidentParams, +) -> Result { + let conn = pool.get()?; + let limit = params.limit.unwrap_or(20).clamp(1, 100) as usize; + let window_secs = i64::from(params.window_minutes.unwrap_or(10).clamp(1, 120)) * 60; + + struct McpEventRow { + id: i64, + timestamp: String, + hostname: String, + tool: String, + project: String, + session_id: String, + mcp_server: String, + mcp_tool: Option, + is_error: Option, + } + + // Only MCP-classified rows participate in incident grouping — general + // tool calls (mcp_server IS NULL) are excluded here (GH #94: "cortex + // assess mcp filters to MCP-classified rows"). + let mut sql = String::from( + "SELECT id, timestamp, hostname, ai_tool, ai_project, ai_session_id, + mcp_server, mcp_tool, is_error + FROM ai_mcp_events + WHERE mcp_server IS NOT NULL", + ); + let mut bindings: Vec = Vec::new(); + let mut idx = 1usize; + if let Some(server) = ¶ms.mcp_server { + sql.push_str(&format!(" AND mcp_server = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(server.clone())); + idx += 1; + } + if let Some(tool) = ¶ms.mcp_tool { + sql.push_str(&format!(" AND mcp_tool = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(tool.clone())); + idx += 1; + } + if let Some(tool_name) = ¶ms.tool_name { + sql.push_str(&format!(" AND tool_name = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(tool_name.clone())); + idx += 1; + } + if let Some(ai_tool) = ¶ms.ai_tool { + sql.push_str(&format!(" AND ai_tool = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(ai_tool.clone())); + idx += 1; + } + if let Some(project) = ¶ms.ai_project { + sql.push_str(&format!(" AND ai_project = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(project.clone())); + idx += 1; + } + if let Some(session_id) = ¶ms.ai_session_id { + sql.push_str(&format!(" AND ai_session_id = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(session_id.clone())); + idx += 1; + } + if let Some(hostname) = ¶ms.hostname { + sql.push_str(&format!(" AND hostname = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(hostname.clone())); + idx += 1; + } + if let Some(from) = ¶ms.since { + sql.push_str(&format!(" AND timestamp >= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(from.clone())); + idx += 1; + } + if let Some(to) = ¶ms.until { + sql.push_str(&format!(" AND timestamp <= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(to.clone())); + } + let _ = idx; + sql.push_str(&format!( + " ORDER BY timestamp ASC LIMIT {}", + MCP_INCIDENT_CANDIDATE_CAP + 1 + )); + + let mut stmt = conn.prepare(&sql)?; + let candidate_events: Vec = stmt + .query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + Ok(McpEventRow { + id: row.get(0)?, + timestamp: row.get(1)?, + hostname: row.get(2)?, + tool: row.get(3)?, + project: row.get(4)?, + session_id: row.get(5)?, + mcp_server: row.get(6)?, + mcp_tool: row.get(7)?, + is_error: row.get::<_, Option>(8)?.map(|v| v != 0), + }) + })? + .collect::>>()?; + + let candidate_window_truncated = candidate_events.len() > MCP_INCIDENT_CANDIDATE_CAP; + let raw_candidate_count = candidate_events.len(); + + // ── Group by (mcp_server, mcp_tool, ai_tool, ai_project, ai_session_id, + // hostname, window_bucket) ─────────────────────────────────────────────── + type GroupKey = (String, Option, String, String, String, String, i64); + let mut groups: HashMap> = HashMap::new(); + + for row in candidate_events.iter().take(MCP_INCIDENT_CANDIDATE_CAP) { + let bucket = chrono::DateTime::parse_from_rfc3339(&row.timestamp) + .map(|dt| (dt.timestamp() / window_secs) * window_secs) + .unwrap_or(0); + let key = ( + row.mcp_server.clone(), + row.mcp_tool.clone(), + row.tool.clone(), + row.project.clone(), + row.session_id.clone(), + row.hostname.clone(), + bucket, + ); + groups.entry(key).or_default().push(row); + } + + let mut incidents: Vec = Vec::with_capacity(groups.len()); + for ((mcp_server, mcp_tool, tool, project, session_id, hostname, _bucket), events) in groups { + let first_seen = events + .first() + .map(|e| e.timestamp.clone()) + .unwrap_or_default(); + let last_seen = events + .last() + .map(|e| e.timestamp.clone()) + .unwrap_or_default(); + let duration_secs = { + let t0 = chrono::DateTime::parse_from_rfc3339(&first_seen) + .map(|dt| dt.timestamp()) + .unwrap_or(0); + let t1 = chrono::DateTime::parse_from_rfc3339(&last_seen) + .map(|dt| dt.timestamp()) + .unwrap_or(0); + (t1 - t0).max(0) + }; + let error_count = events.iter().filter(|e| e.is_error == Some(true)).count(); + + // Window bounds for anchor detection: from first event to + // window_secs after the last one. + let win_from = first_seen.clone(); + let win_to = chrono::DateTime::parse_from_rfc3339(&last_seen) + .map(|dt| { + (dt.with_timezone(&chrono::Utc) + chrono::Duration::seconds(window_secs)) + .format("%Y-%m-%dT%H:%M:%S%.3fZ") + .to_string() + }) + .unwrap_or_else(|_| last_seen.clone()); + + let mut anchor_stmt = conn.prepare_cached( + "SELECT id, message FROM logs + WHERE ai_session_id = ?1 AND ai_project = ?2 AND ai_tool = ?3 + AND timestamp >= ?4 AND timestamp <= ?5 + ORDER BY timestamp ASC + LIMIT 500", + )?; + let anchor_rows: Vec<(i64, String)> = anchor_stmt + .query_map( + rusqlite::params![session_id, project, tool, win_from, win_to], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)), + )? + .collect::>>()?; + + let mut counts = McpSignalCounts::default(); + let mut anchor_log_ids: Vec = Vec::new(); + + for (id, message) in &anchor_rows { + let mut hit = false; + if detect_timeout_or_rate_limit(message) { + counts.timeout_or_rate_limit += 1; + hit = true; + } + if detect_auth_or_permission_failure(message) { + counts.auth_or_permission_failure += 1; + hit = true; + } + if detect_schema_or_validation_error(message) { + counts.schema_or_validation_error += 1; + hit = true; + } + if detect_unknown_tool_or_server(message) { + counts.unknown_tool_or_server += 1; + hit = true; + } + if detect_user_correction_after_tool_call(message) { + counts.user_correction_after_tool_call += 1; + hit = true; + } + if hit { + anchor_log_ids.push(*id); + } + } + if detect_repeated_call_failure(error_count) { + counts.repeated_call_failure = error_count; + } + + anchor_log_ids.sort_unstable(); + anchor_log_ids.dedup(); + + let mut signals_present: Vec = Vec::new(); + if counts.repeated_call_failure > 0 { + signals_present.push("repeated_call_failure".to_string()); + } + if counts.timeout_or_rate_limit > 0 { + signals_present.push("timeout_or_rate_limit".to_string()); + } + if counts.auth_or_permission_failure > 0 { + signals_present.push("auth_or_permission_failure".to_string()); + } + if counts.schema_or_validation_error > 0 { + signals_present.push("schema_or_validation_error".to_string()); + } + if counts.unknown_tool_or_server > 0 { + signals_present.push("unknown_tool_or_server".to_string()); + } + if counts.user_correction_after_tool_call > 0 { + signals_present.push("user_correction_after_tool_call".to_string()); + } + signals_present.sort(); + + // ── Locked scoring formula (mirrors search_ai_skill_incidents' + // weighting shape; weights chosen so a single repeated-failure or + // user-correction signal already crosses into "medium") ─────────── + let signal_variety = signals_present.len() as f64; + let priority_score = events.len() as f64 * 2.0 + + counts.repeated_call_failure as f64 * 10.0 + + counts.timeout_or_rate_limit as f64 * 8.0 + + counts.auth_or_permission_failure as f64 * 12.0 + + counts.schema_or_validation_error as f64 * 10.0 + + counts.unknown_tool_or_server as f64 * 12.0 + + counts.user_correction_after_tool_call as f64 * 15.0 + + signal_variety * 5.0; + + let priority_label = if priority_score < 15.0 { + "low" + } else if priority_score < 35.0 { + "medium" + } else if priority_score < 60.0 { + "high" + } else { + "critical" + } + .to_string(); + + let mut mcp_event_ids: Vec = events.iter().map(|e| e.id).collect(); + mcp_event_ids.sort_unstable(); + + let incident_id = { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + let mut h = DefaultHasher::new(); + mcp_server.hash(&mut h); + mcp_tool.hash(&mut h); + tool.hash(&mut h); + project.hash(&mut h); + session_id.hash(&mut h); + hostname.hash(&mut h); + for id in &anchor_log_ids { + id.hash(&mut h); + } + for id in &mcp_event_ids { + id.hash(&mut h); + } + format!("mcp-inc-{:016x}", h.finish()) + }; + + incidents.push(McpIncident { + incident_id, + mcp_server, + mcp_tool, + tool, + project, + session_id, + hostname, + first_seen, + last_seen, + duration_secs, + event_count: mcp_event_ids.len(), + error_count, + mcp_event_ids, + anchor_log_ids, + signal_counts: counts, + signals_present, + priority_score, + priority_label, + window_minutes: (window_secs / 60) as u32, + }); + } + + if let Some(incident_id) = ¶ms.incident_id { + incidents.retain(|inc| &inc.incident_id == incident_id); + } + if !params.signals.is_empty() { + incidents.retain(|inc| { + inc.signals_present + .iter() + .any(|s| params.signals.contains(s)) + }); + } + if let Some(min_score) = params.min_score { + incidents.retain(|inc| inc.priority_score >= min_score); + } + + // total_cmp (not partial_cmp/unwrap_or(Equal)) — a total order even if a + // NaN score ever appears. + incidents.sort_by(|a, b| { + b.priority_score + .total_cmp(&a.priority_score) + .then_with(|| b.last_seen.cmp(&a.last_seen)) + }); + + let total_incidents = incidents.len(); + let truncated = total_incidents > limit || candidate_window_truncated; + incidents.truncate(limit); + + Ok(AiMcpIncidentResult { + incidents, + total_incidents, + candidate_event_rows: raw_candidate_count.min(MCP_INCIDENT_CANDIDATE_CAP), + candidate_cap: MCP_INCIDENT_CANDIDATE_CAP, + candidate_window_truncated, + truncated, + }) +} + +#[cfg(test)] +#[path = "mcp_incidents_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/mcp_incidents_tests.rs b/crates/shared/cortex/storage-sqlite/src/mcp_incidents_tests.rs new file mode 100644 index 00000000..9485ed2a --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/mcp_incidents_tests.rs @@ -0,0 +1,338 @@ +use super::*; +use crate::config::StorageConfig; +use crate::pool::init_pool; +use crate::{DbPool, LogBatchEntry, insert_logs_batch}; + +fn test_pool() -> (DbPool, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let pool = init_pool(&StorageConfig::for_test(db_path)).unwrap(); + (pool, dir) +} + +fn make_ai_entry( + ts: &str, + host: &str, + tool: &str, + project: &str, + session_id: &str, + message: &str, +) -> LogBatchEntry { + LogBatchEntry { + timestamp: ts.to_string(), + hostname: host.to_string(), + facility: Some("local0".to_string()), + severity: "info".to_string(), + app_name: Some("ai-transcript".to_string()), + process_id: None, + message: message.to_string(), + raw: message.to_string(), + source_ip: "127.0.0.1:514".to_string(), + docker_checkpoint: None, + ai_tool: Some(tool.to_string()), + ai_project: Some(project.to_string()), + ai_session_id: Some(session_id.to_string()), + ai_transcript_path: Some(format!("{project}/{session_id}.jsonl")), + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + } +} + +#[allow(clippy::too_many_arguments)] +fn insert_mcp_event( + pool: &DbPool, + call_log_id: i64, + ai_tool: &str, + ai_project: &str, + ai_session_id: &str, + hostname: &str, + timestamp: &str, + call_id: &str, + tool_name: &str, + mcp_server: Option<&str>, + mcp_tool: Option<&str>, + is_error: Option, +) { + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO ai_mcp_events + (call_log_id, ai_tool, ai_project, ai_session_id, hostname, timestamp, + call_id, tool_name, mcp_server, mcp_tool, event_kind, is_error, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10, 'call', ?11, ?6)", + rusqlite::params![ + call_log_id, + ai_tool, + ai_project, + ai_session_id, + hostname, + timestamp, + call_id, + tool_name, + mcp_server, + mcp_tool, + is_error.map(i64::from), + ], + ) + .unwrap(); +} + +#[test] +fn search_ai_mcp_incidents_groups_by_server_tool_session_window_and_scores() { + let (pool, _dir) = test_pool(); + + let call_log = make_ai_entry( + "2026-01-01T00:00:00Z", + "devhost", + "codex", + "/home/jmagar/workspace/cortex", + "sess-mcp-1", + "called mcp__labby__search", + ); + let correction_log = make_ai_entry( + "2026-01-01T00:02:00Z", + "devhost", + "codex", + "/home/jmagar/workspace/cortex", + "sess-mcp-1", + "no, that's the wrong tool for this", + ); + insert_logs_batch(&pool, &[call_log, correction_log]).unwrap(); + + let log_ids: Vec = { + let conn = pool.get().unwrap(); + let mut stmt = conn.prepare("SELECT id FROM logs ORDER BY id ASC").unwrap(); + stmt.query_map([], |row| row.get::<_, i64>(0)) + .unwrap() + .collect::>>() + .unwrap() + }; + assert_eq!(log_ids.len(), 2); + + insert_mcp_event( + &pool, + log_ids[0], + "codex", + "/home/jmagar/workspace/cortex", + "sess-mcp-1", + "devhost", + "2026-01-01T00:00:00Z", + "call_1", + "mcp__labby__search", + Some("labby"), + Some("search"), + Some(false), + ); + + let result = search_ai_mcp_incidents( + &pool, + &AiMcpIncidentParams { + mcp_server: Some("labby".into()), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.incidents.len(), 1, "expected one grouped incident"); + let incident = &result.incidents[0]; + assert_eq!(incident.mcp_server, "labby"); + assert_eq!(incident.mcp_tool.as_deref(), Some("search")); + assert_eq!(incident.tool, "codex"); + assert_eq!(incident.project, "/home/jmagar/workspace/cortex"); + assert_eq!(incident.session_id, "sess-mcp-1"); + assert_eq!(incident.hostname, "devhost"); + assert_eq!(incident.event_count, 1); + assert_eq!(incident.signal_counts.user_correction_after_tool_call, 1); + assert!( + incident + .signals_present + .contains(&"user_correction_after_tool_call".to_string()) + ); + assert!(!incident.incident_id.is_empty()); + assert!(incident.incident_id.starts_with("mcp-inc-")); +} + +#[test] +fn search_ai_mcp_incidents_excludes_non_mcp_classified_rows() { + let (pool, _dir) = test_pool(); + let call_log = make_ai_entry( + "2026-01-01T00:00:00Z", + "devhost", + "codex", + "/tmp/project", + "sess-builtin", + "called shell", + ); + insert_logs_batch(&pool, &[call_log]).unwrap(); + let log_id: i64 = { + let conn = pool.get().unwrap(); + conn.query_row("SELECT id FROM logs LIMIT 1", [], |row| row.get(0)) + .unwrap() + }; + // Builtin tool call: mcp_server is NULL. + insert_mcp_event( + &pool, + log_id, + "codex", + "/tmp/project", + "sess-builtin", + "devhost", + "2026-01-01T00:00:00Z", + "call_builtin", + "shell", + None, + None, + Some(false), + ); + + let result = search_ai_mcp_incidents(&pool, &AiMcpIncidentParams::default()).unwrap(); + assert!( + result.incidents.is_empty(), + "non-MCP-classified (mcp_server IS NULL) rows must not form incidents" + ); +} + +#[test] +fn search_ai_mcp_incidents_repeated_failures_trigger_signal() { + let (pool, _dir) = test_pool(); + let call_log_1 = make_ai_entry( + "2026-01-01T00:00:00Z", + "devhost", + "claude", + "/tmp/project-e", + "sess-e", + "call 1", + ); + let call_log_2 = make_ai_entry( + "2026-01-01T00:01:00Z", + "devhost", + "claude", + "/tmp/project-e", + "sess-e", + "call 2", + ); + insert_logs_batch(&pool, &[call_log_1, call_log_2]).unwrap(); + let log_ids: Vec = { + let conn = pool.get().unwrap(); + let mut stmt = conn.prepare("SELECT id FROM logs ORDER BY id ASC").unwrap(); + stmt.query_map([], |row| row.get::<_, i64>(0)) + .unwrap() + .collect::>>() + .unwrap() + }; + insert_mcp_event( + &pool, + log_ids[0], + "claude", + "/tmp/project-e", + "sess-e", + "devhost", + "2026-01-01T00:00:00Z", + "call_a", + "mcp__gh__search", + Some("gh"), + Some("search"), + Some(true), + ); + insert_mcp_event( + &pool, + log_ids[1], + "claude", + "/tmp/project-e", + "sess-e", + "devhost", + "2026-01-01T00:01:00Z", + "call_b", + "mcp__gh__search", + Some("gh"), + Some("search"), + Some(true), + ); + + let result = search_ai_mcp_incidents( + &pool, + &AiMcpIncidentParams { + mcp_server: Some("gh".into()), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(result.incidents.len(), 1); + let incident = &result.incidents[0]; + assert_eq!(incident.error_count, 2); + assert!(incident.signal_counts.repeated_call_failure >= 2); + assert!( + incident + .signals_present + .contains(&"repeated_call_failure".to_string()) + ); +} + +#[test] +fn search_ai_mcp_incidents_min_score_and_signals_filters() { + let (pool, _dir) = test_pool(); + let call_log = make_ai_entry( + "2026-01-01T00:00:00Z", + "devhost", + "claude", + "/tmp/project-c", + "sess-c", + "called mcp__labby__search", + ); + insert_logs_batch(&pool, &[call_log]).unwrap(); + let log_id: i64 = { + let conn = pool.get().unwrap(); + conn.query_row("SELECT id FROM logs LIMIT 1", [], |row| row.get(0)) + .unwrap() + }; + insert_mcp_event( + &pool, + log_id, + "claude", + "/tmp/project-c", + "sess-c", + "devhost", + "2026-01-01T00:00:00Z", + "call_c", + "mcp__labby__search", + Some("labby"), + Some("search"), + Some(false), + ); + + let filtered = search_ai_mcp_incidents( + &pool, + &AiMcpIncidentParams { + mcp_server: Some("labby".into()), + min_score: Some(10.0), + ..Default::default() + }, + ) + .unwrap(); + assert!(filtered.incidents.is_empty()); + + let filtered_by_signal = search_ai_mcp_incidents( + &pool, + &AiMcpIncidentParams { + mcp_server: Some("labby".into()), + signals: vec!["timeout_or_rate_limit".into()], + ..Default::default() + }, + ) + .unwrap(); + assert!(filtered_by_signal.incidents.is_empty()); +} + +#[test] +fn search_ai_mcp_incidents_sorts_by_score_with_total_cmp() { + let mut scores = [f64::NAN, 3.0, 1.0, f64::NAN, 2.0]; + scores.sort_by(|a, b| b.total_cmp(a)); + assert_eq!( + scores.len(), + 5, + "total_cmp sort must not panic or drop elements on NaN" + ); +} diff --git a/crates/shared/cortex/storage-sqlite/src/models.rs b/crates/shared/cortex/storage-sqlite/src/models.rs new file mode 100644 index 00000000..a8372c97 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/models.rs @@ -0,0 +1,624 @@ +use serde::{Deserialize, Serialize}; + +pub use cortex_domain::AbuseMatch as AiAbuseMatch; +pub use cortex_domain::{AbuseIncident, AppLogCount, LogEntry, SeverityCount}; + +/// Named struct for a log entry used in batch insertion and the syslog parse pipeline. +/// +/// Replaces the former 8-tuple type alias; named fields prevent silent data corruption +/// from positional swaps between structurally identical `String`/`Option` fields. +/// +/// For syslog input, `source_ip` records the actual network sender address (IP:port) +/// independent of the hostname claimed in the syslog message body. OTLP stores the +/// peer IP without the ephemeral port. Docker ingest uses configured +/// `docker://host/container/stream` and `docker-event://host/container/action` +/// source identifiers instead. +#[derive(Debug, Clone)] +pub struct LogBatchEntry { + pub timestamp: String, + pub hostname: String, + pub facility: Option, + pub severity: String, + pub app_name: Option, + pub process_id: Option, + pub message: String, + pub raw: String, + /// Source identifier. Syslog input uses the actual network sender address + /// (IP:port); OTLP uses peer IP; Docker ingest uses + /// docker://host/container/stream and docker-event://host/container/action. + pub source_ip: String, + pub docker_checkpoint: Option, + pub ai_tool: Option, + pub ai_project: Option, + pub ai_session_id: Option, + pub ai_transcript_path: Option, + pub metadata_json: Option, + /// HTTP status code (3 digits). Indexed column. Set by `swag` parser. + pub http_status: Option, + + /// Authentication outcome ("success" | "failure" | "denied" | "challenge"). + /// Indexed column. Set by `authelia` parser. + pub auth_outcome: Option<&'static str>, + + /// DNS block decision. `Some(true)` = filtered/blocked, `Some(false)` = explicit + /// allow, `None` = N/A (rewrites and non-DNS rows). Indexed column. + pub dns_blocked: Option, + + /// Normalised event verb (closed enum per parser). Indexed column. + pub event_action: Option, + + /// Per-row parser diagnostic: "{parser_name}: {ParserError::Display}", + /// truncated to 512 bytes. No index — diagnostic only. + pub parse_error: Option, +} + +#[derive(Debug, Clone)] +pub struct DockerCheckpoint { + pub host_name: String, + pub container_id: String, + pub timestamp: String, +} + +#[derive(Debug, Clone, Default)] +pub struct ListAiSessionsParams { + pub ai_project: Option, + pub ai_tool: Option, + pub host: Option, + pub since: Option, + pub until: Option, + pub limit: Option, +} + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct AiSessionEntry { + pub ai_project: String, + pub ai_tool: String, + pub ai_session_id: String, + pub ai_transcript_path: Option, + pub hostname: String, + pub first_seen: String, + pub last_seen: String, + pub event_count: i64, +} + +#[derive(Debug, Clone, Default)] +pub struct SearchAiSessionsParams { + pub query: String, + pub ai_project: Option, + pub ai_tool: Option, + /// Filter AI transcript sessions to those where the session's host matches. + pub host: Option, + /// Filter AI transcript sessions to those where the session's app matches. + pub app: Option, + pub since: Option, + pub until: Option, + pub limit: Option, +} + +/// Error/warning summary entry (one row per hostname+severity, plus optional app_name) +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ErrorSummaryEntry { + pub hostname: String, + /// Populated when the summary was requested with `group_by=app_name`. + pub app_name: Option, + pub severity: String, + pub count: i64, +} + +/// Host registry entry with first/last seen and log count +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct HostEntry { + pub hostname: String, + pub first_seen: String, + pub last_seen: String, + pub log_count: i64, +} + +/// Database statistics snapshot +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct DbStats { + pub total_logs: i64, + pub total_hosts: i64, + pub oldest_log: Option, + pub newest_log: Option, + /// Formatted as "X.XX" MB + pub logical_db_size_mb: String, + /// Formatted as "X.XX" MB + pub physical_db_size_mb: String, + /// Formatted as "X.XX" MB when available + pub free_disk_mb: Option, + pub max_db_size_mb: u64, + pub min_free_disk_mb: u64, + pub write_blocked: bool, + /// Phantom FTS rows: entries in logs_fts that no longer have a matching log row. + /// Accumulate between merge cycles; non-zero value is normal and cleaned up by + /// periodic fts_incremental_merge. High values indicate merge is falling behind. + /// + /// `None` when the FTS diagnostic was skipped: computing it requires + /// `COUNT(*) FROM logs_fts`, an external-content FTS5 index scan that is + /// expensive on very large databases. The default `stats` path skips it; + /// pass `include_fts_diagnostics` to compute it explicitly. + pub phantom_fts_rows: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SearchedAiSessionEntry { + pub ai_project: String, + pub ai_tool: String, + pub ai_session_id: String, + pub hostname: String, + pub first_seen: String, + pub last_seen: String, + pub event_count: i64, + pub match_count: i64, + pub best_snippet: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SearchAiSessionsResult { + pub total_candidates: usize, + pub candidate_rows: usize, + pub candidate_cap: usize, + pub candidate_window_truncated: bool, + pub truncated: bool, + pub sessions: Vec, +} + +#[derive(Debug, Clone, Default)] +pub struct AiAbuseParams { + pub ai_project: Option, + pub ai_tool: Option, + pub since: Option, + pub until: Option, + pub limit: Option, + pub before: Option, + pub after: Option, + pub terms: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiAbuseResult { + pub terms: Vec, + pub candidate_rows: usize, + pub candidate_cap: usize, + pub candidate_window_truncated: bool, + pub truncated: bool, + pub matches: Vec, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct AiCorrelateParams { + pub ai_project: Option, + pub ai_tool: Option, + pub ai_session_id: Option, + pub ai_query: Option, + pub since: Option, + pub until: Option, + pub limit: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiRelatedWindow { + pub anchor_index: usize, + pub anchor_time: String, + pub window_from: String, + pub window_to: String, +} + +/// DB-layer carrier for graph-anchored session correlation: the session time +/// bounds, the entities/hosts discovered by traversing the graph from the +/// session entity, and the fanned-out logs. `used_graph` is false when no +/// `ai_session` graph entity exists for the session (time-windowed fallback). +#[derive(Debug, Clone, Default)] +pub struct SessionGraphInputs { + pub bounds: Option<(String, String)>, + pub discovered_hosts: Vec, + pub discovered_entities: Vec, + pub used_graph: bool, + pub logs: Vec, +} + +/// A graph entity matched while resolving a topic string, with how it matched +/// (`exact` canonical key, `prefix` of a key, or `alias`). +#[derive(Debug, Clone)] +pub struct ResolvedTopicEntity { + pub entity_type: String, + pub canonical_key: String, + pub match_kind: &'static str, + /// Resolver outcome: `Resolved` for exact canonical-key and alias + /// identity matches, `Ambiguous` for weak prefix/label candidates that + /// never drive log fan-out. Stringified via + /// [`super::entity_resolution::ResolverStatus::as_str`] only at the serde + /// boundary. + pub resolver_status: super::entity_resolution::ResolverStatus, +} + +/// One correlated log row annotated with why it was included and the +/// resolver outcome for its inclusion path. +#[derive(Debug, Clone)] +pub struct GraphRelatedLogEntry { + pub entry: LogEntry, + pub inclusion_reason: String, + pub resolver_status: super::entity_resolution::ResolverStatus, + pub fallback_kind: Option, +} + +/// DB-layer carrier for topic correlation: the entities the topic resolved to, +/// the entities/hosts reached by graph expansion, and the fanned-out logs. +#[derive(Debug, Clone, Default)] +pub struct TopicGraphInputs { + pub resolved: Vec, + /// Entities reached by traversal that were not themselves resolved seeds. + pub expansion: Vec<(String, String)>, + pub discovered_hosts: Vec, + pub logs: Vec, + /// `true` when the service-topic graph walk + /// (`graph_walk_service_topic`) hit + /// `GRAPH_SERVICE_TOPIC_ENTITY_CAP` and the reached neighborhood was cut + /// off rather than exhaustive. + pub graph_walk_truncated: bool, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct AiRelatedLogsParams { + pub windows: Vec, + pub query: Option, + pub host: Option, + pub source: Option, + pub severity_in: Vec, + pub app: Option, + pub limit_per_anchor: u32, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiRelatedLogsForAnchor { + pub anchor_index: usize, + pub logs: Vec, + pub truncated: bool, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct AiUsageBlocksParams { + pub ai_project: Option, + pub ai_tool: Option, + pub since: Option, + pub until: Option, + pub limit: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiUsageBlock { + pub bucket_start: String, + pub bucket_end: String, + pub project: String, + pub tool: String, + pub session_count: i64, + pub event_count: i64, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiUsageBlocksResult { + pub total_blocks: usize, + pub truncated: bool, + pub blocks: Vec, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct AiProjectContextParams { + pub project: String, + pub ai_tool: Option, + pub limit: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiProjectContext { + pub project: String, + pub tools: Vec, + pub sessions: Vec, + pub hostnames: Vec, + pub first_seen: Option, + pub last_seen: Option, + pub event_count: i64, + pub recent_entries_truncated: bool, + pub recent_entries: Vec, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ListAiToolsParams { + pub ai_project: Option, + pub since: Option, + pub until: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiToolInventoryEntry { + pub tool: String, + pub event_count: i64, + pub session_count: i64, + pub first_seen: String, + pub last_seen: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ListAiToolsResult { + pub total_tools: usize, + pub truncated: bool, + pub tools: Vec, +} + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct ListAiProjectsParams { + pub ai_tool: Option, + pub since: Option, + pub until: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiProjectInventoryEntry { + pub project: String, + pub tools: Vec, + pub event_count: i64, + pub session_count: i64, + pub first_seen: String, + pub last_seen: String, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ListAiProjectsResult { + pub total_projects: usize, + pub truncated: bool, + pub projects: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StorageMetrics { + pub logical_db_size_bytes: u64, + pub physical_db_size_bytes: u64, + pub free_disk_bytes: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StorageRecovery { + pub logical_db_size_bytes: u64, + pub free_disk_bytes: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct StorageEnforcementOutcome { + pub metrics: StorageMetrics, + pub recovery: StorageRecovery, + pub deleted_rows: usize, + pub write_blocked: bool, +} + +#[derive(Debug, Clone)] +pub struct StorageBudgetState { + pub metrics: StorageMetrics, + pub write_blocked: bool, +} + +/// Parameters for searching logs +#[derive(Debug, Clone, Default, Deserialize)] +pub struct SearchParams { + /// Full-text search query (FTS5 syntax) + pub query: Option, + /// Filter by hostname + pub host: Option, + /// Filter by source identifier. Syslog uses verified network sender address + /// (IP:port); OTLP uses peer IP; Docker ingest uses + /// docker://host/container/stream or docker-event://host/container/action. + pub source: Option, + /// Filter by source identifier prefix using an indexed range predicate. + pub source_ip_prefix: Option, + /// Filter by severity (exact match: emerg, alert, crit, err, warning, notice, info, debug) + pub severity: Option, + /// Filter by one of a set of severity levels (for threshold queries) + pub severity_in: Option>, + /// Filter by app name + pub app: Option, + /// Filter by syslog facility name (e.g. `kern`, `auth`, `daemon`) + pub facility: Option, + /// Exclude a syslog facility while keeping rows with unknown facility. + pub exclude_facility: Option, + /// Filter by process_id (exact match) + pub process_id: Option, + /// Start of time range (ISO 8601) + pub since: Option, + /// End of time range (ISO 8601) + pub until: Option, + /// Start of receive-time range (ISO 8601) + pub received_since: Option, + /// End of receive-time range (ISO 8601) + pub received_until: Option, + /// Max results to return + pub limit: Option, + pub ai_tool: Option, + pub ai_project: Option, + pub ai_session_id: Option, + pub event_action: Option, + pub exclude_ai: bool, +} + +impl SearchParams { + /// True when a filter is set on a column backed by a `(col, timestamp)` + /// index AND whose partitions are small enough for the index-led plan + /// (hostname, source_ip, app_name, event_action, ai_project). The FTS + /// search uses this to choose the index-led intersect plan — which leads + /// with the filter's composite index and intersects the FTS match set — + /// instead of scanning the entire match set and filtering post-hoc (the + /// pathology that made `search --host ` ~200s). + /// + /// `severity`/`severity_in` are deliberately EXCLUDED: a single severity + /// can be >90% of the table, so leading with `idx_logs_sev_time` for a + /// rare term walks nearly the entire partition before LIMIT fills + /// (full-review PH1). Severity-only searches take the capped-candidate + /// path instead; severity combined with a selective filter still uses the + /// fast path via the selective column's index. + pub(crate) fn has_indexed_equality_filter(&self) -> bool { + self.host.is_some() + || self.source.is_some() + || self.source_ip_prefix.is_some() + || self.app.is_some() + || self.event_action.is_some() + || self.ai_project.is_some() + } +} + +// --------------------------------------------------------------------------- +// Abuse incident grouping +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct AiIncidentParams { + pub ai_project: Option, + pub ai_tool: Option, + pub since: Option, + pub until: Option, + /// Max incidents to return. Default 20, clamp 1..=100. + pub limit: Option, + /// Grouping window in minutes. Default 10, clamp 1..=120. + pub window_minutes: Option, + pub terms: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiIncidentResult { + pub incidents: Vec, + pub total_incidents: usize, + pub candidate_rows: usize, + pub candidate_cap: usize, + pub candidate_window_truncated: bool, + pub truncated: bool, +} + +// --------------------------------------------------------------------------- +// AI investigate — evidence bundle layer (kmib.2) +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct AiInvestigateParams { + /// Optional exact incident ID. When present, locate one matching incident + /// within the incident-list cap instead of only the top investigation page. + pub incident_id: Option, + pub ai_project: Option, + pub ai_tool: Option, + pub since: Option, + pub until: Option, + /// Max incidents to investigate. Default 3, clamp 1..=10. + pub limit: Option, + /// Incident grouping window minutes. Default 10, clamp 1..=120. + pub window_minutes: Option, + /// Correlation window minutes around incident. Default 5, clamp 1..=120. + pub correlation_window_minutes: Option, + pub terms: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IncidentEvidence { + pub incident: AbuseIncident, + /// Transcript entries before first anchor (same session), capped at 20. + pub transcript_before: Vec, + pub transcript_before_truncated: bool, + /// Transcript entries after last anchor (same session), capped at 20. + pub transcript_after: Vec, + pub transcript_after_truncated: bool, + /// The abuse anchor log entries. + pub anchors: Vec, + /// Non-AI syslog/Docker logs in the correlation window, capped at 50. + pub nearby_logs: Vec, + pub nearby_logs_truncated: bool, + /// Subset of nearby_logs with severity warning or above. + pub nearby_errors: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiInvestigateResult { + pub evidence: Vec, + pub total_incidents: usize, + pub truncated: bool, +} + +// --------------------------------------------------------------------------- +// RAG v1: similar_incidents, incident_context +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Default)] +pub struct SimilarIncidentsParams { + pub query: String, + pub host: Option, + pub app: Option, + /// Minimum severity (e.g. "warning"). None = all severities. + pub severity_min: Option, + pub since: Option, + pub until: Option, + /// Cluster grouping window in minutes. Default 30, clamp 5..=120. + pub window_minutes: Option, + /// Max clusters to return. Default 10, clamp 1..=50. + pub limit: Option, +} + +/// A time-windowed cluster of log hits (one "incident"). +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IncidentCluster { + pub hostname: String, + pub app_name: Option, + /// RFC 3339 timestamp of the first matching log in this cluster. + pub window_start: String, + /// RFC 3339 timestamp of the last matching log in this cluster. + pub window_end: String, + pub log_count: i64, + /// Highest severity in this cluster (emerg > alert > ... > debug). + pub severity_peak: String, + /// Up to 3 representative message snippets (first 256 chars each). + pub representative_messages: Vec, + /// AI sessions whose transcript entries overlap this cluster's time window. + pub correlated_sessions: Vec, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct CorrelatedSession { + pub session_id: String, + pub project: String, + pub tool: String, + pub match_count: i64, + pub best_snippet: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SimilarIncidentsResult { + pub query: String, + pub total_clusters: usize, + pub truncated: bool, + pub clusters: Vec, +} + +#[derive(Debug, Clone, Default)] +pub struct IncidentContextParams { + pub since: String, + pub until: String, + pub host: Option, + pub app: Option, + /// Optional FTS5 query applied to returned error logs. + pub query: Option, + pub severity_min: Option, + /// Max error log rows to return. Default 50, clamp 1..=200. + pub limit: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct IncidentContextResult { + pub window_from: String, + pub window_to: String, + pub total_logs: i64, + pub by_severity: Vec, + pub by_app: Vec, + /// Logs at or above severity_min (default: warning) within the window. + pub error_logs: Vec, + pub error_logs_truncated: bool, + /// AI sessions active in this window (have transcript entries between from..to). + pub ai_sessions: Vec, +} + +#[cfg(test)] +#[path = "models_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/models_tests.rs b/crates/shared/cortex/storage-sqlite/src/models_tests.rs new file mode 100644 index 00000000..2c5832dc --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/models_tests.rs @@ -0,0 +1,61 @@ +use super::*; + +#[test] +fn log_batch_entry_keeps_claimed_hostname_separate_from_source_ip() { + let entry = LogBatchEntry { + timestamp: "2026-01-01T00:00:00Z".to_string(), + hostname: "claimed-host".to_string(), + facility: Some("local0".to_string()), + severity: "info".to_string(), + app_name: Some("app".to_string()), + process_id: Some("123".to_string()), + message: "message".to_string(), + raw: "raw".to_string(), + source_ip: "192.0.2.10:514".to_string(), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + }; + + assert_eq!(entry.hostname, "claimed-host"); + assert_eq!(entry.source_ip, "192.0.2.10:514"); +} + +#[test] +fn log_batch_entry_has_enrichment_fields() { + let entry = super::LogBatchEntry { + timestamp: String::new(), + hostname: String::new(), + facility: None, + severity: String::new(), + app_name: None, + process_id: None, + message: String::new(), + raw: String::new(), + source_ip: String::new(), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + }; + assert!(entry.http_status.is_none()); + assert!(entry.auth_outcome.is_none()); + assert!(entry.dns_blocked.is_none()); + assert!(entry.event_action.is_none()); + assert!(entry.parse_error.is_none()); +} diff --git a/crates/shared/cortex/storage-sqlite/src/notifications.rs b/crates/shared/cortex/storage-sqlite/src/notifications.rs new file mode 100644 index 00000000..2e58a3f9 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/notifications.rs @@ -0,0 +1,336 @@ +//! Database operations for the notifications subsystem. +//! +//! All functions take a `&rusqlite::Connection` so they can be called either +//! from a plain connection or from inside a `rusqlite::Transaction` +//! (Transaction derefs to Connection). +//! +//! Call from inside `tokio::task::spawn_blocking`, never from async context. + +use rusqlite::params; + +// --------------------------------------------------------------------------- +// Public types (cross-bead coupling export) + +/// Parameters for inserting a row into `notifications_outbox`. +pub struct OutboxInsertParams { + pub dedup_key: String, + pub rule_id: String, + pub severity: String, + pub hostname: String, + pub title: String, + pub body: String, + pub apprise_urls_json: String, + /// ISO8601 datetime for next delivery attempt. + pub next_attempt_at: String, +} + +/// A row fetched from `notifications_outbox`. +#[derive(Debug, Clone)] +pub struct OutboxRow { + pub id: i64, + pub dedup_key: String, + pub rule_id: String, + pub severity: String, + pub hostname: String, + pub title: String, + pub body: String, + pub apprise_urls_json: String, + pub next_attempt_at: String, + pub attempt_count: i64, + pub status: String, +} + +/// A row from `notification_firings`. +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct FiringRow { + pub id: i64, + pub outbox_id: i64, + pub rule_id: String, + pub hostname: String, + pub fired_at: String, + pub status_code: Option, +} + +// --------------------------------------------------------------------------- +// Outbox operations + +/// Insert a row into `notifications_outbox`. +/// +/// Idempotent on `(dedup_key, status='pending')` via the partial unique index +/// `idx_outbox_dedup_pending` (migration 12). Uses `INSERT OR IGNORE` to +/// avoid a TOCTOU race between the SELECT COUNT(*) guard and the INSERT. +pub fn outbox_insert( + conn: &rusqlite::Connection, + params: &OutboxInsertParams, +) -> rusqlite::Result<()> { + conn.execute( + "INSERT OR IGNORE INTO notifications_outbox + (dedup_key, rule_id, severity, hostname, title, body, apprise_urls_json, next_attempt_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8)", + rusqlite::params![ + params.dedup_key, + params.rule_id, + params.severity, + params.hostname, + params.title, + params.body, + params.apprise_urls_json, + params.next_attempt_at, + ], + )?; + Ok(()) +} + +/// Claim up to `limit` pending outbox rows whose `next_attempt_at` is in the past. +pub fn outbox_claim_pending( + conn: &rusqlite::Connection, + limit: i64, +) -> rusqlite::Result> { + let mut stmt = conn.prepare( + "SELECT id, dedup_key, rule_id, severity, hostname, title, body, + apprise_urls_json, next_attempt_at, attempt_count, status + FROM notifications_outbox + WHERE status = 'pending' + AND next_attempt_at <= strftime('%Y-%m-%dT%H:%M:%fZ','now') + ORDER BY next_attempt_at ASC + LIMIT ?1", + )?; + let rows = stmt + .query_map(params![limit], |row| { + Ok(OutboxRow { + id: row.get(0)?, + dedup_key: row.get(1)?, + rule_id: row.get(2)?, + severity: row.get(3)?, + hostname: row.get(4)?, + title: row.get(5)?, + body: row.get(6)?, + apprise_urls_json: row.get(7)?, + next_attempt_at: row.get(8)?, + attempt_count: row.get(9)?, + status: row.get(10)?, + }) + })? + .collect::>>()?; + Ok(rows) +} + +/// Mark a row as sent; increment attempt_count. +pub fn outbox_mark_sent( + conn: &rusqlite::Connection, + id: i64, + status_code: Option, +) -> rusqlite::Result<()> { + conn.execute( + "UPDATE notifications_outbox + SET status = 'sent', + attempt_count = attempt_count + 1, + last_status_code = ?2 + WHERE id = ?1", + params![id, status_code], + )?; + Ok(()) +} + +/// Mark a row as dead (exhausted retries). +pub fn outbox_mark_dead( + conn: &rusqlite::Connection, + id: i64, + status_code: Option, + error: &str, +) -> rusqlite::Result<()> { + conn.execute( + "UPDATE notifications_outbox + SET status = 'dead', + attempt_count = attempt_count + 1, + last_status_code = ?2, + last_error = ?3 + WHERE id = ?1", + params![id, status_code, error], + )?; + Ok(()) +} + +/// Mark a row as dropped (e.g. acked, deduplicated). +pub fn outbox_mark_dropped( + conn: &rusqlite::Connection, + id: i64, + notes: &str, +) -> rusqlite::Result<()> { + conn.execute( + "UPDATE notifications_outbox + SET status = 'dropped', + attempt_count = attempt_count + 1, + last_error = ?2 + WHERE id = ?1", + params![id, notes], + )?; + Ok(()) +} + +/// Set next_attempt_at for exponential backoff retry; increment attempt_count. +pub fn outbox_schedule_retry( + conn: &rusqlite::Connection, + id: i64, + next_attempt_at: &str, + last_error: &str, + status_code: Option, +) -> rusqlite::Result<()> { + conn.execute( + "UPDATE notifications_outbox + SET attempt_count = attempt_count + 1, + next_attempt_at = ?2, + last_error = ?3, + last_status_code = ?4 + WHERE id = ?1", + params![id, next_attempt_at, last_error, status_code], + )?; + Ok(()) +} + +// --------------------------------------------------------------------------- +// Firings + +/// Parameters for inserting a row into `notification_firings`. +pub struct FiringInsertParams<'a> { + pub outbox_id: i64, + pub rule_id: &'a str, + pub severity: &'a str, + pub hostname: &'a str, + pub status_code: Option, + pub notes: Option<&'a str>, + /// Mirrors the outbox row's dedup_key so that dedup checks are scoped to + /// a specific error signature rather than all firings for (rule_id, hostname). + pub dedup_key: &'a str, +} + +/// Insert a row into `notification_firings`. +pub fn firings_insert( + conn: &rusqlite::Connection, + p: FiringInsertParams<'_>, +) -> rusqlite::Result<()> { + conn.execute( + "INSERT INTO notification_firings + (outbox_id, rule_id, severity, hostname, status_code, notes, dedup_key) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7)", + params![ + p.outbox_id, + p.rule_id, + p.severity, + p.hostname, + p.status_code, + p.notes, + p.dedup_key + ], + )?; + Ok(()) +} + +/// Check if there is a recent firing for the given rule+hostname+dedup_key +/// within the dedup window (seconds). Returns true if a firing already exists +/// (suppress). +/// +/// The `dedup_key` parameter is essential for rules that share a `rule_id` +/// (e.g. `unaddressed_error_signature` fires once per distinct error hash). +/// Without it, the first firing would suppress all subsequent ones regardless +/// of which signature they belong to. +pub fn firings_recent_dedup_check( + conn: &rusqlite::Connection, + rule_id: &str, + hostname: &str, + dedup_key: &str, + dedup_window_secs: u64, +) -> rusqlite::Result { + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM notification_firings + WHERE rule_id = ?1 + AND hostname = ?2 + AND dedup_key = ?3 + AND fired_at >= strftime('%Y-%m-%dT%H:%M:%fZ', 'now', printf('-%d seconds', ?4))", + params![rule_id, hostname, dedup_key, dedup_window_secs as i64], + |row| row.get(0), + )?; + Ok(count > 0) +} + +/// Check whether a firing has ever been recorded for the exact +/// rule+hostname+dedup_key tuple. +/// +/// This is used by once-per-outage rules whose dedup key includes the +/// observation timestamp that identifies the outage. A new observation gets +/// a new key; an unchanged outage remains suppressed regardless of age. +pub fn firings_any_dedup_check( + conn: &rusqlite::Connection, + rule_id: &str, + hostname: &str, + dedup_key: &str, +) -> rusqlite::Result { + conn.query_row( + "SELECT EXISTS( + SELECT 1 FROM notification_firings + WHERE rule_id = ?1 + AND hostname = ?2 + AND dedup_key = ?3 + )", + params![rule_id, hostname, dedup_key], + |row| row.get(0), + ) +} + +/// Fetch recent firings for a given rule_id (optional) since a given time. +pub fn firings_recent( + conn: &rusqlite::Connection, + limit: i64, + rule_id: Option<&str>, + since: Option<&str>, +) -> rusqlite::Result> { + let clamped_limit = limit.clamp(1, 500); + let mut stmt = conn.prepare( + "SELECT id, outbox_id, rule_id, hostname, fired_at, status_code + FROM notification_firings + WHERE (?1 IS NULL OR rule_id = ?1) + AND (?2 IS NULL OR fired_at >= ?2) + ORDER BY fired_at DESC + LIMIT ?3", + )?; + let rows = stmt + .query_map(params![rule_id, since, clamped_limit], |row| { + Ok(FiringRow { + id: row.get(0)?, + outbox_id: row.get(1)?, + rule_id: row.get(2)?, + hostname: row.get(3)?, + fired_at: row.get(4)?, + status_code: row.get(5)?, + }) + })? + .collect::>>()?; + Ok(rows) +} + +// --------------------------------------------------------------------------- +// Backoff helper + +/// Compute `next_attempt_at` as an ISO8601 string given `attempt_count`. +/// +/// Backoff schedule (capped at 30 minutes): +/// attempt 0 → now+1s +/// attempt 1 → now+5s +/// attempt 2 → now+30s +/// attempt 3 → now+5min +/// attempt 4+ → now+30min +pub fn backoff_next_attempt_at(attempt_count: u8) -> String { + let delay_secs: u64 = match attempt_count { + 0 => 1, + 1 => 5, + 2 => 30, + 3 => 300, + _ => 1800, + }; + let next = chrono::Utc::now() + chrono::Duration::seconds(delay_secs as i64); + next.format("%Y-%m-%dT%H:%M:%S%.3fZ").to_string() +} + +#[cfg(test)] +#[path = "notifications_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/notifications_tests.rs b/crates/shared/cortex/storage-sqlite/src/notifications_tests.rs new file mode 100644 index 00000000..8c6bfd02 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/notifications_tests.rs @@ -0,0 +1,314 @@ +#[cfg(test)] +mod notifications_db_tests { + use rusqlite::Connection; + + use crate::notifications::{ + FiringInsertParams, OutboxInsertParams, backoff_next_attempt_at, firings_insert, + firings_recent, firings_recent_dedup_check, outbox_claim_pending, outbox_insert, + outbox_mark_dead, outbox_mark_dropped, outbox_mark_sent, outbox_schedule_retry, + }; + + fn in_memory_conn() -> Connection { + let conn = Connection::open_in_memory().expect("in-memory db"); + conn.execute_batch( + "CREATE TABLE notifications_outbox ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + dedup_key TEXT NOT NULL, + rule_id TEXT NOT NULL, + severity TEXT NOT NULL, + hostname TEXT NOT NULL, + title TEXT NOT NULL, + body TEXT NOT NULL, + apprise_urls_json TEXT NOT NULL, + apprise_tags TEXT, + enqueued_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + next_attempt_at TEXT NOT NULL, + attempt_count INTEGER NOT NULL DEFAULT 0, + last_status_code INTEGER, + last_error TEXT, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending','sent','dead','dropped')) + ); + CREATE UNIQUE INDEX IF NOT EXISTS idx_outbox_dedup_pending + ON notifications_outbox(dedup_key) WHERE status = 'pending'; + CREATE TABLE notification_firings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + outbox_id INTEGER NOT NULL, + rule_id TEXT NOT NULL, + severity TEXT NOT NULL, + hostname TEXT NOT NULL, + fired_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + status_code INTEGER, + notes TEXT, + dedup_key TEXT NOT NULL DEFAULT '' + );", + ) + .expect("schema"); + conn + } + + fn make_params(dedup_key: &str) -> OutboxInsertParams { + OutboxInsertParams { + dedup_key: dedup_key.to_string(), + rule_id: "oom_kill".to_string(), + severity: "critical".to_string(), + hostname: "host1".to_string(), + title: "OOM Kill on host1".to_string(), + body: "Process was killed".to_string(), + apprise_urls_json: r#"["gotify://host/token"]"#.to_string(), + next_attempt_at: "2030-01-01T00:00:00.000Z".to_string(), + } + } + + #[test] + fn outbox_insert_idempotent() { + let conn = in_memory_conn(); + let params = make_params("dedup-1"); + + // First insert should succeed + outbox_insert(&conn, ¶ms).expect("first insert"); + + // Second insert with same dedup_key should be skipped (idempotent) + outbox_insert(&conn, ¶ms).expect("second insert (no-op)"); + + let count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM notifications_outbox WHERE dedup_key = ?1", + rusqlite::params!["dedup-1"], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(count, 1, "duplicate dedup_key should be suppressed"); + } + + #[test] + fn outbox_insert_different_keys() { + let conn = in_memory_conn(); + outbox_insert(&conn, &make_params("key-a")).expect("insert a"); + outbox_insert(&conn, &make_params("key-b")).expect("insert b"); + let count: i64 = conn + .query_row("SELECT COUNT(*) FROM notifications_outbox", [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!(count, 2); + } + + #[test] + fn outbox_claim_pending_basic() { + let conn = in_memory_conn(); + outbox_insert(&conn, &make_params("key-c")).expect("insert"); + + // Override next_attempt_at to past + conn.execute( + "UPDATE notifications_outbox SET next_attempt_at = '2000-01-01T00:00:00.000Z'", + [], + ) + .unwrap(); + + let rows = outbox_claim_pending(&conn, 10).expect("claim"); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].rule_id, "oom_kill"); + } + + #[test] + fn outbox_mark_sent_and_dead() { + let conn = in_memory_conn(); + outbox_insert(&conn, &make_params("key-d")).expect("insert"); + conn.execute( + "UPDATE notifications_outbox SET next_attempt_at = '2000-01-01T00:00:00.000Z'", + [], + ) + .unwrap(); + + let rows = outbox_claim_pending(&conn, 10).expect("claim"); + let id = rows[0].id; + + outbox_mark_sent(&conn, id, Some(200)).expect("mark sent"); + + let status: String = conn + .query_row( + "SELECT status FROM notifications_outbox WHERE id = ?1", + rusqlite::params![id], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(status, "sent"); + } + + #[test] + fn outbox_mark_dropped_test() { + let conn = in_memory_conn(); + outbox_insert(&conn, &make_params("key-e")).expect("insert"); + let id: i64 = conn + .query_row("SELECT id FROM notifications_outbox LIMIT 1", [], |r| { + r.get(0) + }) + .unwrap(); + outbox_mark_dropped(&conn, id, "acked").expect("mark dropped"); + + let status: String = conn + .query_row( + "SELECT status FROM notifications_outbox WHERE id = ?1", + rusqlite::params![id], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(status, "dropped"); + } + + #[test] + fn outbox_schedule_retry_test() { + let conn = in_memory_conn(); + outbox_insert(&conn, &make_params("key-f")).expect("insert"); + let id: i64 = conn + .query_row("SELECT id FROM notifications_outbox LIMIT 1", [], |r| { + r.get(0) + }) + .unwrap(); + + outbox_schedule_retry(&conn, id, "2030-06-01T00:00:00.000Z", "timeout", Some(503)) + .expect("retry"); + + let (attempt_count, last_error): (i64, String) = conn + .query_row( + "SELECT attempt_count, last_error FROM notifications_outbox WHERE id = ?1", + rusqlite::params![id], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + assert_eq!(attempt_count, 1); + assert_eq!(last_error, "timeout"); + } + + #[test] + fn outbox_mark_dead_test() { + let conn = in_memory_conn(); + outbox_insert(&conn, &make_params("key-g")).expect("insert"); + let id: i64 = conn + .query_row("SELECT id FROM notifications_outbox LIMIT 1", [], |r| { + r.get(0) + }) + .unwrap(); + outbox_mark_dead(&conn, id, Some(500), "server error").expect("mark dead"); + + let status: String = conn + .query_row( + "SELECT status FROM notifications_outbox WHERE id = ?1", + rusqlite::params![id], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(status, "dead"); + } + + #[test] + fn firings_insert_and_dedup_check() { + let conn = in_memory_conn(); + outbox_insert(&conn, &make_params("key-h")).expect("insert"); + let id: i64 = conn + .query_row("SELECT id FROM notifications_outbox LIMIT 1", [], |r| { + r.get(0) + }) + .unwrap(); + + firings_insert( + &conn, + FiringInsertParams { + outbox_id: id, + rule_id: "oom_kill", + severity: "critical", + hostname: "host1", + status_code: Some(200), + notes: None, + dedup_key: "oom_kill:host1:key-h", + }, + ) + .expect("firings insert"); + + // Within window, same dedup_key -> should dedup + let should_dedup = + firings_recent_dedup_check(&conn, "oom_kill", "host1", "oom_kill:host1:key-h", 3600) + .expect("dedup check"); + assert!(should_dedup, "should suppress within dedup window"); + + // Different hostname -> no dedup + let no_dedup = + firings_recent_dedup_check(&conn, "oom_kill", "host2", "oom_kill:host1:key-h", 3600) + .expect("dedup check 2"); + assert!(!no_dedup, "different host should not dedup"); + + // Different dedup_key -> no dedup (this is the key fix: per-signature isolation) + let no_dedup_dk = firings_recent_dedup_check( + &conn, + "oom_kill", + "host1", + "oom_kill:host1:other-key", + 3600, + ) + .expect("dedup check 3"); + assert!(!no_dedup_dk, "different dedup_key should not dedup"); + } + + #[test] + fn firings_recent_list() { + let conn = in_memory_conn(); + outbox_insert(&conn, &make_params("key-i")).expect("insert"); + let id: i64 = conn + .query_row("SELECT id FROM notifications_outbox LIMIT 1", [], |r| { + r.get(0) + }) + .unwrap(); + firings_insert( + &conn, + FiringInsertParams { + outbox_id: id, + rule_id: "oom_kill", + severity: "critical", + hostname: "host1", + status_code: Some(200), + notes: None, + dedup_key: "key-oom", + }, + ) + .unwrap(); + firings_insert( + &conn, + FiringInsertParams { + outbox_id: id, + rule_id: "fail2ban_ban", + severity: "notice", + hostname: "host2", + status_code: Some(200), + notes: None, + dedup_key: "key-fail2ban", + }, + ) + .unwrap(); + + let all = firings_recent(&conn, 10, None, None).expect("all firings"); + assert_eq!(all.len(), 2); + + let filtered = firings_recent(&conn, 10, Some("oom_kill"), None).expect("filtered"); + assert_eq!(filtered.len(), 1); + assert_eq!(filtered[0].rule_id, "oom_kill"); + } + + #[test] + fn backoff_delays_are_increasing() { + // Verify the specific backoff schedule: 1s, 5s, 30s, 5min, 30min cap. + let expected_secs: &[u64] = &[1, 5, 30, 300, 1800, 1800, 1800, 1800]; + let now = chrono::Utc::now(); + for (i, &expected) in expected_secs.iter().enumerate() { + let s = backoff_next_attempt_at(i as u8); + let parsed = chrono::DateTime::parse_from_rfc3339(&s) + .unwrap_or_else(|_| panic!("attempt {i}: invalid ISO8601: {s}")); + let actual_delay = (parsed.with_timezone(&chrono::Utc) - now).num_seconds(); + // Allow +/-2s tolerance for test execution time. + assert!( + (actual_delay - expected as i64).abs() <= 2, + "attempt {i}: expected ~{expected}s delay, got {actual_delay}s" + ); + } + } +} diff --git a/crates/shared/cortex/storage-sqlite/src/otlp_metrics.rs b/crates/shared/cortex/storage-sqlite/src/otlp_metrics.rs new file mode 100644 index 00000000..3158b2aa --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/otlp_metrics.rs @@ -0,0 +1,100 @@ +use super::agent_observatory::EnumParseError; +use serde::{Deserialize, Deserializer, Serialize, Serializer}; +use std::fmt; +use std::str::FromStr; + +#[derive(Debug, Clone, Copy, PartialEq, Eq, Hash)] +pub enum MetricInstrumentKind { + Gauge, + Sum, + Histogram, + ExponentialHistogram, + Summary, +} + +impl MetricInstrumentKind { + pub const ALL: &'static [Self] = &[ + Self::Gauge, + Self::Sum, + Self::Histogram, + Self::ExponentialHistogram, + Self::Summary, + ]; + pub const fn as_str(self) -> &'static str { + match self { + Self::Gauge => "gauge", + Self::Sum => "sum", + Self::Histogram => "histogram", + Self::ExponentialHistogram => "exponential_histogram", + Self::Summary => "summary", + } + } +} +impl fmt::Display for MetricInstrumentKind { + fn fmt(&self, formatter: &mut fmt::Formatter<'_>) -> fmt::Result { + formatter.write_str(self.as_str()) + } +} +impl FromStr for MetricInstrumentKind { + type Err = EnumParseError; + fn from_str(value: &str) -> Result { + match value { + "gauge" => Ok(Self::Gauge), + "sum" => Ok(Self::Sum), + "histogram" => Ok(Self::Histogram), + "exponential_histogram" => Ok(Self::ExponentialHistogram), + "summary" => Ok(Self::Summary), + _ => Err(EnumParseError::new("MetricInstrumentKind", value)), + } + } +} +impl Serialize for MetricInstrumentKind { + fn serialize(&self, serializer: S) -> Result + where + S: Serializer, + { + serializer.serialize_str(self.as_str()) + } +} +impl<'de> Deserialize<'de> for MetricInstrumentKind { + fn deserialize(deserializer: D) -> Result + where + D: Deserializer<'de>, + { + let value = String::deserialize(deserializer)?; + value.parse().map_err(serde::de::Error::custom) + } +} + +#[derive(Debug, Clone, PartialEq, Serialize, Deserialize)] +pub struct OtelMetricPointRow { + pub id: i64, + pub point_key: String, + pub metric_name: String, + pub description: String, + pub unit: String, + pub instrument_kind: MetricInstrumentKind, + pub aggregation_temporality: Option, + pub monotonic: Option, + pub start_time_unix_nano: Option, + pub time_unix_nano: i64, + pub hostname: String, + pub service_name: Option, + pub service_version: Option, + pub scope_name: Option, + pub scope_version: Option, + pub ai_tool: Option, + pub ai_project: Option, + pub ai_session_id: Option, + pub run_id: Option, + pub resource_json: String, + pub attributes_json: String, + pub value_json: String, + pub exemplars_json: String, + pub received_at: String, + pub content_scrubbed: bool, +} + +#[cfg(test)] +#[path = "otlp_metrics_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/otlp_metrics_tests.rs b/crates/shared/cortex/storage-sqlite/src/otlp_metrics_tests.rs new file mode 100644 index 00000000..a44743c1 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/otlp_metrics_tests.rs @@ -0,0 +1,12 @@ +use super::*; + +#[test] +fn instrument_kind_wire_values_round_trip() { + for kind in MetricInstrumentKind::ALL { + let json = serde_json::to_string(kind).unwrap(); + let decoded: MetricInstrumentKind = serde_json::from_str(&json).unwrap(); + assert_eq!(*kind, decoded); + assert_eq!(kind.to_string(), kind.as_str()); + } + assert!("bogus".parse::().is_err()); +} diff --git a/crates/shared/cortex/storage-sqlite/src/otlp_traces.rs b/crates/shared/cortex/storage-sqlite/src/otlp_traces.rs new file mode 100644 index 00000000..3867ff49 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/otlp_traces.rs @@ -0,0 +1,37 @@ +use serde::{Deserialize, Serialize}; + +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +pub struct OtelSpanRow { + pub id: i64, + pub trace_id: String, + pub span_id: String, + pub parent_span_id: Option, + pub trace_state: Option, + pub flags: i64, + pub span_name: String, + pub span_kind: i64, + pub start_time_unix_nano: i64, + pub end_time_unix_nano: i64, + pub duration_nano: i64, + pub status_code: i64, + pub status_message: Option, + pub hostname: String, + pub service_name: Option, + pub service_version: Option, + pub scope_name: Option, + pub scope_version: Option, + pub ai_tool: Option, + pub ai_project: Option, + pub ai_session_id: Option, + pub run_id: Option, + pub resource_json: String, + pub attributes_json: String, + pub events_json: String, + pub links_json: String, + pub received_at: String, + pub content_scrubbed: bool, +} + +#[cfg(test)] +#[path = "otlp_traces_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/otlp_traces_tests.rs b/crates/shared/cortex/storage-sqlite/src/otlp_traces_tests.rs new file mode 100644 index 00000000..be806e42 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/otlp_traces_tests.rs @@ -0,0 +1,37 @@ +use super::*; + +#[test] +fn span_row_json_round_trip_preserves_trace_identity() { + let row = OtelSpanRow { + id: 1, + trace_id: "trace".into(), + span_id: "span".into(), + parent_span_id: Some("parent".into()), + trace_state: None, + flags: 1, + span_name: "request".into(), + span_kind: 2, + start_time_unix_nano: 10, + end_time_unix_nano: 20, + duration_nano: 10, + status_code: 1, + status_message: None, + hostname: "dookie".into(), + service_name: Some("soma".into()), + service_version: None, + scope_name: None, + scope_version: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + run_id: None, + resource_json: "{}".into(), + attributes_json: "{}".into(), + events_json: "[]".into(), + links_json: "[]".into(), + received_at: "2026-08-18T00:00:00Z".into(), + content_scrubbed: true, + }; + let decoded: OtelSpanRow = serde_json::from_str(&serde_json::to_string(&row).unwrap()).unwrap(); + assert_eq!(decoded, row); +} diff --git a/crates/shared/cortex/storage-sqlite/src/pool.rs b/crates/shared/cortex/storage-sqlite/src/pool.rs new file mode 100644 index 00000000..4fca5867 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/pool.rs @@ -0,0 +1,3643 @@ +//! SQLite pool construction, schema, and migrations for the log intelligence +//! core. +//! +//! Owns the full schema: the `logs` table + FTS5 index, AI/graph/heartbeat +//! projections, and the **44 sequential migrations** tracked by +//! `KNOWN_SCHEMA_VERSION`. Migrations run at startup; heavy ones log +//! `Migration N: starting ...` lines, and the one-time +//! `auto_vacuum=INCREMENTAL` conversion VACUUM is logged loudly (it can take +//! minutes on large DBs). +//! +//! Invariants: SQLite allows a single writer — callers serialize writes via +//! `write_lock()`, and the service layer issues only `pool_size - 1` read +//! permits so the ingest batch writer can always reach a connection. WAL +//! mode plus `synchronous=NORMAL` is the standing durability trade-off. + +use anyhow::Result; +use r2d2::Pool; +use r2d2_sqlite::SqliteConnectionManager; +use rusqlite::Connection; +use scheduled_thread_pool::ScheduledThreadPool; +use std::sync::{Arc, OnceLock}; + +use crate::config::StorageConfig; + +pub type DbPool = Pool; + +/// Process-wide SQLite **write serialization** lock. +/// +/// SQLite permits only one writer at a time, but cortex runs an r2d2 pool of several +/// connections with multiple concurrent writer subsystems (syslog/docker ingest, +/// heartbeat, notifications, AI index, retention maintenance). Without serialization +/// these race SQLite's single write lock, exceed `busy_timeout`, and surface as +/// `database is locked` — dropping log batches. Every write transaction acquires this +/// guard so writers queue in-process instead of colliding at the SQLite layer; reads +/// stay concurrent on the pool (WAL allows many readers). Reentrant so a write path that +/// nests guarded helpers on a single thread cannot deadlock. +pub fn write_lock() -> parking_lot::ReentrantMutexGuard<'static, ()> { + static WRITE_LOCK: parking_lot::ReentrantMutex<()> = parking_lot::ReentrantMutex::new(()); + WRITE_LOCK.lock() +} + +pub const KNOWN_SCHEMA_VERSION: i64 = 47; + +#[derive(Debug, Clone, serde::Serialize, serde::Deserialize)] +pub struct SchemaVersionInfo { + pub version: i64, + pub last_migration_at: Option, + pub known_version: i64, +} + +/// Process-wide r2d2 background thread pool, shared across every `DbPool` +/// this process creates. +/// +/// In production a process holds exactly one `DbPool`, so a single thread +/// would suffice. But `cargo test` runs in one process and each test that +/// calls `init_pool()` creates its own independent `DbPool` sharing this same +/// static pool — under full test-suite parallelism, dozens of pools' worth +/// of background connection work queues behind a single thread, exceeding +/// the 6s `connection_timeout` and surfacing as spurious "timed out waiting +/// for connection" failures unrelated to any actual bug under test. Sized +/// with headroom for concurrent test execution, not just single-process +/// production use. +fn shared_scheduled_thread_pool() -> Arc { + static POOL: OnceLock> = OnceLock::new(); + Arc::clone(POOL.get_or_init(|| Arc::new(ScheduledThreadPool::new(8)))) +} + +pub fn read_schema_version_info(pool: &DbPool) -> Result { + let conn = pool.get()?; + read_schema_version_info_conn(&conn) +} + +/// Probe `schema_migrations` from an already-borrowed connection. Used by +/// callers that do not own a [`DbPool`] (e.g. the scanner's checkpoint store). +pub fn read_schema_version_info_conn(conn: &Connection) -> Result { + let (version, last_migration_at): (Option, Option) = conn + .query_row( + "SELECT MAX(version), MAX(applied_at) FROM schema_migrations", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .map_err(|err| anyhow::anyhow!("schema_migrations probe failed: {err}"))?; + Ok(SchemaVersionInfo { + version: version.unwrap_or(0), + last_migration_at, + known_version: KNOWN_SCHEMA_VERSION, + }) +} + +/// Initialize the database pool and schema +pub fn init_pool(config: &StorageConfig) -> Result { + // Ensure parent directory exists + if let Some(parent) = config.db_path.parent() { + std::fs::create_dir_all(parent)?; + } + + let storage = config.clone(); + let manager = SqliteConnectionManager::file(&config.db_path) + .with_init(move |conn| configure_connection_pragmas(conn, &storage)); + // connection_timeout is set to 6s — slightly above the service layer's 5s + // DB_ACQUIRE_TIMEOUT so the semaphore fires first, giving a clean ServiceError::Busy + // rather than an r2d2 timeout on the rare path where background tasks exhaust the pool. + let pool = Pool::builder() + .max_size(config.pool_size) + .connection_timeout(std::time::Duration::from_secs(6)) + .thread_pool(shared_scheduled_thread_pool()) + .build(manager)?; + + // Initialize schema. `mut` so migration 25's backfill can open an explicit + // transaction (`Connection::transaction_with_behavior` needs `&mut`). + let mut conn = pool.get()?; + + let auto_vacuum_mode: i64 = conn.query_row("PRAGMA auto_vacuum", [], |r| r.get(0))?; + if auto_vacuum_mode != 2 { + conn.execute_batch("PRAGMA auto_vacuum=INCREMENTAL;")?; + let page_count: i64 = conn.query_row("PRAGMA page_count", [], |r| r.get(0))?; + if page_count > 0 { + // One-time conversion: a full VACUUM rewrites the whole file with + // the write lock held — minutes on a multi-GB DB. Log loudly so a + // long first boot after this policy change is explainable and the + // compose healthcheck start_period can be tuned (full-review PM7). + let page_size: i64 = conn.query_row("PRAGMA page_size", [], |r| r.get(0))?; + let db_mb = (page_count * page_size) / (1024 * 1024); + tracing::info!( + db_size_mb = db_mb, + "Converting database to auto_vacuum=INCREMENTAL — one-time full \ + VACUUM; this can take minutes on large databases" + ); + let vacuum_started = std::time::Instant::now(); + conn.execute_batch("VACUUM;")?; + tracing::info!( + db_size_mb = db_mb, + elapsed_ms = vacuum_started.elapsed().as_millis() as u64, + "auto_vacuum conversion VACUUM complete" + ); + } + } + + conn.execute_batch( + " + CREATE TABLE IF NOT EXISTS logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + hostname TEXT NOT NULL, + facility TEXT, + severity TEXT NOT NULL, + app_name TEXT, + process_id TEXT, + message TEXT NOT NULL, + raw TEXT NOT NULL, + received_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + source_ip TEXT NOT NULL DEFAULT '', + ai_tool TEXT, + ai_project TEXT, + ai_session_id TEXT, + ai_transcript_path TEXT, + metadata_json TEXT + ); + + CREATE INDEX IF NOT EXISTS idx_logs_timestamp ON logs(timestamp); + CREATE INDEX IF NOT EXISTS idx_logs_hostname ON logs(hostname); + CREATE INDEX IF NOT EXISTS idx_logs_severity ON logs(severity); + CREATE INDEX IF NOT EXISTS idx_logs_app_name ON logs(app_name); + CREATE INDEX IF NOT EXISTS idx_logs_host_time ON logs(hostname, timestamp); + CREATE INDEX IF NOT EXISTS idx_logs_sev_time ON logs(severity, timestamp); + CREATE INDEX IF NOT EXISTS idx_logs_app_name_timestamp ON logs(app_name, timestamp); + CREATE INDEX IF NOT EXISTS idx_logs_received_at ON logs(received_at); + CREATE INDEX IF NOT EXISTS idx_logs_hostname_received_at ON logs(hostname, received_at); + CREATE INDEX IF NOT EXISTS idx_logs_source_ip_timestamp ON logs(source_ip, timestamp); + DROP INDEX IF EXISTS idx_logs_source_ip; + + -- FTS5 virtual table for full-text search on messages + CREATE VIRTUAL TABLE IF NOT EXISTS logs_fts USING fts5( + message, + content='logs', + content_rowid='id', + tokenize='porter unicode61' + ); + + -- Trigger to keep FTS in sync on INSERT only. + -- DELETE and UPDATE triggers are intentionally absent: bulk DELETEs during + -- retention purge and storage-budget enforcement fire the trigger for every + -- deleted row inside a single implicit transaction, holding the SQLite write + -- lock long enough to starve the batch writer. FTS5 content tables tolerate + -- phantom rows — stale entries are skipped at query time and cleaned up by + -- periodic incremental merge (merge=500,250). + CREATE TRIGGER IF NOT EXISTS logs_ai AFTER INSERT ON logs BEGIN + INSERT INTO logs_fts(rowid, message) VALUES (new.id, new.message); + END; + + -- Hostname registry for quick lookups + CREATE TABLE IF NOT EXISTS hosts ( + hostname TEXT PRIMARY KEY, + first_seen TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + last_seen TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + log_count INTEGER NOT NULL DEFAULT 0 + ); + + -- Migration version table: each row records a completed schema migration. + -- Guards migrations so they run exactly once per database, not on every startup. + CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) + ); + ", + )?; + + // Migration: add source_ip column to existing databases that predate this column. + // ALTER TABLE ADD COLUMN is a no-op if the column already exists in SQLite ≥ 3.37, + // but older SQLite returns an error on duplicate columns, so we check first. + let col_exists: bool = conn + .query_row( + "SELECT COUNT(*) FROM pragma_table_info('logs') WHERE name = 'source_ip'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap_or(0) + > 0; + if !col_exists { + conn.execute_batch("ALTER TABLE logs ADD COLUMN source_ip TEXT NOT NULL DEFAULT ''")?; + tracing::info!("Migration: added source_ip column to logs table"); + } + + // Migration 1: drop FTS5 DELETE/UPDATE triggers from existing databases. + // These triggers caused write-lock contention during bulk deletes (retention + // purge, storage enforcement). See schema comment above for rationale. + // Guarded by schema_migrations so it runs exactly once per database. + let migration_1_applied: bool = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 1", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap_or(0) + > 0; + if !migration_1_applied { + conn.execute_batch( + "DROP TRIGGER IF EXISTS logs_ad; + DROP TRIGGER IF EXISTS logs_au; + INSERT INTO schema_migrations (version) VALUES (1);", + )?; + tracing::info!("Migration 1: dropped FTS5 DELETE/UPDATE triggers"); + } + + // Migration 2: store per Docker host/container checkpoints for optional + // docker-socket-proxy log ingestion. This lets short cortex outages + // replay from Docker's local log store with /containers/{id}/logs?since=. + let migration_2_applied: bool = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 2", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap_or(0) + > 0; + if !migration_2_applied { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS docker_ingest_checkpoints ( + host_name TEXT NOT NULL, + container_id TEXT NOT NULL, + last_timestamp TEXT NOT NULL, + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + PRIMARY KEY (host_name, container_id) + ); + INSERT INTO schema_migrations (version) VALUES (2);", + )?; + tracing::info!("Migration 2: created docker_ingest_checkpoints table"); + } + + // Migration 3: composite index on (app_name, received_at). + // + // The new `purge_by_tag_window` function deletes rows by `app_name` within + // a `received_at` window (e.g. all `adguard-allowed` older than 7 days). + // Without this composite index, each chunked DELETE scans the entire + // app_name partition before applying the time filter — pathological at + // AdGuard volumes. + // + // First-run cost: on a multi-million-row database the CREATE INDEX may + // take several minutes and holds the write lock for that duration. The + // /health endpoint will not respond and syslog UDP packets may be dropped + // at the kernel buffer during that window. Operators upgrading on a + // populated DB should plan for a brief health-check gap. + let migration_3_applied: bool = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 3", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap_or(0) + > 0; + if !migration_3_applied { + tracing::info!( + "Migration 3: starting CREATE INDEX idx_logs_app_name_received_at \ + — may take several minutes on large databases, write lock held" + ); + let started = std::time::Instant::now(); + conn.execute_batch( + "CREATE INDEX IF NOT EXISTS idx_logs_app_name_received_at \ + ON logs(app_name, received_at); + INSERT INTO schema_migrations (version) VALUES (3);", + )?; + tracing::info!( + elapsed_ms = started.elapsed().as_millis(), + "Migration 3: composite index (app_name, received_at) created" + ); + } + + // Migration 4: add AI transcript metadata columns and indexes. + let migration_4_applied: bool = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 4", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap_or(0) + > 0; + if !migration_4_applied { + for (column, sql_type) in [ + ("ai_tool", "TEXT"), + ("ai_project", "TEXT"), + ("ai_session_id", "TEXT"), + ("ai_transcript_path", "TEXT"), + ] { + let exists: bool = conn + .query_row( + "SELECT COUNT(*) FROM pragma_table_info('logs') WHERE name = ?1", + [column], + |row| row.get::<_, i64>(0), + ) + .unwrap_or(0) + > 0; + if !exists { + conn.execute_batch(&format!("ALTER TABLE logs ADD COLUMN {column} {sql_type}"))?; + } + } + conn.execute_batch( + "CREATE INDEX IF NOT EXISTS idx_logs_ai_project_time + ON logs(ai_project, timestamp) + WHERE ai_project IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_logs_ai_session + ON logs(ai_tool, ai_project, ai_session_id) + WHERE ai_tool IS NOT NULL; + INSERT INTO schema_migrations (version) VALUES (4);", + )?; + tracing::info!("Migration 4: added AI transcript metadata columns and indexes"); + } + + let migration_5_applied: bool = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 5", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap_or(0) + > 0; + if !migration_5_applied { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS transcript_sources ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + canonical_path TEXT NOT NULL UNIQUE, + source_kind TEXT NOT NULL, + file_size INTEGER, + file_mtime INTEGER, + content_hash TEXT, + last_offset INTEGER NOT NULL DEFAULT 0, + last_indexed_at TEXT, + last_error TEXT + ); + INSERT INTO schema_migrations (version) VALUES (5);", + )?; + tracing::info!("Migration 5: created transcript_sources table"); + } + + let migration_6_applied: bool = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 6", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap_or(0) + > 0; + if !migration_6_applied { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS transcript_import_records ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id INTEGER NOT NULL REFERENCES transcript_sources(id), + record_key TEXT NOT NULL, + imported_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(source_id, record_key) + ); + CREATE INDEX IF NOT EXISTS idx_transcript_import_records_source_id + ON transcript_import_records(source_id); + INSERT INTO schema_migrations (version) VALUES (6);", + )?; + tracing::info!("Migration 6: created transcript_import_records table"); + } + + let migration_7_applied: bool = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 7", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap_or(0) + > 0; + if !migration_7_applied { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS transcript_parse_errors ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + source_id INTEGER NOT NULL REFERENCES transcript_sources(id), + line_no INTEGER NOT NULL, + error TEXT NOT NULL, + record_preview TEXT NOT NULL, + seen_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(source_id, line_no, error, record_preview) + ); + CREATE INDEX IF NOT EXISTS idx_transcript_parse_errors_source_seen + ON transcript_parse_errors(source_id, seen_at DESC); + CREATE INDEX IF NOT EXISTS idx_transcript_parse_errors_seen + ON transcript_parse_errors(seen_at DESC); + INSERT INTO schema_migrations (version) VALUES (7);", + )?; + tracing::info!("Migration 7: created transcript_parse_errors table"); + } + + let migration_8_applied: bool = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 8", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap_or(0) + > 0; + if !migration_8_applied { + conn.execute_batch( + "DROP INDEX IF EXISTS idx_logs_ai_project_time; + DROP INDEX IF EXISTS idx_logs_ai_session; + CREATE INDEX IF NOT EXISTS idx_logs_ai_project_time + ON logs(ai_project, timestamp) + WHERE ai_project IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_logs_ai_session + ON logs(ai_tool, ai_project, ai_session_id) + WHERE ai_tool IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_logs_ai_transcript_path + ON logs(ai_transcript_path) + WHERE ai_transcript_path IS NOT NULL; + INSERT INTO schema_migrations (version) VALUES (8);", + )?; + tracing::info!("Migration 8: rebuilt AI metadata indexes as partial indexes"); + } + + let migration_9_applied: bool = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 9", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap_or(0) + > 0; + if !migration_9_applied { + let metadata_col_exists: bool = conn + .query_row( + "SELECT COUNT(*) FROM pragma_table_info('logs') WHERE name = 'metadata_json'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap_or(0) + > 0; + if !metadata_col_exists { + conn.execute_batch("ALTER TABLE logs ADD COLUMN metadata_json TEXT")?; + } + conn.execute_batch("INSERT INTO schema_migrations (version) VALUES (9);")?; + tracing::info!("Migration 9: added logs.metadata_json"); + } + + // Migration 10: error signature detection tables. + let migration_10_applied: bool = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 10", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap_or(0) + > 0; + if !migration_10_applied { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS error_signatures ( + signature_hash TEXT NOT NULL, + normalizer_version INTEGER NOT NULL, + template TEXT NOT NULL, + sample_message TEXT NOT NULL, + sample_hostname TEXT NOT NULL, + sample_app_name TEXT, + severity TEXT NOT NULL, + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + total_count INTEGER NOT NULL DEFAULT 0, + acknowledged_at TEXT, + acknowledged_by TEXT, + PRIMARY KEY (signature_hash, normalizer_version) + ); + CREATE INDEX IF NOT EXISTS idx_error_sigs_last_seen + ON error_signatures(last_seen_at DESC); + CREATE INDEX IF NOT EXISTS idx_error_sigs_ack + ON error_signatures(acknowledged_at) + WHERE acknowledged_at IS NULL; + + CREATE TABLE IF NOT EXISTS error_signature_windows ( + signature_hash TEXT NOT NULL, + normalizer_version INTEGER NOT NULL, + window_start TEXT NOT NULL, + window_end TEXT NOT NULL, + count_in_window INTEGER NOT NULL, + PRIMARY KEY (signature_hash, normalizer_version, window_start, window_end) + ); + + CREATE TABLE IF NOT EXISTS error_signature_ack_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + signature_hash TEXT NOT NULL, + normalizer_version INTEGER NOT NULL, + event_type TEXT NOT NULL CHECK (event_type IN ('ack','unack')), + actor TEXT NOT NULL, + notes TEXT CHECK (notes IS NULL OR length(notes) <= 4096), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')) + ); + CREATE INDEX IF NOT EXISTS idx_ack_events_sig + ON error_signature_ack_events(signature_hash, created_at DESC); + + CREATE TABLE IF NOT EXISTS error_scan_cursor ( + id INTEGER PRIMARY KEY CHECK (id = 1), + last_scanned_log_id INTEGER NOT NULL DEFAULT 0, + last_scan_completed_at TEXT + ); + INSERT OR IGNORE INTO error_scan_cursor (id, last_scanned_log_id) VALUES (1, 0); + + INSERT INTO schema_migrations (version) VALUES (10);", + )?; + tracing::info!("Migration 10: created error signature detection tables"); + } + + // Migration 11: notifications outbox and firings tables. + let migration_11_applied: bool = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 11", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap_or(0) + > 0; + if !migration_11_applied { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS notifications_outbox ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + dedup_key TEXT NOT NULL, + rule_id TEXT NOT NULL, + severity TEXT NOT NULL, + hostname TEXT NOT NULL, + title TEXT NOT NULL, + body TEXT NOT NULL, + apprise_urls_json TEXT NOT NULL, + apprise_tags TEXT, + enqueued_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + next_attempt_at TEXT NOT NULL, + attempt_count INTEGER NOT NULL DEFAULT 0, + last_status_code INTEGER, + last_error TEXT, + status TEXT NOT NULL DEFAULT 'pending' + CHECK (status IN ('pending','sent','dead','dropped')) + ); + CREATE INDEX IF NOT EXISTS idx_outbox_pending + ON notifications_outbox(status, next_attempt_at) + WHERE status = 'pending'; + CREATE INDEX IF NOT EXISTS idx_outbox_dedup + ON notifications_outbox(dedup_key, enqueued_at DESC); + + CREATE TABLE IF NOT EXISTS notification_firings ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + outbox_id INTEGER NOT NULL, + rule_id TEXT NOT NULL, + severity TEXT NOT NULL, + hostname TEXT NOT NULL, + fired_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ','now')), + status_code INTEGER, + notes TEXT + ); + CREATE INDEX IF NOT EXISTS idx_firings_fired_at + ON notification_firings(fired_at DESC); + CREATE INDEX IF NOT EXISTS idx_firings_rule + ON notification_firings(rule_id, fired_at DESC); + + INSERT INTO schema_migrations (version) VALUES (11);", + )?; + tracing::info!("Migration 11: created notifications outbox and firings tables"); + } + + // Migration 12: add dedup_key column to notification_firings and unique partial + // index on notifications_outbox to fix TOCTOU on outbox_insert. + let migration_12_applied: bool = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 12", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap_or(0) + > 0; + if !migration_12_applied { + // Add dedup_key to notification_firings so dedup checks are scoped per + // (rule_id, hostname, dedup_key) rather than just (rule_id, hostname). + // Without this, all error_sig firings share rule_id='unaddressed_error_signature' + // and the first firing suppresses all subsequent ones regardless of signature. + let dedup_col_exists: bool = conn + .query_row( + "SELECT COUNT(*) FROM pragma_table_info('notification_firings') WHERE name = 'dedup_key'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap_or(0) + > 0; + if !dedup_col_exists { + conn.execute_batch( + "ALTER TABLE notification_firings ADD COLUMN dedup_key TEXT NOT NULL DEFAULT '';", + )?; + } + conn.execute_batch( + "CREATE UNIQUE INDEX IF NOT EXISTS idx_outbox_dedup_pending + ON notifications_outbox(dedup_key) WHERE status = 'pending'; + INSERT INTO schema_migrations (version) VALUES (12);", + )?; + tracing::info!( + "Migration 12: added notification_firings.dedup_key, unique partial index on outbox" + ); + } + + // Migration 13: enrichment-framework columns + partial indexes. + // Spec: docs/superpowers/specs/2026-05-16-enrichment-framework-design.md §5 + // Contract: docs/contracts/db-additions.sql Epic B section + if !migration_applied(&conn, 13)? { + apply_migration_13(&conn)?; + tracing::info!("Migration 13: added enrichment columns + partial indexes"); + } + + let already_applied_14: i64 = conn.query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 14", + [], + |r| r.get(0), + )?; + if already_applied_14 == 0 { + tracing::info!( + "Migration 14: starting CREATE INDEX idx_logs_ai_session_host_time \ + — may take time on large AI transcript databases" + ); + let started = std::time::Instant::now(); + conn.execute_batch( + "CREATE INDEX IF NOT EXISTS idx_logs_ai_session_host_time + ON logs(ai_project, ai_tool, ai_session_id, hostname, timestamp) + WHERE ai_project IS NOT NULL + AND ai_tool IS NOT NULL + AND ai_session_id IS NOT NULL; + INSERT INTO schema_migrations (version) VALUES (14);", + )?; + tracing::info!( + elapsed_ms = started.elapsed().as_millis(), + "Migration 14: AI session host/time index created" + ); + } + + // Migration 15: first-class heartbeat telemetry storage. + // Contract: docs/contracts/heartbeat-telemetry.md + if !migration_applied(&conn, 15)? { + apply_migration_15_heartbeat(&conn)?; + tracing::info!("Migration 15: created heartbeat telemetry tables and indexes"); + } + + if !migration_applied(&conn, 16)? { + tracing::info!( + "Migration 16: starting CREATE INDEX idx_logs_app_name_timestamp \ + — may take time on large databases" + ); + let started = std::time::Instant::now(); + conn.execute_batch( + "CREATE INDEX IF NOT EXISTS idx_logs_app_name_timestamp + ON logs(app_name, timestamp); + INSERT INTO schema_migrations (version) VALUES (16);", + )?; + tracing::info!( + elapsed_ms = started.elapsed().as_millis(), + "Migration 16: app_name/timestamp search index created" + ); + } + + if !migration_applied(&conn, 17)? { + apply_migration_17_inventory_stats(&conn)?; + tracing::info!("Migration 17: created app/source inventory stats"); + } + + // Migration 18: add restarting column to heartbeat_containers. + if !migration_applied(&conn, 18)? { + apply_migration_18_heartbeat_restarting(&conn)?; + tracing::info!("Migration 18: added restarting column to heartbeat_containers"); + } + + // Migration 19: add host_heartbeats_latest fleet cache table. + if !migration_applied(&conn, 19)? { + apply_migration_19_heartbeat_latest(&conn)?; + tracing::info!("Migration 19: created host_heartbeats_latest fleet cache table"); + } + + // Migration 20: composite index on error_signature_windows(window_end, ...) for sig list queries. + // The `sig list` action filters unaddressed signatures by recency, ordering on window_end DESC. + // Without this index, every query does a full scan of error_signature_windows. + if !migration_applied(&conn, 20)? { + conn.execute_batch( + "CREATE INDEX IF NOT EXISTS idx_error_sig_windows_end + ON error_signature_windows(window_end, signature_hash, normalizer_version); + INSERT INTO schema_migrations (version) VALUES (20);", + )?; + tracing::info!("Migration 20: added index on error_signature_windows(window_end)"); + } + + // Migration 21: AI session rollup table (bead cortex-2vre). + // `list_ai_sessions` aggregates GROUP BY (project, tool, session, hostname) + // over the full AI-row partition then sorts by MAX(timestamp) — an + // unavoidable temp-btree that grows with AI-row count (~4s at 10M rows). + // The rollup is a periodically-refreshed materialization read in O(#sessions) + // via idx, decoupling read latency from row count. It is REFRESH-based, not + // trigger-based: trigger-maintained MIN/MAX is wrong on DELETE (deleting the + // row holding the current MAX can't recover the new extreme without a rescan, + // and that rescan reintroduces bulk-purge lock contention). Staleness is + // exposed via ai_session_rollup_meta.refreshed_at. + if !migration_applied(&conn, 21)? { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS ai_session_rollup ( + ai_project TEXT NOT NULL, + ai_tool TEXT NOT NULL, + ai_session_id TEXT NOT NULL, + hostname TEXT NOT NULL, + ai_transcript_path TEXT, + first_seen TEXT NOT NULL, + last_seen TEXT NOT NULL, + event_count INTEGER NOT NULL, + PRIMARY KEY (ai_project, ai_tool, ai_session_id, hostname) + ); + CREATE INDEX IF NOT EXISTS idx_ai_session_rollup_last_seen + ON ai_session_rollup(last_seen DESC); + CREATE TABLE IF NOT EXISTS ai_session_rollup_meta ( + id INTEGER PRIMARY KEY CHECK (id = 1), + refreshed_at TEXT, + row_count INTEGER NOT NULL DEFAULT 0 + ); + INSERT OR IGNORE INTO ai_session_rollup_meta (id, refreshed_at, row_count) + VALUES (1, NULL, 0); + INSERT INTO schema_migrations (version) VALUES (21);", + )?; + tracing::info!("Migration 21: created AI session rollup table"); + } + + // Migration 22: source watermark for the AI session rollup (bead + // cortex-g33v). The background refresh recomputed the full GROUP-BY + // over `logs` every cadence even when no AI rows had changed. These two + // columns record the source-side `(COUNT(*), MAX(id))` of AI rows captured + // by the last refresh; the refresh task compares the live watermark against + // them and skips the recompute entirely when nothing changed. Both default + // to 0 so the first post-migration refresh always runs (live watermark > 0 + // whenever AI rows exist, and `refreshed_at` is still NULL regardless). + if !migration_applied(&conn, 22)? { + apply_migration_22(&conn)?; + tracing::info!("Migration 22: added AI session rollup source watermark"); + } + + // Migration 23: covering indexes for the `errors` summary and `ai projects` + // aggregation. Both previously read every matching row from the table to + // fetch columns absent from the leading index (hostname for the error + // GROUP BY; ai_tool / ai_session_id for the project rollup), making them + // O(matching-rows) table-lookup scans (~10s and ~48s on a multi-million-row + // DB). These covering indexes make both aggregations index-only — verified + // via EXPLAIN QUERY PLAN flipping to `USING COVERING INDEX`. + // + // First-run cost: building these on a populated DB scans the table and + // holds the write lock for the duration (seconds to minutes at multi- + // million-row volumes); /health may gap and syslog packets may drop during + // that window — the same one-time cost as the earlier index migrations. + if !migration_applied(&conn, 23)? { + tracing::info!( + "Migration 23: building covering indexes (idx_logs_ai_project_cover, \ + idx_logs_sev_host_time) — may take minutes on large DBs, write lock held" + ); + let started = std::time::Instant::now(); + conn.execute_batch( + "CREATE INDEX IF NOT EXISTS idx_logs_ai_project_cover + ON logs(ai_project, ai_tool, ai_session_id, timestamp) + WHERE ai_project IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_logs_sev_host_time + ON logs(severity, hostname, timestamp); + INSERT INTO schema_migrations (version) VALUES (23);", + )?; + tracing::info!( + elapsed_ms = started.elapsed().as_millis(), + "Migration 23: covering indexes for errors + ai projects created" + ); + } + + // Migration 24: timestamp-positioned covering indexes for the AI + // aggregations, plus baseline ANALYZE stats. + // + // Migration 23's idx_logs_ai_project_cover (ai_project, ai_tool, + // ai_session_id, timestamp) made `ai projects` index-only, but with a + // timestamp-range filter (e.g. `ai blocks`'s 30-day default) the planner + // can't use its trailing `timestamp` as a seek and instead chose + // idx_logs_timestamp — scanning all recent high-volume syslog and filtering + // AI rows out one by one (~28s). Putting `timestamp` SECOND + // (ai_project, timestamp, ai_tool, ai_session_id) gives both a seekable + // range and full coverage, and supersedes the old index for every AI query + // (verified: nothing picks idx_logs_ai_project_cover once this exists), so + // it is dropped. idx_logs_ai_tool_cover does the same for `ai tools` + // (GROUP BY ai_tool needs session_id + timestamp). + // + // CRITICAL: these indexes are only *chosen* when ANALYZE statistics exist — + // without `sqlite_stat1`, the planner's no-stats heuristics still pick + // idx_logs_timestamp (verified empirically). So this migration also runs an + // initial ANALYZE (bounded by the connection's analysis_limit=400), and the + // optimize maintenance task keeps stats fresh as the DB grows. Same first- + // run write-lock cost as the other index migrations. + if !migration_applied(&conn, 24)? { + tracing::info!( + "Migration 24: rebuilding AI covering indexes (timestamp-positioned) \ + + initial ANALYZE — may take minutes on large DBs, write lock held" + ); + let started = std::time::Instant::now(); + conn.execute_batch( + "DROP INDEX IF EXISTS idx_logs_ai_project_cover; + CREATE INDEX IF NOT EXISTS idx_logs_ai_project_ts_cover + ON logs(ai_project, timestamp, ai_tool, ai_session_id) + WHERE ai_project IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_logs_ai_tool_cover + ON logs(ai_tool, ai_session_id, timestamp) + WHERE ai_tool IS NOT NULL;", + )?; + // Only seed stats when the table already has data. ANALYZE on an empty + // `logs` (fresh install / tests) records "0 rows", which mis-guides the + // planner once rows arrive; an empty DB instead gets its first stats + // from the optimize maintenance task (or the next restart) once + // populated. The existing populated DB analyzes immediately here. + let has_rows: bool = + conn.query_row("SELECT EXISTS(SELECT 1 FROM logs)", [], |r| r.get(0))?; + if has_rows { + conn.execute_batch("ANALYZE;")?; + } + conn.execute_batch("INSERT INTO schema_migrations (version) VALUES (24);")?; + tracing::info!( + elapsed_ms = started.elapsed().as_millis(), + "Migration 24: AI covering indexes rebuilt + baseline ANALYZE done" + ); + } + + // Migration 25: timeline_hourly rollup (bead syslog-mcp-kcvq). + // + // `timeline` (bucket=hour/day/week/month) and `stats.total_logs` previously + // scanned the whole `logs` table (`strftime` GROUP BY ~3s; `COUNT(*)` ~7s on + // a multi-million-row DB). This table materializes per-hour event counts at + // grain (bucket_hour, hostname, app_name, severity) — ~9.3k rows over 2.65M + // raw logs (~280x reduction), so timeline/stats reads become O(#buckets). + // + // INCREMENTAL, not full-recompute (contrast ai_session_rollup): a full + // recompute is the 63s `strftime`-over-2.65M scan. The rollup holds ONLY + // COUNT(*) — no MIN/MAX — so it is self-maintainable for ADDs: aggregate only + // `logs WHERE id > source_max_id` and upsert-add into existing buckets. A + // late-arriving high-id row with an old timestamp correctly lands in its old + // bucket. The only incremental hazard is DELETEs (retention purges oldest + // rows by received_at); the retention task prunes stale low buckets after + // each purge (see spawn_retention_task), accepting a transient overcount only + // in the single boundary hour. + // + // `app_name` is stored NOT NULL via COALESCE(app_name,'') — SQLite treats + // NULLs as DISTINCT in UNIQUE/PK indexes, so a nullable column would make + // ON CONFLICT never match for null-app grains and double-count every tick. + // + // First-run backfill is the one-time 63s scan, guarded by a has_rows check so + // empty/test DBs skip it. It runs at server STARTUP before ingest begins, so + // holding the write lock here is acceptable (same pattern as migration 24). + if !migration_applied(&conn, 25)? { + tracing::info!( + "Migration 25: creating timeline_hourly rollup + backfill — backfill is \ + a one-time full scan (~60s on large DBs, write lock held)" + ); + let started = std::time::Instant::now(); + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS timeline_hourly ( + bucket TEXT NOT NULL, + hostname TEXT NOT NULL, + app_name TEXT NOT NULL, + severity TEXT NOT NULL, + event_count INTEGER NOT NULL, + PRIMARY KEY (bucket, hostname, app_name, severity) + ); + CREATE TABLE IF NOT EXISTS timeline_hourly_meta ( + id INTEGER PRIMARY KEY CHECK (id = 1), + refreshed_at TEXT, + source_max_id INTEGER NOT NULL DEFAULT 0 + ); + INSERT OR IGNORE INTO timeline_hourly_meta (id, refreshed_at, source_max_id) + VALUES (1, NULL, 0);", + )?; + // Backfill only when the table has data (fresh installs / tests skip the + // scan and start from an empty rollup at watermark 0). + let has_rows: bool = + conn.query_row("SELECT EXISTS(SELECT 1 FROM logs)", [], |r| r.get(0))?; + if has_rows { + let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + let max_id: i64 = + tx.query_row("SELECT COALESCE(MAX(id), 0) FROM logs", [], |r| r.get(0))?; + tx.execute( + "INSERT INTO timeline_hourly (bucket, hostname, app_name, severity, event_count) + SELECT strftime('%Y-%m-%dT%H:00:00Z', timestamp) AS bucket, + hostname, + COALESCE(app_name, '') AS app_name, + severity, + COUNT(*) AS event_count + FROM logs + WHERE id <= ?1 + GROUP BY bucket, hostname, app_name, severity + ON CONFLICT(bucket, hostname, app_name, severity) + DO UPDATE SET event_count = event_count + excluded.event_count", + [max_id], + )?; + tx.execute( + "UPDATE timeline_hourly_meta + SET refreshed_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), + source_max_id = ?1 + WHERE id = 1", + [max_id], + )?; + tx.commit()?; + } + conn.execute_batch("INSERT INTO schema_migrations (version) VALUES (25);")?; + tracing::info!( + elapsed_ms = started.elapsed().as_millis(), + "Migration 25: timeline_hourly rollup created + backfilled" + ); + } + + // Migration 26: maintenance_jobs table (bead syslog-mcp-a4pd). + // + // `db integrity` on a 5GB DB is ~147s (PRAGMA quick_check reads every page — + // unfixable). This table backs a server-side background job: the HTTP path + // inserts a 'running' row, spawns the check on a blocking thread, and updates + // the row to 'done'/'failed' + result_json; clients poll by id. quick_check + // is read-only so it never blocks ingest writes. + if !migration_applied(&conn, 26)? { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS maintenance_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + kind TEXT NOT NULL, + status TEXT NOT NULL, + started_at TEXT NOT NULL, + finished_at TEXT, + result_json TEXT + ); + CREATE INDEX IF NOT EXISTS idx_maintenance_jobs_kind_status + ON maintenance_jobs(kind, status); + INSERT INTO schema_migrations (version) VALUES (26);", + )?; + tracing::info!("Migration 26: created maintenance_jobs table"); + } + + // Migration 27: derived investigation graph projection (bead syslog-mcp-24vc.1). + // + // This is schema only: no ingest-path graph writes, no triggers, and no + // service/API behavior. Raw logs, heartbeats, signatures, inventory, and AI + // session rows remain authoritative; graph rows are rebuildable projection + // data. Source references are intentionally soft references because this + // process does not enable PRAGMA foreign_keys on pooled connections. + if !migration_applied(&conn, 27)? { + conn.execute_batch( + "BEGIN IMMEDIATE; + + CREATE TABLE IF NOT EXISTS graph_entities ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entity_type TEXT NOT NULL CHECK (entity_type IN ( + 'host', 'container', 'service', 'app', 'source_ip', + 'ai_project', 'ai_session', 'error_signature', + 'compose_project', 'reverse_proxy', 'domain', 'network', + 'storage', 'config_artifact' + )), + canonical_key TEXT NOT NULL, + display_label TEXT NOT NULL, + source_kind TEXT NOT NULL DEFAULT '', + source_id TEXT NOT NULL DEFAULT '', + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated' + )), + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(entity_type, canonical_key) + ); + CREATE INDEX IF NOT EXISTS idx_graph_entities_type_key + ON graph_entities(entity_type, canonical_key); + + CREATE TABLE IF NOT EXISTS graph_entity_aliases ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entity_id INTEGER NOT NULL, + alias_type TEXT NOT NULL, + alias_key TEXT NOT NULL, + alias_value TEXT NOT NULL, + source_kind TEXT NOT NULL DEFAULT '', + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated' + )), + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(entity_id, alias_type, alias_key, source_kind) + ); + CREATE INDEX IF NOT EXISTS idx_graph_aliases_lookup + ON graph_entity_aliases(alias_type, alias_key); + CREATE INDEX IF NOT EXISTS idx_graph_aliases_entity + ON graph_entity_aliases(entity_id); + + CREATE TABLE IF NOT EXISTS graph_relationships ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relationship_key TEXT NOT NULL UNIQUE, + src_entity_id INTEGER NOT NULL, + dst_entity_id INTEGER NOT NULL, + relationship_type TEXT NOT NULL CHECK (relationship_type IN ( + 'observed_as', 'runs_on', 'emitted_by', 'worked_on', + 'matches_signature', 'defines_service', 'routes_to', + 'exposes_domain', 'attached_to', 'mounts', 'backed_by', + 'has_artifact' + )), + reason_code TEXT NOT NULL CHECK (reason_code IN ( + 'syslog_claimed_hostname', 'log_app_name', + 'docker_container_id', 'docker_service_label', + 'ai_session_project', 'heartbeat_host_state', + 'error_signature_match', 'inventory_node', + 'inventory_service', 'compose_config', + 'reverse_proxy_config', 'docker_network', 'storage_probe', + 'config_artifact' + )), + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated' + )), + confidence REAL NOT NULL DEFAULT 0.0 CHECK (confidence >= 0.0 AND confidence <= 1.0), + evidence_count INTEGER NOT NULL DEFAULT 0 CHECK (evidence_count >= 0), + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(src_entity_id, dst_entity_id, relationship_type, relationship_key) + ); + CREATE INDEX IF NOT EXISTS idx_graph_relationships_src_type_seen + ON graph_relationships(src_entity_id, relationship_type, last_seen_at DESC); + CREATE INDEX IF NOT EXISTS idx_graph_relationships_dst_type_seen + ON graph_relationships(dst_entity_id, relationship_type, last_seen_at DESC); + CREATE INDEX IF NOT EXISTS idx_graph_relationships_type_seen + ON graph_relationships(relationship_type, last_seen_at DESC); + + CREATE TABLE IF NOT EXISTS graph_relationship_evidence ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relationship_id INTEGER NOT NULL, + evidence_key TEXT NOT NULL, + source_kind TEXT NOT NULL CHECK (source_kind IN ( + 'log', 'heartbeat', 'ai_session_rollup', 'source_inventory', + 'app_inventory', 'error_signature' + )), + source_id TEXT NOT NULL DEFAULT '', + source_log_id INTEGER, + source_heartbeat_id INTEGER, + source_signature_hash TEXT, + observed_at TEXT NOT NULL, + reason_code TEXT NOT NULL CHECK (reason_code IN ( + 'syslog_claimed_hostname', 'log_app_name', + 'docker_container_id', 'docker_service_label', + 'ai_session_project', 'heartbeat_host_state', + 'error_signature_match', 'inventory_node', + 'inventory_service', 'compose_config', + 'reverse_proxy_config', 'docker_network', 'storage_probe', + 'config_artifact' + )), + reason_text TEXT, + confidence_delta REAL NOT NULL DEFAULT 0.0 CHECK (confidence_delta >= -1.0 AND confidence_delta <= 1.0), + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated' + )), + safe_excerpt TEXT CHECK (safe_excerpt IS NULL OR length(safe_excerpt) <= 512), + metadata_path TEXT, + evidence_count INTEGER NOT NULL DEFAULT 1 CHECK (evidence_count >= 1), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(relationship_id, evidence_key) + ); + CREATE INDEX IF NOT EXISTS idx_graph_evidence_relationship_seen + ON graph_relationship_evidence(relationship_id, observed_at DESC); + CREATE INDEX IF NOT EXISTS idx_graph_evidence_source_ref + ON graph_relationship_evidence(source_kind, source_id); + CREATE INDEX IF NOT EXISTS idx_graph_evidence_log_id + ON graph_relationship_evidence(source_log_id) + WHERE source_log_id IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_graph_evidence_heartbeat_id + ON graph_relationship_evidence(source_heartbeat_id) + WHERE source_heartbeat_id IS NOT NULL; + + CREATE TABLE IF NOT EXISTS graph_projection_meta ( + id INTEGER PRIMARY KEY CHECK (id = 1), + projection_status TEXT NOT NULL CHECK (projection_status IN ( + 'never_built', 'building', 'ready', 'stale', 'failed' + )), + last_started_at TEXT, + last_completed_at TEXT, + source_watermark TEXT NOT NULL DEFAULT '', + source_row_count INTEGER NOT NULL DEFAULT 0 CHECK (source_row_count >= 0), + entity_count INTEGER NOT NULL DEFAULT 0 CHECK (entity_count >= 0), + relationship_count INTEGER NOT NULL DEFAULT 0 CHECK (relationship_count >= 0), + evidence_count INTEGER NOT NULL DEFAULT 0 CHECK (evidence_count >= 0), + is_degraded INTEGER NOT NULL DEFAULT 0 CHECK (is_degraded IN (0, 1)), + last_error TEXT CHECK (last_error IS NULL OR length(last_error) <= 2048), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) + ); + INSERT OR IGNORE INTO graph_projection_meta + (id, projection_status, source_watermark) + VALUES (1, 'never_built', ''); + + INSERT INTO schema_migrations (version) VALUES (27); + + COMMIT;", + )?; + tracing::info!("Migration 27: created graph projection schema"); + } + + // Migration 28: add graph rebuild runtime metrics. + if !migration_applied(&conn, 28)? { + let tx = conn.transaction()?; + add_column_if_missing( + &tx, + "graph_projection_meta", + "last_runtime_ms", + "INTEGER NOT NULL DEFAULT 0 CHECK (last_runtime_ms >= 0)", + )?; + add_column_if_missing( + &tx, + "graph_projection_meta", + "last_chunk_count", + "INTEGER NOT NULL DEFAULT 0 CHECK (last_chunk_count >= 0)", + )?; + tx.execute( + "INSERT OR IGNORE INTO schema_migrations (version) VALUES (28)", + [], + )?; + tx.commit()?; + tracing::info!("Migration 28: added graph projection runtime metrics"); + } + + // Migration 29: add covering indexes for get_error_summary (group_by_app path), + // tail_logs severity filter, and extend the ai_session index to include timestamp + // so ORDER BY timestamp DESC is index-sortable without a temp b-tree. + if !migration_applied(&conn, 29)? { + conn.execute_batch( + "DROP INDEX IF EXISTS idx_logs_ai_session; + CREATE INDEX IF NOT EXISTS idx_logs_ai_session + ON logs(ai_tool, ai_project, ai_session_id, timestamp) + WHERE ai_tool IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_logs_sev_app_hostname_time + ON logs(severity, app_name, hostname, timestamp); + CREATE INDEX IF NOT EXISTS idx_logs_sev_time_id + ON logs(severity, timestamp, id); + INSERT OR IGNORE INTO schema_migrations (version) VALUES (29);", + )?; + tracing::info!( + "Migration 29: added covering indexes for error_summary, tail_logs, and ai_session sort" + ); + } + + // Migration 30: widen graph vocabulary for homelab inventory topology. + // + // SQLite CHECK constraints are part of the table definition, so adding + // entity/relationship/reason values requires rebuilding the constrained + // graph tables. The migration is a strict superset and preserves existing + // ids so aliases and evidence references remain valid. + if !migration_applied(&conn, 30)? { + conn.execute_batch( + "BEGIN IMMEDIATE; + + CREATE TABLE graph_entities_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entity_type TEXT NOT NULL CHECK (entity_type IN ( + 'host', 'container', 'service', 'app', 'source_ip', + 'ai_project', 'ai_session', 'error_signature', + 'compose_project', 'reverse_proxy', 'domain', 'network', + 'storage', 'config_artifact' + )), + canonical_key TEXT NOT NULL, + display_label TEXT NOT NULL, + source_kind TEXT NOT NULL DEFAULT '', + source_id TEXT NOT NULL DEFAULT '', + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated' + )), + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(entity_type, canonical_key) + ); + INSERT INTO graph_entities_new + (id, entity_type, canonical_key, display_label, source_kind, + source_id, trust_level, first_seen_at, last_seen_at, + created_at, updated_at) + SELECT id, entity_type, canonical_key, display_label, source_kind, + source_id, trust_level, first_seen_at, last_seen_at, + created_at, updated_at + FROM graph_entities; + DROP TABLE graph_entities; + ALTER TABLE graph_entities_new RENAME TO graph_entities; + CREATE INDEX idx_graph_entities_type_key + ON graph_entities(entity_type, canonical_key); + + CREATE TABLE graph_relationships_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relationship_key TEXT NOT NULL UNIQUE, + src_entity_id INTEGER NOT NULL, + dst_entity_id INTEGER NOT NULL, + relationship_type TEXT NOT NULL CHECK (relationship_type IN ( + 'observed_as', 'runs_on', 'emitted_by', 'worked_on', + 'matches_signature', 'defines_service', 'routes_to', + 'exposes_domain', 'attached_to', 'mounts', 'backed_by', + 'has_artifact' + )), + reason_code TEXT NOT NULL CHECK (reason_code IN ( + 'syslog_claimed_hostname', 'log_app_name', + 'docker_container_id', 'docker_service_label', + 'ai_session_project', 'heartbeat_host_state', + 'error_signature_match', 'inventory_node', + 'inventory_service', 'compose_config', + 'reverse_proxy_config', 'docker_network', 'storage_probe', + 'config_artifact' + )), + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated' + )), + confidence REAL NOT NULL DEFAULT 0.0 CHECK (confidence >= 0.0 AND confidence <= 1.0), + evidence_count INTEGER NOT NULL DEFAULT 0 CHECK (evidence_count >= 0), + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(src_entity_id, dst_entity_id, relationship_type, relationship_key) + ); + INSERT INTO graph_relationships_new + (id, relationship_key, src_entity_id, dst_entity_id, + relationship_type, reason_code, trust_level, confidence, + evidence_count, first_seen_at, last_seen_at, created_at, + updated_at) + SELECT id, relationship_key, src_entity_id, dst_entity_id, + relationship_type, reason_code, trust_level, confidence, + evidence_count, first_seen_at, last_seen_at, created_at, + updated_at + FROM graph_relationships; + DROP TABLE graph_relationships; + ALTER TABLE graph_relationships_new RENAME TO graph_relationships; + CREATE INDEX idx_graph_relationships_src_type_seen + ON graph_relationships(src_entity_id, relationship_type, last_seen_at DESC); + CREATE INDEX idx_graph_relationships_dst_type_seen + ON graph_relationships(dst_entity_id, relationship_type, last_seen_at DESC); + CREATE INDEX idx_graph_relationships_type_seen + ON graph_relationships(relationship_type, last_seen_at DESC); + + CREATE TABLE graph_relationship_evidence_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relationship_id INTEGER NOT NULL, + evidence_key TEXT NOT NULL, + source_kind TEXT NOT NULL CHECK (source_kind IN ( + 'log', 'heartbeat', 'ai_session_rollup', 'source_inventory', + 'app_inventory', 'error_signature' + )), + source_id TEXT NOT NULL DEFAULT '', + source_log_id INTEGER, + source_heartbeat_id INTEGER, + source_signature_hash TEXT, + observed_at TEXT NOT NULL, + reason_code TEXT NOT NULL CHECK (reason_code IN ( + 'syslog_claimed_hostname', 'log_app_name', + 'docker_container_id', 'docker_service_label', + 'ai_session_project', 'heartbeat_host_state', + 'error_signature_match', 'inventory_node', + 'inventory_service', 'compose_config', + 'reverse_proxy_config', 'docker_network', 'storage_probe', + 'config_artifact' + )), + reason_text TEXT, + confidence_delta REAL NOT NULL DEFAULT 0.0 CHECK (confidence_delta >= -1.0 AND confidence_delta <= 1.0), + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated' + )), + safe_excerpt TEXT CHECK (safe_excerpt IS NULL OR length(safe_excerpt) <= 512), + metadata_path TEXT, + evidence_count INTEGER NOT NULL DEFAULT 1 CHECK (evidence_count >= 1), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(relationship_id, evidence_key) + ); + INSERT INTO graph_relationship_evidence_new + (id, relationship_id, evidence_key, source_kind, source_id, + source_log_id, source_heartbeat_id, source_signature_hash, + observed_at, reason_code, reason_text, confidence_delta, + trust_level, safe_excerpt, metadata_path, evidence_count, + created_at) + SELECT id, relationship_id, evidence_key, source_kind, source_id, + source_log_id, source_heartbeat_id, source_signature_hash, + observed_at, reason_code, reason_text, confidence_delta, + trust_level, safe_excerpt, metadata_path, evidence_count, + created_at + FROM graph_relationship_evidence; + DROP TABLE graph_relationship_evidence; + ALTER TABLE graph_relationship_evidence_new RENAME TO graph_relationship_evidence; + CREATE INDEX idx_graph_evidence_relationship_seen + ON graph_relationship_evidence(relationship_id, observed_at DESC); + CREATE INDEX idx_graph_evidence_source_ref + ON graph_relationship_evidence(source_kind, source_id); + CREATE INDEX idx_graph_evidence_log_id + ON graph_relationship_evidence(source_log_id) + WHERE source_log_id IS NOT NULL; + CREATE INDEX idx_graph_evidence_heartbeat_id + ON graph_relationship_evidence(source_heartbeat_id) + WHERE source_heartbeat_id IS NOT NULL; + + INSERT OR IGNORE INTO schema_migrations (version) VALUES (30); + COMMIT;", + )?; + tracing::info!("Migration 30: widened graph vocabulary for inventory topology"); + } + + // Migration 31: relationship-type covering index for bounded topology + // findings. Findings ask for all relationships of one graph vocabulary + // type, so the src/dst-specific graph indexes are not sufficient. + if !migration_applied(&conn, 31)? { + conn.execute_batch( + "CREATE INDEX IF NOT EXISTS idx_graph_relationships_type_seen + ON graph_relationships(relationship_type, last_seen_at DESC); + INSERT OR IGNORE INTO schema_migrations (version) VALUES (31);", + )?; + tracing::info!("Migration 31: added graph relationship type index for topology findings"); + } + + // Migration 32: covering index for the graph→log join used by graph-anchored + // correlation (topic_correlate, ai_correlate rewrite). N-hop graph traversal + // resolves to a set of entity canonical keys, which become a + // `hostname IN (...)` filter over a bounded time window, frequently further + // narrowed by app_name. The existing idx_logs_host_time (hostname, timestamp) + // seeks the hostname+time range but must then fetch each candidate row from + // the heap to evaluate app_name — pathological across 5+ hostnames on wide + // windows. (hostname, app_name, timestamp) lets the planner satisfy the inner + // filter index-only. The second index covers the session_id-anchored fan-out + // (all logs for one AI session ordered by time) without touching the heap. + // + // First-run cost: building these on a populated DB scans the table and holds + // the write lock for the duration (seconds to minutes at multi-million-row + // volumes); /health may gap and syslog packets may drop during that window — + // the same one-time cost as the earlier index migrations. + if !migration_applied(&conn, 32)? { + tracing::info!( + "Migration 32: building graph→log covering indexes \ + (idx_logs_hostname_appname_time, idx_logs_ai_session_time) \ + — may take minutes on large DBs, write lock held" + ); + let started = std::time::Instant::now(); + conn.execute_batch( + "CREATE INDEX IF NOT EXISTS idx_logs_hostname_appname_time + ON logs(hostname, app_name, timestamp); + CREATE INDEX IF NOT EXISTS idx_logs_ai_session_time + ON logs(ai_session_id, timestamp) + WHERE ai_session_id IS NOT NULL; + INSERT OR IGNORE INTO schema_migrations (version) VALUES (32);", + )?; + tracing::info!( + elapsed_ms = started.elapsed().as_millis(), + "Migration 32: graph→log covering indexes created" + ); + } + + // Migration 33: widen graph reason-code vocabulary for agent-command + // projection (agent_command_session, agent_command_cwd_infer). + // + // SQLite CHECK constraints are part of the table definition, so adding + // reason_code values requires rebuilding the two constrained graph tables. + // The migration is a strict superset and preserves existing ids so evidence + // references remain valid. Mirrors migration 30's rebuild shape. + if !migration_applied(&conn, 33)? { + conn.execute_batch( + "BEGIN IMMEDIATE; + + CREATE TABLE graph_relationships_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relationship_key TEXT NOT NULL UNIQUE, + src_entity_id INTEGER NOT NULL, + dst_entity_id INTEGER NOT NULL, + relationship_type TEXT NOT NULL CHECK (relationship_type IN ( + 'observed_as', 'runs_on', 'emitted_by', 'worked_on', + 'matches_signature', 'defines_service', 'routes_to', + 'exposes_domain', 'attached_to', 'mounts', 'backed_by', + 'has_artifact' + )), + reason_code TEXT NOT NULL CHECK (reason_code IN ( + 'syslog_claimed_hostname', 'log_app_name', + 'docker_container_id', 'docker_service_label', + 'ai_session_project', 'heartbeat_host_state', + 'error_signature_match', 'inventory_node', + 'inventory_service', 'compose_config', + 'reverse_proxy_config', 'docker_network', 'storage_probe', + 'config_artifact', 'agent_command_session', + 'agent_command_cwd_infer' + )), + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated' + )), + confidence REAL NOT NULL DEFAULT 0.0 CHECK (confidence >= 0.0 AND confidence <= 1.0), + evidence_count INTEGER NOT NULL DEFAULT 0 CHECK (evidence_count >= 0), + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(src_entity_id, dst_entity_id, relationship_type, relationship_key) + ); + INSERT INTO graph_relationships_new + (id, relationship_key, src_entity_id, dst_entity_id, + relationship_type, reason_code, trust_level, confidence, + evidence_count, first_seen_at, last_seen_at, created_at, + updated_at) + SELECT id, relationship_key, src_entity_id, dst_entity_id, + relationship_type, reason_code, trust_level, confidence, + evidence_count, first_seen_at, last_seen_at, created_at, + updated_at + FROM graph_relationships; + DROP TABLE graph_relationships; + ALTER TABLE graph_relationships_new RENAME TO graph_relationships; + CREATE INDEX idx_graph_relationships_src_type_seen + ON graph_relationships(src_entity_id, relationship_type, last_seen_at DESC); + CREATE INDEX idx_graph_relationships_dst_type_seen + ON graph_relationships(dst_entity_id, relationship_type, last_seen_at DESC); + CREATE INDEX idx_graph_relationships_type_seen + ON graph_relationships(relationship_type, last_seen_at DESC); + + CREATE TABLE graph_relationship_evidence_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relationship_id INTEGER NOT NULL, + evidence_key TEXT NOT NULL, + source_kind TEXT NOT NULL CHECK (source_kind IN ( + 'log', 'heartbeat', 'ai_session_rollup', 'source_inventory', + 'app_inventory', 'error_signature' + )), + source_id TEXT NOT NULL DEFAULT '', + source_log_id INTEGER, + source_heartbeat_id INTEGER, + source_signature_hash TEXT, + observed_at TEXT NOT NULL, + reason_code TEXT NOT NULL CHECK (reason_code IN ( + 'syslog_claimed_hostname', 'log_app_name', + 'docker_container_id', 'docker_service_label', + 'ai_session_project', 'heartbeat_host_state', + 'error_signature_match', 'inventory_node', + 'inventory_service', 'compose_config', + 'reverse_proxy_config', 'docker_network', 'storage_probe', + 'config_artifact', 'agent_command_session', + 'agent_command_cwd_infer' + )), + reason_text TEXT, + confidence_delta REAL NOT NULL DEFAULT 0.0 CHECK (confidence_delta >= -1.0 AND confidence_delta <= 1.0), + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated' + )), + safe_excerpt TEXT CHECK (safe_excerpt IS NULL OR length(safe_excerpt) <= 512), + metadata_path TEXT, + evidence_count INTEGER NOT NULL DEFAULT 1 CHECK (evidence_count >= 1), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(relationship_id, evidence_key) + ); + INSERT INTO graph_relationship_evidence_new + (id, relationship_id, evidence_key, source_kind, source_id, + source_log_id, source_heartbeat_id, source_signature_hash, + observed_at, reason_code, reason_text, confidence_delta, + trust_level, safe_excerpt, metadata_path, evidence_count, + created_at) + SELECT id, relationship_id, evidence_key, source_kind, source_id, + source_log_id, source_heartbeat_id, source_signature_hash, + observed_at, reason_code, reason_text, confidence_delta, + trust_level, safe_excerpt, metadata_path, evidence_count, + created_at + FROM graph_relationship_evidence; + DROP TABLE graph_relationship_evidence; + ALTER TABLE graph_relationship_evidence_new RENAME TO graph_relationship_evidence; + CREATE INDEX idx_graph_evidence_relationship_seen + ON graph_relationship_evidence(relationship_id, observed_at DESC); + CREATE INDEX idx_graph_evidence_source_ref + ON graph_relationship_evidence(source_kind, source_id); + CREATE INDEX idx_graph_evidence_log_id + ON graph_relationship_evidence(source_log_id) + WHERE source_log_id IS NOT NULL; + CREATE INDEX idx_graph_evidence_heartbeat_id + ON graph_relationship_evidence(source_heartbeat_id) + WHERE source_heartbeat_id IS NOT NULL; + + INSERT OR IGNORE INTO schema_migrations (version) VALUES (33); + COMMIT;", + )?; + tracing::info!("Migration 33: widened graph reason-code vocabulary for agent commands"); + } + + // Migration 34: add the `git_commit` entity type and the two git-commit + // reason codes (agent_command_git_commit, shell_history_git_commit). + // Rebuilds the three constrained graph tables; strict superset, ids + // preserved. Mirrors migrations 30/33. + if !migration_applied(&conn, 34)? { + conn.execute_batch( + "BEGIN IMMEDIATE; + + CREATE TABLE graph_entities_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entity_type TEXT NOT NULL CHECK (entity_type IN ( + 'host', 'container', 'service', 'app', 'source_ip', + 'ai_project', 'ai_session', 'error_signature', + 'compose_project', 'reverse_proxy', 'domain', 'network', + 'storage', 'config_artifact', 'git_commit' + )), + canonical_key TEXT NOT NULL, + display_label TEXT NOT NULL, + source_kind TEXT NOT NULL DEFAULT '', + source_id TEXT NOT NULL DEFAULT '', + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated' + )), + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(entity_type, canonical_key) + ); + INSERT INTO graph_entities_new + (id, entity_type, canonical_key, display_label, source_kind, + source_id, trust_level, first_seen_at, last_seen_at, + created_at, updated_at) + SELECT id, entity_type, canonical_key, display_label, source_kind, + source_id, trust_level, first_seen_at, last_seen_at, + created_at, updated_at + FROM graph_entities; + DROP TABLE graph_entities; + ALTER TABLE graph_entities_new RENAME TO graph_entities; + CREATE INDEX idx_graph_entities_type_key + ON graph_entities(entity_type, canonical_key); + + CREATE TABLE graph_relationships_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relationship_key TEXT NOT NULL UNIQUE, + src_entity_id INTEGER NOT NULL, + dst_entity_id INTEGER NOT NULL, + relationship_type TEXT NOT NULL CHECK (relationship_type IN ( + 'observed_as', 'runs_on', 'emitted_by', 'worked_on', + 'matches_signature', 'defines_service', 'routes_to', + 'exposes_domain', 'attached_to', 'mounts', 'backed_by', + 'has_artifact' + )), + reason_code TEXT NOT NULL CHECK (reason_code IN ( + 'syslog_claimed_hostname', 'log_app_name', + 'docker_container_id', 'docker_service_label', + 'ai_session_project', 'heartbeat_host_state', + 'error_signature_match', 'inventory_node', + 'inventory_service', 'compose_config', + 'reverse_proxy_config', 'docker_network', 'storage_probe', + 'config_artifact', 'agent_command_session', + 'agent_command_cwd_infer', 'agent_command_git_commit', + 'shell_history_git_commit' + )), + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated' + )), + confidence REAL NOT NULL DEFAULT 0.0 CHECK (confidence >= 0.0 AND confidence <= 1.0), + evidence_count INTEGER NOT NULL DEFAULT 0 CHECK (evidence_count >= 0), + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(src_entity_id, dst_entity_id, relationship_type, relationship_key) + ); + INSERT INTO graph_relationships_new + (id, relationship_key, src_entity_id, dst_entity_id, + relationship_type, reason_code, trust_level, confidence, + evidence_count, first_seen_at, last_seen_at, created_at, + updated_at) + SELECT id, relationship_key, src_entity_id, dst_entity_id, + relationship_type, reason_code, trust_level, confidence, + evidence_count, first_seen_at, last_seen_at, created_at, + updated_at + FROM graph_relationships; + DROP TABLE graph_relationships; + ALTER TABLE graph_relationships_new RENAME TO graph_relationships; + CREATE INDEX idx_graph_relationships_src_type_seen + ON graph_relationships(src_entity_id, relationship_type, last_seen_at DESC); + CREATE INDEX idx_graph_relationships_dst_type_seen + ON graph_relationships(dst_entity_id, relationship_type, last_seen_at DESC); + CREATE INDEX idx_graph_relationships_type_seen + ON graph_relationships(relationship_type, last_seen_at DESC); + + CREATE TABLE graph_relationship_evidence_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relationship_id INTEGER NOT NULL, + evidence_key TEXT NOT NULL, + source_kind TEXT NOT NULL CHECK (source_kind IN ( + 'log', 'heartbeat', 'ai_session_rollup', 'source_inventory', + 'app_inventory', 'error_signature' + )), + source_id TEXT NOT NULL DEFAULT '', + source_log_id INTEGER, + source_heartbeat_id INTEGER, + source_signature_hash TEXT, + observed_at TEXT NOT NULL, + reason_code TEXT NOT NULL CHECK (reason_code IN ( + 'syslog_claimed_hostname', 'log_app_name', + 'docker_container_id', 'docker_service_label', + 'ai_session_project', 'heartbeat_host_state', + 'error_signature_match', 'inventory_node', + 'inventory_service', 'compose_config', + 'reverse_proxy_config', 'docker_network', 'storage_probe', + 'config_artifact', 'agent_command_session', + 'agent_command_cwd_infer', 'agent_command_git_commit', + 'shell_history_git_commit' + )), + reason_text TEXT, + confidence_delta REAL NOT NULL DEFAULT 0.0 CHECK (confidence_delta >= -1.0 AND confidence_delta <= 1.0), + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated' + )), + safe_excerpt TEXT CHECK (safe_excerpt IS NULL OR length(safe_excerpt) <= 512), + metadata_path TEXT, + evidence_count INTEGER NOT NULL DEFAULT 1 CHECK (evidence_count >= 1), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(relationship_id, evidence_key) + ); + INSERT INTO graph_relationship_evidence_new + (id, relationship_id, evidence_key, source_kind, source_id, + source_log_id, source_heartbeat_id, source_signature_hash, + observed_at, reason_code, reason_text, confidence_delta, + trust_level, safe_excerpt, metadata_path, evidence_count, + created_at) + SELECT id, relationship_id, evidence_key, source_kind, source_id, + source_log_id, source_heartbeat_id, source_signature_hash, + observed_at, reason_code, reason_text, confidence_delta, + trust_level, safe_excerpt, metadata_path, evidence_count, + created_at + FROM graph_relationship_evidence; + DROP TABLE graph_relationship_evidence; + ALTER TABLE graph_relationship_evidence_new RENAME TO graph_relationship_evidence; + CREATE INDEX idx_graph_evidence_relationship_seen + ON graph_relationship_evidence(relationship_id, observed_at DESC); + CREATE INDEX idx_graph_evidence_source_ref + ON graph_relationship_evidence(source_kind, source_id); + CREATE INDEX idx_graph_evidence_log_id + ON graph_relationship_evidence(source_log_id) + WHERE source_log_id IS NOT NULL; + CREATE INDEX idx_graph_evidence_heartbeat_id + ON graph_relationship_evidence(source_heartbeat_id) + WHERE source_heartbeat_id IS NOT NULL; + + INSERT OR IGNORE INTO schema_migrations (version) VALUES (34); + COMMIT;", + )?; + tracing::info!("Migration 34: added git_commit entity type and git-commit reason codes"); + } + + // Migration 35: add the `refuted` trust level to the three graph tables' + // trust_level CHECK. Refuted edges record disproved/retracted relationships + // (manual override) and are excluded from every traversal result. Rebuilds + // the constrained tables; strict superset, ids preserved. + if !migration_applied(&conn, 35)? { + conn.execute_batch( + "BEGIN IMMEDIATE; + + CREATE TABLE graph_entities_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entity_type TEXT NOT NULL CHECK (entity_type IN ( + 'host', 'container', 'service', 'app', 'source_ip', + 'ai_project', 'ai_session', 'error_signature', + 'compose_project', 'reverse_proxy', 'domain', 'network', + 'storage', 'config_artifact', 'git_commit' + )), + canonical_key TEXT NOT NULL, + display_label TEXT NOT NULL, + source_kind TEXT NOT NULL DEFAULT '', + source_id TEXT NOT NULL DEFAULT '', + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated', 'refuted' + )), + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(entity_type, canonical_key) + ); + INSERT INTO graph_entities_new + (id, entity_type, canonical_key, display_label, source_kind, + source_id, trust_level, first_seen_at, last_seen_at, + created_at, updated_at) + SELECT id, entity_type, canonical_key, display_label, source_kind, + source_id, trust_level, first_seen_at, last_seen_at, + created_at, updated_at + FROM graph_entities; + DROP TABLE graph_entities; + ALTER TABLE graph_entities_new RENAME TO graph_entities; + CREATE INDEX idx_graph_entities_type_key + ON graph_entities(entity_type, canonical_key); + + CREATE TABLE graph_relationships_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relationship_key TEXT NOT NULL UNIQUE, + src_entity_id INTEGER NOT NULL, + dst_entity_id INTEGER NOT NULL, + relationship_type TEXT NOT NULL CHECK (relationship_type IN ( + 'observed_as', 'runs_on', 'emitted_by', 'worked_on', + 'matches_signature', 'defines_service', 'routes_to', + 'exposes_domain', 'attached_to', 'mounts', 'backed_by', + 'has_artifact' + )), + reason_code TEXT NOT NULL CHECK (reason_code IN ( + 'syslog_claimed_hostname', 'log_app_name', + 'docker_container_id', 'docker_service_label', + 'ai_session_project', 'heartbeat_host_state', + 'error_signature_match', 'inventory_node', + 'inventory_service', 'compose_config', + 'reverse_proxy_config', 'docker_network', 'storage_probe', + 'config_artifact', 'agent_command_session', + 'agent_command_cwd_infer', 'agent_command_git_commit', + 'shell_history_git_commit' + )), + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated', 'refuted' + )), + confidence REAL NOT NULL DEFAULT 0.0 CHECK (confidence >= 0.0 AND confidence <= 1.0), + evidence_count INTEGER NOT NULL DEFAULT 0 CHECK (evidence_count >= 0), + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(src_entity_id, dst_entity_id, relationship_type, relationship_key) + ); + INSERT INTO graph_relationships_new + (id, relationship_key, src_entity_id, dst_entity_id, + relationship_type, reason_code, trust_level, confidence, + evidence_count, first_seen_at, last_seen_at, created_at, + updated_at) + SELECT id, relationship_key, src_entity_id, dst_entity_id, + relationship_type, reason_code, trust_level, confidence, + evidence_count, first_seen_at, last_seen_at, created_at, + updated_at + FROM graph_relationships; + DROP TABLE graph_relationships; + ALTER TABLE graph_relationships_new RENAME TO graph_relationships; + CREATE INDEX idx_graph_relationships_src_type_seen + ON graph_relationships(src_entity_id, relationship_type, last_seen_at DESC); + CREATE INDEX idx_graph_relationships_dst_type_seen + ON graph_relationships(dst_entity_id, relationship_type, last_seen_at DESC); + CREATE INDEX idx_graph_relationships_type_seen + ON graph_relationships(relationship_type, last_seen_at DESC); + + CREATE TABLE graph_relationship_evidence_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relationship_id INTEGER NOT NULL, + evidence_key TEXT NOT NULL, + source_kind TEXT NOT NULL CHECK (source_kind IN ( + 'log', 'heartbeat', 'ai_session_rollup', 'source_inventory', + 'app_inventory', 'error_signature' + )), + source_id TEXT NOT NULL DEFAULT '', + source_log_id INTEGER, + source_heartbeat_id INTEGER, + source_signature_hash TEXT, + observed_at TEXT NOT NULL, + reason_code TEXT NOT NULL CHECK (reason_code IN ( + 'syslog_claimed_hostname', 'log_app_name', + 'docker_container_id', 'docker_service_label', + 'ai_session_project', 'heartbeat_host_state', + 'error_signature_match', 'inventory_node', + 'inventory_service', 'compose_config', + 'reverse_proxy_config', 'docker_network', 'storage_probe', + 'config_artifact', 'agent_command_session', + 'agent_command_cwd_infer', 'agent_command_git_commit', + 'shell_history_git_commit' + )), + reason_text TEXT, + confidence_delta REAL NOT NULL DEFAULT 0.0 CHECK (confidence_delta >= -1.0 AND confidence_delta <= 1.0), + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated', 'refuted' + )), + safe_excerpt TEXT CHECK (safe_excerpt IS NULL OR length(safe_excerpt) <= 512), + metadata_path TEXT, + evidence_count INTEGER NOT NULL DEFAULT 1 CHECK (evidence_count >= 1), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(relationship_id, evidence_key) + ); + INSERT INTO graph_relationship_evidence_new + (id, relationship_id, evidence_key, source_kind, source_id, + source_log_id, source_heartbeat_id, source_signature_hash, + observed_at, reason_code, reason_text, confidence_delta, + trust_level, safe_excerpt, metadata_path, evidence_count, + created_at) + SELECT id, relationship_id, evidence_key, source_kind, source_id, + source_log_id, source_heartbeat_id, source_signature_hash, + observed_at, reason_code, reason_text, confidence_delta, + trust_level, safe_excerpt, metadata_path, evidence_count, + created_at + FROM graph_relationship_evidence; + DROP TABLE graph_relationship_evidence; + ALTER TABLE graph_relationship_evidence_new RENAME TO graph_relationship_evidence; + CREATE INDEX idx_graph_evidence_relationship_seen + ON graph_relationship_evidence(relationship_id, observed_at DESC); + CREATE INDEX idx_graph_evidence_source_ref + ON graph_relationship_evidence(source_kind, source_id); + CREATE INDEX idx_graph_evidence_log_id + ON graph_relationship_evidence(source_log_id) + WHERE source_log_id IS NOT NULL; + CREATE INDEX idx_graph_evidence_heartbeat_id + ON graph_relationship_evidence(source_heartbeat_id) + WHERE source_heartbeat_id IS NOT NULL; + + INSERT OR IGNORE INTO schema_migrations (version) VALUES (35); + COMMIT;", + )?; + tracing::info!("Migration 35: added refuted trust level to graph tables"); + } + + // Migration 36: add user/device entity types, the authenticated_as/accessed/ + // communicates_with relationship types, and the three identity reason codes + // (adguard_client_query, shell_history_user, authelia_auth). Rebuilds the + // constrained graph tables; strict superset, ids preserved. + if !migration_applied(&conn, 36)? { + conn.execute_batch( + "BEGIN IMMEDIATE; + + CREATE TABLE graph_entities_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entity_type TEXT NOT NULL CHECK (entity_type IN ( + 'host', 'container', 'service', 'app', 'source_ip', + 'ai_project', 'ai_session', 'error_signature', + 'compose_project', 'reverse_proxy', 'domain', 'network', + 'storage', 'config_artifact', 'git_commit', 'user', 'device' + )), + canonical_key TEXT NOT NULL, + display_label TEXT NOT NULL, + source_kind TEXT NOT NULL DEFAULT '', + source_id TEXT NOT NULL DEFAULT '', + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated', 'refuted' + )), + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(entity_type, canonical_key) + ); + INSERT INTO graph_entities_new + (id, entity_type, canonical_key, display_label, source_kind, + source_id, trust_level, first_seen_at, last_seen_at, + created_at, updated_at) + SELECT id, entity_type, canonical_key, display_label, source_kind, + source_id, trust_level, first_seen_at, last_seen_at, + created_at, updated_at + FROM graph_entities; + DROP TABLE graph_entities; + ALTER TABLE graph_entities_new RENAME TO graph_entities; + CREATE INDEX idx_graph_entities_type_key + ON graph_entities(entity_type, canonical_key); + + CREATE TABLE graph_relationships_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relationship_key TEXT NOT NULL UNIQUE, + src_entity_id INTEGER NOT NULL, + dst_entity_id INTEGER NOT NULL, + relationship_type TEXT NOT NULL CHECK (relationship_type IN ( + 'observed_as', 'runs_on', 'emitted_by', 'worked_on', + 'matches_signature', 'defines_service', 'routes_to', + 'exposes_domain', 'attached_to', 'mounts', 'backed_by', + 'has_artifact', 'authenticated_as', 'accessed', + 'communicates_with' + )), + reason_code TEXT NOT NULL CHECK (reason_code IN ( + 'syslog_claimed_hostname', 'log_app_name', + 'docker_container_id', 'docker_service_label', + 'ai_session_project', 'heartbeat_host_state', + 'error_signature_match', 'inventory_node', + 'inventory_service', 'compose_config', + 'reverse_proxy_config', 'docker_network', 'storage_probe', + 'config_artifact', 'agent_command_session', + 'agent_command_cwd_infer', 'agent_command_git_commit', + 'shell_history_git_commit', 'adguard_client_query', + 'shell_history_user', 'authelia_auth' + )), + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated', 'refuted' + )), + confidence REAL NOT NULL DEFAULT 0.0 CHECK (confidence >= 0.0 AND confidence <= 1.0), + evidence_count INTEGER NOT NULL DEFAULT 0 CHECK (evidence_count >= 0), + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(src_entity_id, dst_entity_id, relationship_type, relationship_key) + ); + INSERT INTO graph_relationships_new + (id, relationship_key, src_entity_id, dst_entity_id, + relationship_type, reason_code, trust_level, confidence, + evidence_count, first_seen_at, last_seen_at, created_at, + updated_at) + SELECT id, relationship_key, src_entity_id, dst_entity_id, + relationship_type, reason_code, trust_level, confidence, + evidence_count, first_seen_at, last_seen_at, created_at, + updated_at + FROM graph_relationships; + DROP TABLE graph_relationships; + ALTER TABLE graph_relationships_new RENAME TO graph_relationships; + CREATE INDEX idx_graph_relationships_src_type_seen + ON graph_relationships(src_entity_id, relationship_type, last_seen_at DESC); + CREATE INDEX idx_graph_relationships_dst_type_seen + ON graph_relationships(dst_entity_id, relationship_type, last_seen_at DESC); + CREATE INDEX idx_graph_relationships_type_seen + ON graph_relationships(relationship_type, last_seen_at DESC); + + CREATE TABLE graph_relationship_evidence_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relationship_id INTEGER NOT NULL, + evidence_key TEXT NOT NULL, + source_kind TEXT NOT NULL CHECK (source_kind IN ( + 'log', 'heartbeat', 'ai_session_rollup', 'source_inventory', + 'app_inventory', 'error_signature' + )), + source_id TEXT NOT NULL DEFAULT '', + source_log_id INTEGER, + source_heartbeat_id INTEGER, + source_signature_hash TEXT, + observed_at TEXT NOT NULL, + reason_code TEXT NOT NULL CHECK (reason_code IN ( + 'syslog_claimed_hostname', 'log_app_name', + 'docker_container_id', 'docker_service_label', + 'ai_session_project', 'heartbeat_host_state', + 'error_signature_match', 'inventory_node', + 'inventory_service', 'compose_config', + 'reverse_proxy_config', 'docker_network', 'storage_probe', + 'config_artifact', 'agent_command_session', + 'agent_command_cwd_infer', 'agent_command_git_commit', + 'shell_history_git_commit', 'adguard_client_query', + 'shell_history_user', 'authelia_auth' + )), + reason_text TEXT, + confidence_delta REAL NOT NULL DEFAULT 0.0 CHECK (confidence_delta >= -1.0 AND confidence_delta <= 1.0), + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated', 'refuted' + )), + safe_excerpt TEXT CHECK (safe_excerpt IS NULL OR length(safe_excerpt) <= 512), + metadata_path TEXT, + evidence_count INTEGER NOT NULL DEFAULT 1 CHECK (evidence_count >= 1), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(relationship_id, evidence_key) + ); + INSERT INTO graph_relationship_evidence_new + (id, relationship_id, evidence_key, source_kind, source_id, + source_log_id, source_heartbeat_id, source_signature_hash, + observed_at, reason_code, reason_text, confidence_delta, + trust_level, safe_excerpt, metadata_path, evidence_count, + created_at) + SELECT id, relationship_id, evidence_key, source_kind, source_id, + source_log_id, source_heartbeat_id, source_signature_hash, + observed_at, reason_code, reason_text, confidence_delta, + trust_level, safe_excerpt, metadata_path, evidence_count, + created_at + FROM graph_relationship_evidence; + DROP TABLE graph_relationship_evidence; + ALTER TABLE graph_relationship_evidence_new RENAME TO graph_relationship_evidence; + CREATE INDEX idx_graph_evidence_relationship_seen + ON graph_relationship_evidence(relationship_id, observed_at DESC); + CREATE INDEX idx_graph_evidence_source_ref + ON graph_relationship_evidence(source_kind, source_id); + CREATE INDEX idx_graph_evidence_log_id + ON graph_relationship_evidence(source_log_id) + WHERE source_log_id IS NOT NULL; + CREATE INDEX idx_graph_evidence_heartbeat_id + ON graph_relationship_evidence(source_heartbeat_id) + WHERE source_heartbeat_id IS NOT NULL; + + INSERT OR IGNORE INTO schema_migrations (version) VALUES (36); + COMMIT;", + )?; + tracing::info!("Migration 36: added user/device entities and identity relationships"); + } + + // Migration 37: create llm_invocations, the shared audit table for every + // LLM-backed assessment call (ai_assess today; skill_assess/mcp_assess/ + // hook_assess in later phases). A start row is written before the + // process/API call begins (status='running') and updated on completion. + // Concurrency/rate-limit/circuit-open/disabled denials also write a row + // (status set to the denial reason) so the audit trail covers every call + // attempt, not just ones that reached the LLM. + if !migration_applied(&conn, 37)? { + conn.execute_batch( + "BEGIN IMMEDIATE; + + CREATE TABLE IF NOT EXISTS llm_invocations ( + id TEXT PRIMARY KEY, + started_at TEXT NOT NULL, + finished_at TEXT, + duration_ms INTEGER, + caller_surface TEXT NOT NULL, + action TEXT NOT NULL, + provider TEXT NOT NULL, + model TEXT, + program TEXT, + incident_id TEXT, + ai_tool TEXT, + ai_project TEXT, + ai_session_id TEXT, + evidence_counts_json TEXT, + prompt_bytes INTEGER, + output_bytes INTEGER, + status TEXT NOT NULL, + error TEXT, + metadata_json TEXT + ); + + CREATE INDEX IF NOT EXISTS idx_llm_invocations_started + ON llm_invocations(started_at); + CREATE INDEX IF NOT EXISTS idx_llm_invocations_action_started + ON llm_invocations(action, started_at); + CREATE INDEX IF NOT EXISTS idx_llm_invocations_status_started + ON llm_invocations(status, started_at); + + INSERT OR IGNORE INTO schema_migrations (version) VALUES (37); + COMMIT;", + )?; + tracing::info!("Migration 37: created llm_invocations audit table"); + } + + // Migration 38: ai_skill_events — one row per detected skill invocation + // extracted from an AI transcript log row (Claude `attributionSkill` / + // `attributionPlugin` structured fields, Codex `` transcript + // tags). UNIQUE(log_id, skill_name, event_kind, evidence_kind) makes + // INSERT OR IGNORE idempotent across re-ingest and backfill re-runs. + // Eng review Fix 2: no skill_path/metadata_json — neither extractor sets + // them in this PR, so they are not part of the shipped schema. + // Eng review Fix 4: index set matches the actual shipped CLI filter + // surface (--skill, --plugin, --tool, --project, --session-id, --host, + // plus the unfiltered default `ORDER BY timestamp DESC`). + // Eng review Fix 5: idx_logs_ai_tool_id is added on the EXISTING `logs` + // table in this same migration batch — the backfill's `id > ?` keyset + // scan needs it and idx_logs_ai_tool_cover (ai_tool, ai_session_id, + // timestamp) doesn't include `id`. + if !migration_applied(&conn, 38)? { + conn.execute_batch( + "BEGIN IMMEDIATE; + + CREATE TABLE IF NOT EXISTS ai_skill_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + log_id INTEGER NOT NULL REFERENCES logs(id) ON DELETE CASCADE, + ai_tool TEXT NOT NULL, + ai_project TEXT, + ai_session_id TEXT, + hostname TEXT NOT NULL, + timestamp TEXT NOT NULL, + skill_name TEXT NOT NULL, + skill_plugin TEXT, + event_kind TEXT NOT NULL, + evidence_kind TEXT NOT NULL, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(log_id, skill_name, event_kind, evidence_kind) + ); + + CREATE INDEX IF NOT EXISTS idx_ai_skill_events_timestamp + ON ai_skill_events(timestamp); + CREATE INDEX IF NOT EXISTS idx_ai_skill_events_skill_time + ON ai_skill_events(skill_name, timestamp); + CREATE INDEX IF NOT EXISTS idx_ai_skill_events_plugin_time + ON ai_skill_events(skill_plugin, timestamp); + CREATE INDEX IF NOT EXISTS idx_ai_skill_events_hostname_time + ON ai_skill_events(hostname, timestamp); + CREATE INDEX IF NOT EXISTS idx_ai_skill_events_session_time + ON ai_skill_events(ai_tool, ai_project, ai_session_id, timestamp); + CREATE INDEX IF NOT EXISTS idx_ai_skill_events_project_skill_time + ON ai_skill_events(ai_project, skill_name, timestamp) + WHERE ai_project IS NOT NULL; + + CREATE INDEX IF NOT EXISTS idx_logs_ai_tool_id + ON logs(ai_tool, id) + WHERE ai_tool IN ('claude', 'codex'); + + INSERT OR IGNORE INTO schema_migrations (version) VALUES (38); + COMMIT;", + )?; + tracing::info!("Migration 38: created ai_skill_events table + idx_logs_ai_tool_id"); + } + + // Migration 39: ai_mcp_events — one row per normalized MCP/tool-call + // event extracted from an AI transcript log row (Claude `tool_use` / + // `tool_result` content items linked by `id`/`tool_use_id`; Codex + // `response_item.payload.type = "function_call"` / + // `"function_call_output"` linked by `payload.call_id`). Schema matches + // GH #94's "MCP assessment design" section verbatim. + // + // Idempotency key is `(ai_tool, ai_session_id, call_id, event_kind)`, + // enforced via an expression index over `COALESCE(ai_session_id, '')` + // rather than a plain `UNIQUE(...)` table constraint — SQLite (like + // standard SQL) never treats two NULLs as equal in a UNIQUE index, so a + // plain constraint on a nullable `ai_session_id` would silently let + // duplicate rows back in for sessionless transcripts (verified by a + // regression test in `mcp_events_tests.rs`). This makes `INSERT OR + // IGNORE` idempotent across re-ingest and backfill re-runs, mirroring + // the ai_skill_events idempotency pattern from migration 38 (whose own + // UNIQUE key is safe because its `log_id` column is NOT NULL). + // + // Index set is designed against the actual shipped query filter surface + // (search_ai_mcp_incidents groups/filters on mcp_server+mcp_tool+time, + // list_mcp_events filters on tool_name+time and the session tuple) — + // deliberately NOT copy-pasted blind from the skill_events index set, + // per the eng-review lesson called out in GH #104 (PR1/PR2 both shipped + // indexes that didn't match their query's actual filter/sort shape). + if !migration_applied(&conn, 39)? { + conn.execute_batch( + "BEGIN IMMEDIATE; + + CREATE TABLE IF NOT EXISTS ai_mcp_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + call_log_id INTEGER REFERENCES logs(id) ON DELETE CASCADE, + result_log_id INTEGER REFERENCES logs(id) ON DELETE SET NULL, + ai_tool TEXT NOT NULL, + ai_project TEXT, + ai_session_id TEXT, + hostname TEXT NOT NULL, + timestamp TEXT NOT NULL, + turn_id TEXT, + call_id TEXT NOT NULL, + tool_name TEXT NOT NULL, + mcp_server TEXT, + mcp_tool TEXT, + event_kind TEXT NOT NULL, + status TEXT, + duration_ms INTEGER, + is_error INTEGER, + arguments_json TEXT, + output_preview TEXT, + error_text TEXT, + metadata_json TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) + ); + + CREATE UNIQUE INDEX IF NOT EXISTS idx_ai_mcp_events_dedupe + ON ai_mcp_events(ai_tool, COALESCE(ai_session_id, ''), call_id, event_kind); + CREATE INDEX IF NOT EXISTS idx_ai_mcp_events_hostname_time + ON ai_mcp_events(hostname, timestamp); + CREATE INDEX IF NOT EXISTS idx_ai_mcp_events_tool_time + ON ai_mcp_events(tool_name, timestamp); + CREATE INDEX IF NOT EXISTS idx_ai_mcp_events_server_time + ON ai_mcp_events(mcp_server, timestamp) + WHERE mcp_server IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_ai_mcp_events_server_tool_time + ON ai_mcp_events(mcp_server, mcp_tool, timestamp) + WHERE mcp_server IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_ai_mcp_events_session_time + ON ai_mcp_events(ai_tool, ai_project, ai_session_id, timestamp); + CREATE INDEX IF NOT EXISTS idx_ai_mcp_events_error_time + ON ai_mcp_events(is_error, timestamp) + WHERE is_error = 1; + + INSERT OR IGNORE INTO schema_migrations (version) VALUES (39); + COMMIT;", + )?; + tracing::info!("Migration 39: created ai_mcp_events table"); + } + + // Migration 40: ai_hook_events — one row per detected hook signal, either + // a Claude runtime hook-execution attachment (`evidence_kind = + // 'runtime_transcript'`) or a Claude/Codex hook config-inventory / + // trust-state entry (`evidence_kind = 'config_inventory'` / + // `'trusted_hash_state'`). `log_id` is nullable because config-inventory + // rows are collected from local host config files, not a transcript log + // row — see GH #105's "Hook assessment design" section. + // + // Uniqueness is enforced via a UNIQUE INDEX over + // ai_tool, hostname, COALESCE(ai_session_id, ''), hook_event, + // COALESCE(hook_name, ''), timestamp, evidence_kind rather than a + // table-level UNIQUE(...) constraint: SQLite treats every NULL as + // distinct from every other NULL in a UNIQUE constraint, and + // config-inventory rows always have `ai_session_id = NULL` (they are + // host-global, not session-scoped) — a bare UNIQUE(ai_session_id, ...) + // would let `collect_and_store` insert an unbounded number of duplicate + // rows on every repeated collection instead of deduping via + // `INSERT OR IGNORE`. Wrapping the nullable columns in COALESCE collapses + // NULL to a consistent sentinel so repeated collections at the same + // hook_event/hook_name/timestamp/evidence_kind correctly dedupe. + // `hostname` is part of the key so two different hosts collecting + // identical config/trust-state rows at the same timestamp (both with + // `ai_session_id = NULL`) don't collide and silently drop one host's row. + if !migration_applied(&conn, 40)? { + conn.execute_batch( + "BEGIN IMMEDIATE; + + CREATE TABLE IF NOT EXISTS ai_hook_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + log_id INTEGER REFERENCES logs(id) ON DELETE SET NULL, + ai_tool TEXT NOT NULL, + ai_project TEXT, + ai_session_id TEXT, + hostname TEXT NOT NULL, + timestamp TEXT NOT NULL, + hook_event TEXT NOT NULL, + hook_name TEXT, + hook_source TEXT, + hook_command TEXT, + status TEXT NOT NULL, + exit_code INTEGER, + duration_ms INTEGER, + stdout_preview TEXT, + stderr_preview TEXT, + persisted_output_path TEXT, + trusted_hash TEXT, + evidence_kind TEXT NOT NULL, + metadata_json TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) + ); + + CREATE UNIQUE INDEX IF NOT EXISTS idx_ai_hook_events_unique + ON ai_hook_events( + ai_tool, + hostname, + COALESCE(ai_session_id, ''), + hook_event, + COALESCE(hook_name, ''), + timestamp, + evidence_kind + ); + + CREATE INDEX IF NOT EXISTS idx_ai_hook_events_hostname_time + ON ai_hook_events(hostname, timestamp); + CREATE INDEX IF NOT EXISTS idx_ai_hook_events_hook_time + ON ai_hook_events(hook_event, hook_name, timestamp); + CREATE INDEX IF NOT EXISTS idx_ai_hook_events_status_time + ON ai_hook_events(status, timestamp); + CREATE INDEX IF NOT EXISTS idx_ai_hook_events_session_time + ON ai_hook_events(ai_tool, ai_project, ai_session_id, timestamp); + CREATE INDEX IF NOT EXISTS idx_ai_hook_events_evidence_time + ON ai_hook_events(evidence_kind, timestamp); + + INSERT OR IGNORE INTO schema_migrations (version) VALUES (40); + COMMIT;", + )?; + tracing::info!("Migration 40: created ai_hook_events table"); + } + + // Migration 41: canonical entity-resolution graph contract + // (entity_resolution_v2). Adds the `logical_service` / `service_instance` + // entity types, the `instance_of` relationship type, and the three + // resolver reason codes to the constrained graph tables (rebuild, strict + // superset, ids preserved — mirrors migrations 33/34/35/36). The + // hard-break cutover for old populated DBs happens inside the copy + // itself: legacy `service` topology rows and nested `app` labels + // (`plex/plex/plex`) are excluded from the `INSERT … SELECT` (never + // migrated, no copy-then-delete), a `projection_contract` column records + // the active contract, and any previously-ready projection is marked + // stale so the next rebuild reprojects through the resolver. + if !migration_applied(&conn, 41)? { + conn.execute_batch(&format!( + "BEGIN IMMEDIATE; + + CREATE TABLE graph_entities_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entity_type TEXT NOT NULL CHECK (entity_type IN ( + 'host', 'container', 'service', 'app', 'source_ip', + 'ai_project', 'ai_session', 'error_signature', + 'compose_project', 'reverse_proxy', 'domain', 'network', + 'storage', 'config_artifact', 'git_commit', 'user', 'device', + 'logical_service', 'service_instance' + )), + canonical_key TEXT NOT NULL, + display_label TEXT NOT NULL, + source_kind TEXT NOT NULL DEFAULT '', + source_id TEXT NOT NULL DEFAULT '', + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated', 'refuted' + )), + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(entity_type, canonical_key) + ); + INSERT INTO graph_entities_new + (id, entity_type, canonical_key, display_label, source_kind, + source_id, trust_level, first_seen_at, last_seen_at, + created_at, updated_at) + SELECT id, entity_type, canonical_key, display_label, source_kind, + source_id, trust_level, first_seen_at, last_seen_at, + created_at, updated_at + FROM graph_entities + WHERE entity_type != 'service' + AND NOT (entity_type = 'app' AND canonical_key LIKE '%/%/%'); + DROP TABLE graph_entities; + ALTER TABLE graph_entities_new RENAME TO graph_entities; + CREATE INDEX idx_graph_entities_type_key + ON graph_entities(entity_type, canonical_key); + CREATE INDEX idx_graph_entities_canonical_key + ON graph_entities(canonical_key); + + CREATE TABLE graph_relationships_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relationship_key TEXT NOT NULL UNIQUE, + src_entity_id INTEGER NOT NULL, + dst_entity_id INTEGER NOT NULL, + relationship_type TEXT NOT NULL CHECK (relationship_type IN ( + 'observed_as', 'runs_on', 'emitted_by', 'worked_on', + 'matches_signature', 'defines_service', 'routes_to', + 'exposes_domain', 'attached_to', 'mounts', 'backed_by', + 'has_artifact', 'authenticated_as', 'accessed', + 'communicates_with', 'instance_of' + )), + reason_code TEXT NOT NULL CHECK (reason_code IN ( + 'syslog_claimed_hostname', 'log_app_name', + 'docker_container_id', 'docker_service_label', + 'ai_session_project', 'heartbeat_host_state', + 'error_signature_match', 'inventory_node', + 'inventory_service', 'compose_config', + 'reverse_proxy_config', 'docker_network', 'storage_probe', + 'config_artifact', 'agent_command_session', + 'agent_command_cwd_infer', 'agent_command_git_commit', + 'shell_history_git_commit', 'adguard_client_query', + 'shell_history_user', 'authelia_auth', + 'resolver_instance_of', 'resolver_service_instance', + 'resolver_raw_app_label' + )), + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated', 'refuted' + )), + confidence REAL NOT NULL DEFAULT 0.0 CHECK (confidence >= 0.0 AND confidence <= 1.0), + evidence_count INTEGER NOT NULL DEFAULT 0 CHECK (evidence_count >= 0), + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(src_entity_id, dst_entity_id, relationship_type, relationship_key) + ); + INSERT INTO graph_relationships_new + (id, relationship_key, src_entity_id, dst_entity_id, + relationship_type, reason_code, trust_level, confidence, + evidence_count, first_seen_at, last_seen_at, created_at, + updated_at) + SELECT id, relationship_key, src_entity_id, dst_entity_id, + relationship_type, reason_code, trust_level, confidence, + evidence_count, first_seen_at, last_seen_at, created_at, + updated_at + FROM graph_relationships + WHERE src_entity_id IN (SELECT id FROM graph_entities) + AND dst_entity_id IN (SELECT id FROM graph_entities); + DROP TABLE graph_relationships; + ALTER TABLE graph_relationships_new RENAME TO graph_relationships; + CREATE INDEX idx_graph_relationships_src_type_seen + ON graph_relationships(src_entity_id, relationship_type, last_seen_at DESC); + CREATE INDEX idx_graph_relationships_dst_type_seen + ON graph_relationships(dst_entity_id, relationship_type, last_seen_at DESC); + CREATE INDEX idx_graph_relationships_type_seen + ON graph_relationships(relationship_type, last_seen_at DESC); + + CREATE TABLE graph_relationship_evidence_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relationship_id INTEGER NOT NULL, + evidence_key TEXT NOT NULL, + source_kind TEXT NOT NULL CHECK (source_kind IN ( + 'log', 'heartbeat', 'ai_session_rollup', 'source_inventory', + 'app_inventory', 'error_signature' + )), + source_id TEXT NOT NULL DEFAULT '', + source_log_id INTEGER, + source_heartbeat_id INTEGER, + source_signature_hash TEXT, + observed_at TEXT NOT NULL, + reason_code TEXT NOT NULL CHECK (reason_code IN ( + 'syslog_claimed_hostname', 'log_app_name', + 'docker_container_id', 'docker_service_label', + 'ai_session_project', 'heartbeat_host_state', + 'error_signature_match', 'inventory_node', + 'inventory_service', 'compose_config', + 'reverse_proxy_config', 'docker_network', 'storage_probe', + 'config_artifact', 'agent_command_session', + 'agent_command_cwd_infer', 'agent_command_git_commit', + 'shell_history_git_commit', 'adguard_client_query', + 'shell_history_user', 'authelia_auth', + 'resolver_instance_of', 'resolver_service_instance', + 'resolver_raw_app_label' + )), + reason_text TEXT, + confidence_delta REAL NOT NULL DEFAULT 0.0 CHECK (confidence_delta >= -1.0 AND confidence_delta <= 1.0), + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated', 'refuted' + )), + safe_excerpt TEXT CHECK (safe_excerpt IS NULL OR length(safe_excerpt) <= 512), + metadata_path TEXT, + evidence_count INTEGER NOT NULL DEFAULT 1 CHECK (evidence_count >= 1), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(relationship_id, evidence_key) + ); + INSERT INTO graph_relationship_evidence_new + (id, relationship_id, evidence_key, source_kind, source_id, + source_log_id, source_heartbeat_id, source_signature_hash, + observed_at, reason_code, reason_text, confidence_delta, + trust_level, safe_excerpt, metadata_path, evidence_count, + created_at) + SELECT id, relationship_id, evidence_key, source_kind, source_id, + source_log_id, source_heartbeat_id, source_signature_hash, + observed_at, reason_code, reason_text, confidence_delta, + trust_level, safe_excerpt, metadata_path, evidence_count, + created_at + FROM graph_relationship_evidence + WHERE relationship_id IN (SELECT id FROM graph_relationships); + DROP TABLE graph_relationship_evidence; + ALTER TABLE graph_relationship_evidence_new RENAME TO graph_relationship_evidence; + CREATE INDEX idx_graph_evidence_relationship_seen + ON graph_relationship_evidence(relationship_id, observed_at DESC); + CREATE INDEX idx_graph_evidence_source_ref + ON graph_relationship_evidence(source_kind, source_id); + CREATE INDEX idx_graph_evidence_log_id + ON graph_relationship_evidence(source_log_id) + WHERE source_log_id IS NOT NULL; + CREATE INDEX idx_graph_evidence_heartbeat_id + ON graph_relationship_evidence(source_heartbeat_id) + WHERE source_heartbeat_id IS NOT NULL; + + DELETE FROM graph_entity_aliases + WHERE entity_id NOT IN (SELECT id FROM graph_entities); + + ALTER TABLE graph_projection_meta + ADD COLUMN projection_contract TEXT NOT NULL DEFAULT '{contract_v2}'; + UPDATE graph_projection_meta + SET projection_status = 'stale', + updated_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now') + WHERE id = 1 AND projection_status = 'ready'; + + INSERT OR IGNORE INTO schema_migrations (version) VALUES (41); + COMMIT;", + contract_v2 = crate::entity_resolution::vocab::GRAPH_PROJECTION_CONTRACT_V2, + ))?; + tracing::info!( + contract_key = crate::entity_resolution::vocab::GRAPH_PROJECTION_CONTRACT_KEY, + contract = crate::entity_resolution::vocab::GRAPH_PROJECTION_CONTRACT_V2, + "Migration 41: canonical entity-resolution graph contract" + ); + } + + // Migration 42: add the `refuted` trust level to graph_entity_aliases' + // trust_level CHECK. Migrations 35 and 41 added `refuted` to + // graph_entities, graph_relationships, and graph_relationship_evidence, + // but graph_entity_aliases was missed — an alias write at `refuted` + // trust fails this CHECK. Rebuilds the constrained table; strict + // superset, ids preserved (mirrors migrations 33/34/35/36/41). + if !migration_applied(&conn, 42)? { + conn.execute_batch( + "BEGIN IMMEDIATE; + + CREATE TABLE graph_entity_aliases_new ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entity_id INTEGER NOT NULL, + alias_type TEXT NOT NULL, + alias_key TEXT NOT NULL, + alias_value TEXT NOT NULL, + source_kind TEXT NOT NULL DEFAULT '', + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated', 'refuted' + )), + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(entity_id, alias_type, alias_key, source_kind) + ); + INSERT INTO graph_entity_aliases_new + (id, entity_id, alias_type, alias_key, alias_value, source_kind, + trust_level, first_seen_at, last_seen_at, created_at, updated_at) + SELECT id, entity_id, alias_type, alias_key, alias_value, source_kind, + trust_level, first_seen_at, last_seen_at, created_at, updated_at + FROM graph_entity_aliases; + DROP TABLE graph_entity_aliases; + ALTER TABLE graph_entity_aliases_new RENAME TO graph_entity_aliases; + CREATE INDEX idx_graph_aliases_lookup + ON graph_entity_aliases(alias_type, alias_key); + CREATE INDEX idx_graph_aliases_entity + ON graph_entity_aliases(entity_id); + + INSERT OR IGNORE INTO schema_migrations (version) VALUES (42); + COMMIT;", + )?; + tracing::info!("Migration 42: added refuted trust level to graph_entity_aliases"); + } + + // Migration 43: `stream_last_seen` — one row per (hostname, source_kind), + // maintained by the notification evaluator each cycle. Foundation for the + // stream_silence rule: alerting needs "newest row per host + source kind" + // and the logs table cannot answer that cheaply (source kind lives inside + // metadata_json). No backfill here — the evaluator seeds the table from a + // bounded window on its first cycle, keeping migration time flat. + if !migration_applied(&conn, 43)? { + conn.execute_batch( + "BEGIN IMMEDIATE; + CREATE TABLE IF NOT EXISTS stream_last_seen ( + hostname TEXT NOT NULL, + source_kind TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + PRIMARY KEY (hostname, source_kind) + ) WITHOUT ROWID; + INSERT OR IGNORE INTO schema_migrations (version) VALUES (43); + COMMIT;", + )?; + tracing::info!("Migration 43: stream_last_seen rollup for stream-silence alerting"); + } + + // Migration 44: Agent Observatory repository, worktree, observation, + // and exact-commit topology. The DDL and version marker share one + // transaction so startup never reports a partially applied migration. + if !migration_applied(&conn, 44)? { + conn.execute_batch( + "BEGIN IMMEDIATE; + CREATE TABLE IF NOT EXISTS repositories ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + repository_key TEXT NOT NULL UNIQUE, + hostname TEXT NOT NULL, + common_git_dir TEXT NOT NULL, + primary_path TEXT NOT NULL, + display_name TEXT NOT NULL, + remote_url_hash TEXT, + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + removed_at TEXT, + metadata_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(metadata_json)), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(hostname, common_git_dir) + ); + CREATE INDEX IF NOT EXISTS idx_repositories_host_seen + ON repositories(hostname, last_seen_at DESC); + CREATE INDEX IF NOT EXISTS idx_repositories_display + ON repositories(display_name COLLATE NOCASE); + + CREATE TABLE IF NOT EXISTS repository_worktrees ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + worktree_key TEXT NOT NULL UNIQUE, + repository_id INTEGER NOT NULL REFERENCES repositories(id) ON DELETE CASCADE, + hostname TEXT NOT NULL, + path TEXT NOT NULL, + git_dir TEXT NOT NULL, + branch_ref TEXT, + branch_name TEXT, + head_sha TEXT, + upstream_ref TEXT, + detached INTEGER NOT NULL DEFAULT 0 CHECK (detached IN (0, 1)), + bare INTEGER NOT NULL DEFAULT 0 CHECK (bare IN (0, 1)), + locked INTEGER NOT NULL DEFAULT 0 CHECK (locked IN (0, 1)), + lock_reason TEXT, + prunable INTEGER NOT NULL DEFAULT 0 CHECK (prunable IN (0, 1)), + prune_reason TEXT, + dirty INTEGER NOT NULL DEFAULT 0 CHECK (dirty IN (0, 1)), + staged_count INTEGER NOT NULL DEFAULT 0 CHECK (staged_count >= 0), + unstaged_count INTEGER NOT NULL DEFAULT 0 CHECK (unstaged_count >= 0), + untracked_count INTEGER NOT NULL DEFAULT 0 CHECK (untracked_count >= 0), + ahead INTEGER CHECK (ahead IS NULL OR ahead >= 0), + behind INTEGER CHECK (behind IS NULL OR behind >= 0), + status_hash TEXT, + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + removed_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(hostname, path) + ); + CREATE INDEX IF NOT EXISTS idx_worktrees_repo_active + ON repository_worktrees(repository_id, removed_at, last_seen_at DESC); + CREATE INDEX IF NOT EXISTS idx_worktrees_branch + ON repository_worktrees(branch_name, last_seen_at DESC); + CREATE INDEX IF NOT EXISTS idx_worktrees_head + ON repository_worktrees(repository_id, head_sha); + + CREATE TABLE IF NOT EXISTS repository_observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + observation_key TEXT NOT NULL UNIQUE, + repository_id INTEGER NOT NULL REFERENCES repositories(id) ON DELETE CASCADE, + worktree_id INTEGER REFERENCES repository_worktrees(id) ON DELETE CASCADE, + observed_at TEXT NOT NULL, + observation_kind TEXT NOT NULL CHECK (observation_kind IN ( + 'discovered', 'status', 'head', 'branch', 'worktree_added', + 'worktree_removed', 'overflow_reconcile', 'periodic_reconcile', 'error' + )), + old_head_sha TEXT, + new_head_sha TEXT, + summary TEXT NOT NULL DEFAULT '', + payload_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(payload_json)), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) + ); + CREATE INDEX IF NOT EXISTS idx_repository_observations_worktree_time + ON repository_observations(worktree_id, observed_at DESC, id DESC); + CREATE INDEX IF NOT EXISTS idx_repository_observations_repo_time + ON repository_observations(repository_id, observed_at DESC, id DESC); + + CREATE TABLE IF NOT EXISTS git_commits ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + repository_id INTEGER NOT NULL REFERENCES repositories(id) ON DELETE CASCADE, + sha TEXT NOT NULL, + parent_shas_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(parent_shas_json)), + author_name TEXT, + author_email_hash TEXT, + authored_at TEXT, + committed_at TEXT, + subject TEXT NOT NULL DEFAULT '', + changed_files INTEGER CHECK (changed_files IS NULL OR changed_files >= 0), + insertions INTEGER CHECK (insertions IS NULL OR insertions >= 0), + deletions INTEGER CHECK (deletions IS NULL OR deletions >= 0), + changed_paths_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(changed_paths_json)), + first_observed_at TEXT NOT NULL, + last_observed_at TEXT NOT NULL, + reachable INTEGER NOT NULL DEFAULT 1 CHECK (reachable IN (0, 1)), + metadata_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(metadata_json)), + UNIQUE(repository_id, sha) + ); + CREATE INDEX IF NOT EXISTS idx_git_commits_repo_time + ON git_commits(repository_id, committed_at DESC, id DESC); + INSERT OR IGNORE INTO schema_migrations (version) VALUES (44); + COMMIT;", + )?; + tracing::info!("Migration 44: Agent Observatory repository topology"); + } + + // Agent Observatory migration 45: run events, evidence, cursors, and outbox. + // This migration is wrapped in a transaction with the version marker. + let migration_45_applied: bool = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 45", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap_or(0) + > 0; + + if !migration_45_applied { + conn.execute_batch( + "BEGIN IMMEDIATE; + + CREATE TABLE IF NOT EXISTS agent_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_key TEXT NOT NULL UNIQUE, + native_session_id TEXT NOT NULL, + tool TEXT NOT NULL, + provider_tool TEXT, + hostname TEXT NOT NULL, + parent_run_id INTEGER REFERENCES agent_runs(id) ON DELETE SET NULL, + previous_run_id INTEGER REFERENCES agent_runs(id) ON DELETE SET NULL, + primary_worktree_id INTEGER REFERENCES repository_worktrees(id) ON DELETE SET NULL, + transcript_path TEXT, + process_id TEXT, + status TEXT NOT NULL CHECK (status IN ( + 'starting', 'active', 'waiting', 'idle', 'stale', + 'completed', 'failed', 'abandoned' + )), + status_reason TEXT NOT NULL DEFAULT '', + status_observed_at TEXT NOT NULL, + started_at TEXT NOT NULL, + last_activity_at TEXT NOT NULL, + ended_at TEXT, + first_source_log_id INTEGER, + last_source_log_id INTEGER, + last_event_id INTEGER, + event_count INTEGER NOT NULL DEFAULT 0 CHECK (event_count >= 0), + error_count INTEGER NOT NULL DEFAULT 0 CHECK (error_count >= 0), + primary_branch TEXT, + start_head_sha TEXT, + current_head_sha TEXT, + projection_version INTEGER NOT NULL DEFAULT 1, + freshness_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(freshness_json)), + metadata_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(metadata_json)), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(hostname, tool, native_session_id) + ); + CREATE INDEX IF NOT EXISTS idx_agent_runs_activity + ON agent_runs(last_activity_at DESC, id DESC); + CREATE INDEX IF NOT EXISTS idx_agent_runs_status_activity + ON agent_runs(status, last_activity_at DESC, id DESC); + CREATE INDEX IF NOT EXISTS idx_agent_runs_worktree_activity + ON agent_runs(primary_worktree_id, last_activity_at DESC, id DESC); + CREATE INDEX IF NOT EXISTS idx_agent_runs_tool_host + ON agent_runs(tool, hostname, last_activity_at DESC); + + CREATE TABLE IF NOT EXISTS agent_run_actors ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + actor_key TEXT NOT NULL UNIQUE, + run_id INTEGER NOT NULL REFERENCES agent_runs(id) ON DELETE CASCADE, + native_actor_id TEXT NOT NULL, + actor_type TEXT, + display_name TEXT, + started_at TEXT, + last_activity_at TEXT, + ended_at TEXT, + metadata_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(metadata_json)), + UNIQUE(run_id, native_actor_id) + ); + CREATE INDEX IF NOT EXISTS idx_agent_run_actors_run + ON agent_run_actors(run_id, last_activity_at DESC); + + CREATE TABLE IF NOT EXISTS agent_run_worktrees ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relation_key TEXT NOT NULL UNIQUE, + run_id INTEGER NOT NULL REFERENCES agent_runs(id) ON DELETE CASCADE, + worktree_id INTEGER NOT NULL REFERENCES repository_worktrees(id) ON DELETE CASCADE, + evidence_kind TEXT NOT NULL, + evidence_source TEXT NOT NULL, + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'correlated', 'inferred', 'refuted' + )), + confidence REAL NOT NULL CHECK (confidence >= 0.0 AND confidence <= 1.0), + is_primary INTEGER NOT NULL DEFAULT 0 CHECK (is_primary IN (0, 1)), + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + metadata_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(metadata_json)), + UNIQUE(run_id, worktree_id, evidence_kind, evidence_source) + ); + CREATE INDEX IF NOT EXISTS idx_agent_run_worktrees_run + ON agent_run_worktrees(run_id, is_primary DESC, confidence DESC, last_seen_at DESC); + CREATE INDEX IF NOT EXISTS idx_agent_run_worktrees_worktree + ON agent_run_worktrees(worktree_id, last_seen_at DESC, run_id); + + CREATE TABLE IF NOT EXISTS agent_run_events ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + event_key TEXT NOT NULL UNIQUE, + run_id INTEGER NOT NULL REFERENCES agent_runs(id) ON DELETE CASCADE, + actor_id INTEGER REFERENCES agent_run_actors(id) ON DELETE SET NULL, + worktree_id INTEGER REFERENCES repository_worktrees(id) ON DELETE SET NULL, + commit_id INTEGER REFERENCES git_commits(id) ON DELETE SET NULL, + observed_at TEXT NOT NULL, + ingested_at TEXT NOT NULL, + event_kind TEXT NOT NULL CHECK (event_kind IN ( + 'lifecycle', 'transcript', 'command', 'shell_history', + 'git_status', 'git_head', 'git_commit', 'file_operation', + 'mcp', 'hook', 'skill', 'llm', 'otlp_log', 'otlp_span', + 'otlp_metric', 'heartbeat', 'error', 'provider_event' + )), + source_kind TEXT NOT NULL, + source_id TEXT NOT NULL, + source_log_id INTEGER, + provider_sequence INTEGER, + trace_id TEXT, + span_id TEXT, + severity TEXT NOT NULL DEFAULT 'info', + title TEXT NOT NULL DEFAULT '', + summary TEXT NOT NULL DEFAULT '', + payload_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(payload_json)), + content_scrubbed INTEGER NOT NULL DEFAULT 1 CHECK (content_scrubbed IN (0, 1)), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) + ); + CREATE INDEX IF NOT EXISTS idx_agent_run_events_run_order + ON agent_run_events(run_id, observed_at DESC, id DESC); + CREATE INDEX IF NOT EXISTS idx_agent_run_events_run_kind + ON agent_run_events(run_id, event_kind, observed_at DESC, id DESC); + CREATE INDEX IF NOT EXISTS idx_agent_run_events_trace + ON agent_run_events(trace_id, span_id); + CREATE INDEX IF NOT EXISTS idx_agent_run_events_source_log + ON agent_run_events(source_log_id) WHERE source_log_id IS NOT NULL; + + CREATE TABLE IF NOT EXISTS agent_run_commits ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relation_key TEXT NOT NULL UNIQUE, + run_id INTEGER NOT NULL REFERENCES agent_runs(id) ON DELETE CASCADE, + commit_id INTEGER NOT NULL REFERENCES git_commits(id) ON DELETE CASCADE, + worktree_id INTEGER REFERENCES repository_worktrees(id) ON DELETE SET NULL, + evidence_kind TEXT NOT NULL, + evidence_source TEXT NOT NULL, + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'correlated', 'inferred', 'refuted' + )), + confidence REAL NOT NULL CHECK (confidence >= 0.0 AND confidence <= 1.0), + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + metadata_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(metadata_json)), + UNIQUE(run_id, commit_id, evidence_kind, evidence_source) + ); + CREATE INDEX IF NOT EXISTS idx_agent_run_commits_run + ON agent_run_commits(run_id, last_seen_at DESC, id DESC); + CREATE INDEX IF NOT EXISTS idx_agent_run_commits_commit + ON agent_run_commits(commit_id, last_seen_at DESC, run_id); + + CREATE TABLE IF NOT EXISTS agent_projection_cursors ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + cursor_type TEXT NOT NULL, + source_name TEXT NOT NULL DEFAULT 'default', + cursor_value TEXT NOT NULL DEFAULT '', + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(cursor_type, source_name) + ); + CREATE INDEX IF NOT EXISTS idx_agent_projection_cursors_type + ON agent_projection_cursors(cursor_type, source_name); + + INSERT OR IGNORE INTO agent_projection_cursors (cursor_type, source_name, cursor_value) VALUES + ('repositories', 'default', ''), + ('repository_worktrees', 'default', ''), + ('repository_observations', 'default', ''), + ('git_commits', 'default', ''), + ('agent_runs', 'default', ''), + ('agent_run_events', 'default', ''), + ('otel_spans', 'default', ''), + ('otel_metric_points', 'default', ''); + + CREATE TABLE IF NOT EXISTS agent_stream_outbox ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + outbox_key TEXT NOT NULL UNIQUE, + run_id INTEGER NOT NULL REFERENCES agent_runs(id) ON DELETE CASCADE, + stream_event_type TEXT NOT NULL, + expires_at TEXT NOT NULL, + payload_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(payload_json)), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) + ); + CREATE INDEX IF NOT EXISTS idx_agent_stream_outbox_run + ON agent_stream_outbox(run_id, id ASC); + CREATE INDEX IF NOT EXISTS idx_agent_stream_outbox_expiry + ON agent_stream_outbox(expires_at ASC); + + INSERT OR IGNORE INTO schema_migrations (version) VALUES (45); + COMMIT;", + )?; + tracing::info!("Migration 45: Agent Observatory run events, evidence, cursors, and outbox"); + } + + // Migration 46: OTLP traces. + // The DDL and version marker share one transaction for atomicity. + if !migration_applied(&conn, 46)? { + conn.execute_batch( + "BEGIN IMMEDIATE; + + CREATE TABLE IF NOT EXISTS otel_spans ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + trace_id TEXT NOT NULL CHECK (length(trace_id) = 32), + span_id TEXT NOT NULL CHECK (length(span_id) = 16), + parent_span_id TEXT CHECK (parent_span_id IS NULL OR length(parent_span_id) = 16), + trace_state TEXT, + flags INTEGER NOT NULL DEFAULT 0, + span_name TEXT NOT NULL, + span_kind INTEGER NOT NULL, + start_time_unix_nano INTEGER NOT NULL, + end_time_unix_nano INTEGER NOT NULL, + duration_nano INTEGER NOT NULL CHECK (duration_nano >= 0), + status_code INTEGER NOT NULL DEFAULT 0, + status_message TEXT, + hostname TEXT NOT NULL DEFAULT '', + service_name TEXT, + service_version TEXT, + scope_name TEXT, + scope_version TEXT, + ai_tool TEXT, + ai_project TEXT, + ai_session_id TEXT, + run_id INTEGER REFERENCES agent_runs(id) ON DELETE SET NULL, + resource_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(resource_json)), + attributes_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(attributes_json)), + events_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(events_json)), + links_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(links_json)), + received_at TEXT NOT NULL, + content_scrubbed INTEGER NOT NULL DEFAULT 1 CHECK (content_scrubbed IN (0, 1)), + UNIQUE(trace_id, span_id) + ); + CREATE INDEX IF NOT EXISTS idx_otel_spans_run_time + ON otel_spans(run_id, start_time_unix_nano DESC, id DESC); + CREATE INDEX IF NOT EXISTS idx_otel_spans_session_time + ON otel_spans(hostname, ai_tool, ai_session_id, start_time_unix_nano DESC); + CREATE INDEX IF NOT EXISTS idx_otel_spans_trace + ON otel_spans(trace_id, start_time_unix_nano, span_id); + CREATE INDEX IF NOT EXISTS idx_otel_spans_service_time + ON otel_spans(service_name, start_time_unix_nano DESC); + + INSERT OR IGNORE INTO schema_migrations (version) VALUES (46); + COMMIT;", + )?; + tracing::info!("Migration 46: OTLP traces"); + } + + // Agent Observatory migration 47: OTLP metric points. + // This migration is wrapped in a transaction with the version marker. + let migration_47_applied: bool = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 47", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap_or(0) + > 0; + + if !migration_47_applied { + conn.execute_batch( + "BEGIN IMMEDIATE; + + CREATE TABLE IF NOT EXISTS otel_metric_points ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + point_key TEXT NOT NULL UNIQUE, + metric_name TEXT NOT NULL, + description TEXT NOT NULL DEFAULT '', + unit TEXT NOT NULL DEFAULT '', + instrument_kind TEXT NOT NULL CHECK (instrument_kind IN ( + 'gauge', 'sum', 'histogram', 'exponential_histogram', 'summary' + )), + aggregation_temporality INTEGER, + monotonic INTEGER CHECK (monotonic IS NULL OR monotonic IN (0, 1)), + start_time_unix_nano INTEGER, + time_unix_nano INTEGER NOT NULL, + hostname TEXT NOT NULL DEFAULT '', + service_name TEXT, + service_version TEXT, + scope_name TEXT, + scope_version TEXT, + ai_tool TEXT, + ai_project TEXT, + ai_session_id TEXT, + run_id INTEGER REFERENCES agent_runs(id) ON DELETE SET NULL, + resource_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(resource_json)), + attributes_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(attributes_json)), + value_json TEXT NOT NULL CHECK (json_valid(value_json)), + exemplars_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(exemplars_json)), + received_at TEXT NOT NULL, + content_scrubbed INTEGER NOT NULL DEFAULT 1 CHECK (content_scrubbed IN (0, 1)) + ); + + CREATE INDEX IF NOT EXISTS idx_otel_metric_points_run_time + ON otel_metric_points(run_id, time_unix_nano DESC, id DESC); + CREATE INDEX IF NOT EXISTS idx_otel_metric_points_name_time + ON otel_metric_points(metric_name, time_unix_nano DESC, id DESC); + CREATE INDEX IF NOT EXISTS idx_otel_metric_points_session_time + ON otel_metric_points(hostname, ai_tool, ai_session_id, time_unix_nano DESC); + + INSERT OR IGNORE INTO schema_migrations (version) VALUES (47); + COMMIT;", + )?; + tracing::info!("Migration 47: OTLP metric points"); + } + + if table_exists(&conn, "host_heartbeats")? && table_exists(&conn, "host_heartbeats_latest")? { + let deleted_heartbeat_latest = conn.execute( + "DELETE FROM host_heartbeats_latest + WHERE NOT EXISTS ( + SELECT 1 FROM host_heartbeats + WHERE host_heartbeats.id = host_heartbeats_latest.heartbeat_id + )", + [], + )?; + if deleted_heartbeat_latest > 0 { + tracing::info!( + deleted_rows = deleted_heartbeat_latest, + "Reconciled orphan heartbeat latest cache rows" + ); + } + } + + conn.execute_batch( + "CREATE INDEX IF NOT EXISTS idx_logs_ai_project_time + ON logs(ai_project, timestamp) + WHERE ai_project IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_logs_ai_session + ON logs(ai_tool, ai_project, ai_session_id) + WHERE ai_tool IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_logs_ai_session_host_time + ON logs(ai_project, ai_tool, ai_session_id, hostname, timestamp) + WHERE ai_project IS NOT NULL + AND ai_tool IS NOT NULL + AND ai_session_id IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_logs_ai_transcript_path + ON logs(ai_transcript_path) + WHERE ai_transcript_path IS NOT NULL;", + )?; + + tracing::info!(path = %config.db_path.display(), "Database initialized"); + Ok(pool) +} + +/// Reconcile work that could only remain `running` after the authoritative +/// server process exited. Query-only CLI processes may open the same live +/// database concurrently, so this must be called by server startup rather +/// than by [`init_pool`]. +pub fn reconcile_interrupted_server_work(pool: &DbPool) -> Result<()> { + let conn = pool.get()?; + let _write_guard = write_lock(); + conn.execute_batch( + "UPDATE maintenance_jobs + SET status = 'failed', + finished_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), + result_json = json_object('error', \"interrupted by server restart\") + WHERE status = 'running'; + + UPDATE llm_invocations + SET status = 'interrupted', + finished_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), + error = 'interrupted by server restart' + WHERE status = 'running';", + )?; + Ok(()) +} + +fn table_exists(conn: &Connection, table: &str) -> Result { + let count: i64 = conn.query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1", + [table], + |row| row.get(0), + )?; + Ok(count > 0) +} + +fn migration_applied(conn: &Connection, version: i64) -> rusqlite::Result { + conn.query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = ?1", + [version], + |row| row.get::<_, i64>(0), + ) + .map(|count| count > 0) +} + +fn column_exists(conn: &Connection, table: &str, column: &str) -> rusqlite::Result { + conn.query_row( + "SELECT COUNT(*) FROM pragma_table_info(?1) WHERE name = ?2", + [table, column], + |row| row.get::<_, i64>(0), + ) + .map(|count| count > 0) +} + +fn add_column_if_missing( + conn: &Connection, + table: &str, + column: &str, + column_type: &str, +) -> rusqlite::Result<()> { + if !column_exists(conn, table, column)? { + conn.execute_batch(&format!( + "ALTER TABLE {table} ADD COLUMN {column} {column_type};" + ))?; + } + Ok(()) +} + +fn apply_migration_17_inventory_stats(conn: &Connection) -> rusqlite::Result<()> { + conn.execute_batch( + "BEGIN IMMEDIATE; + + CREATE TABLE IF NOT EXISTS app_inventory_stats ( + app_name TEXT PRIMARY KEY, + log_count INTEGER NOT NULL DEFAULT 0, + first_seen TEXT NOT NULL, + last_seen TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_app_inventory_last_seen + ON app_inventory_stats(last_seen DESC, app_name ASC); + + CREATE TABLE IF NOT EXISTS app_host_inventory_stats ( + app_name TEXT NOT NULL, + hostname TEXT NOT NULL, + log_count INTEGER NOT NULL DEFAULT 0, + first_seen TEXT NOT NULL, + last_seen TEXT NOT NULL, + PRIMARY KEY (app_name, hostname) + ); + CREATE INDEX IF NOT EXISTS idx_app_host_inventory_count + ON app_host_inventory_stats(app_name, log_count DESC, hostname ASC); + + CREATE TABLE IF NOT EXISTS source_ip_inventory_stats ( + source_ip TEXT PRIMARY KEY, + log_count INTEGER NOT NULL DEFAULT 0, + first_seen TEXT NOT NULL, + last_seen TEXT NOT NULL + ); + CREATE INDEX IF NOT EXISTS idx_source_ip_inventory_count + ON source_ip_inventory_stats(log_count DESC, source_ip ASC); + + CREATE TABLE IF NOT EXISTS source_ip_host_inventory_stats ( + source_ip TEXT NOT NULL, + hostname TEXT NOT NULL, + log_count INTEGER NOT NULL DEFAULT 0, + first_seen TEXT NOT NULL, + last_seen TEXT NOT NULL, + PRIMARY KEY (source_ip, hostname) + ); + CREATE INDEX IF NOT EXISTS idx_source_ip_host_inventory_count + ON source_ip_host_inventory_stats(source_ip, log_count DESC, hostname ASC); + + CREATE TABLE IF NOT EXISTS inventory_backfill_state ( + name TEXT PRIMARY KEY, + completed_at TEXT, + last_error TEXT, + last_log_id INTEGER NOT NULL DEFAULT 0, + high_watermark_id INTEGER + ); + INSERT OR IGNORE INTO inventory_backfill_state(name) + VALUES ('app_source_inventory'); + + DROP TRIGGER IF EXISTS logs_inventory_app_ai; + DROP TRIGGER IF EXISTS logs_inventory_app_ad; + DROP TRIGGER IF EXISTS logs_inventory_source_ip_ai; + DROP TRIGGER IF EXISTS logs_inventory_source_ip_ad; + + CREATE TRIGGER logs_inventory_app_ai AFTER INSERT ON logs + WHEN NEW.app_name IS NOT NULL AND NEW.app_name != '' + BEGIN + INSERT INTO app_inventory_stats(app_name, log_count, first_seen, last_seen) + VALUES (NEW.app_name, 1, NEW.received_at, NEW.received_at) + ON CONFLICT(app_name) DO UPDATE SET + log_count = log_count + 1, + first_seen = min(first_seen, excluded.first_seen), + last_seen = max(last_seen, excluded.last_seen); + + INSERT INTO app_host_inventory_stats(app_name, hostname, log_count, first_seen, last_seen) + VALUES (NEW.app_name, NEW.hostname, 1, NEW.received_at, NEW.received_at) + ON CONFLICT(app_name, hostname) DO UPDATE SET + log_count = log_count + 1, + first_seen = min(first_seen, excluded.first_seen), + last_seen = max(last_seen, excluded.last_seen); + END; + + CREATE TRIGGER logs_inventory_app_ad AFTER DELETE ON logs + WHEN OLD.app_name IS NOT NULL AND OLD.app_name != '' + BEGIN + UPDATE app_inventory_stats + SET log_count = log_count - 1 + WHERE app_name = OLD.app_name; + DELETE FROM app_inventory_stats + WHERE app_name = OLD.app_name AND log_count <= 0; + + UPDATE app_host_inventory_stats + SET log_count = log_count - 1 + WHERE app_name = OLD.app_name AND hostname = OLD.hostname; + DELETE FROM app_host_inventory_stats + WHERE app_name = OLD.app_name AND hostname = OLD.hostname AND log_count <= 0; + END; + + CREATE TRIGGER logs_inventory_source_ip_ai AFTER INSERT ON logs + WHEN NEW.source_ip != '' + BEGIN + INSERT INTO source_ip_inventory_stats(source_ip, log_count, first_seen, last_seen) + VALUES (NEW.source_ip, 1, NEW.received_at, NEW.received_at) + ON CONFLICT(source_ip) DO UPDATE SET + log_count = log_count + 1, + first_seen = min(first_seen, excluded.first_seen), + last_seen = max(last_seen, excluded.last_seen); + + INSERT INTO source_ip_host_inventory_stats(source_ip, hostname, log_count, first_seen, last_seen) + VALUES (NEW.source_ip, NEW.hostname, 1, NEW.received_at, NEW.received_at) + ON CONFLICT(source_ip, hostname) DO UPDATE SET + log_count = log_count + 1, + first_seen = min(first_seen, excluded.first_seen), + last_seen = max(last_seen, excluded.last_seen); + END; + + CREATE TRIGGER logs_inventory_source_ip_ad AFTER DELETE ON logs + WHEN OLD.source_ip != '' + BEGIN + UPDATE source_ip_inventory_stats + SET log_count = log_count - 1 + WHERE source_ip = OLD.source_ip; + DELETE FROM source_ip_inventory_stats + WHERE source_ip = OLD.source_ip AND log_count <= 0; + + UPDATE source_ip_host_inventory_stats + SET log_count = log_count - 1 + WHERE source_ip = OLD.source_ip AND hostname = OLD.hostname; + DELETE FROM source_ip_host_inventory_stats + WHERE source_ip = OLD.source_ip AND hostname = OLD.hostname AND log_count <= 0; + END; + + INSERT OR IGNORE INTO schema_migrations (version) VALUES (17); + COMMIT;", + )?; + tracing::info!("Migration 17: created app/source inventory stats tables and triggers"); + Ok(()) +} + +pub fn inventory_backfill_complete(pool: &DbPool) -> Result { + let conn = pool.get()?; + let complete = conn.query_row( + "SELECT completed_at IS NOT NULL + FROM inventory_backfill_state + WHERE name = 'app_source_inventory'", + [], + |row| row.get::<_, bool>(0), + )?; + Ok(complete) +} + +fn ensure_inventory_backfill_state_columns(conn: &Connection) -> rusqlite::Result<()> { + add_column_if_missing( + conn, + "inventory_backfill_state", + "last_log_id", + "INTEGER NOT NULL DEFAULT 0", + )?; + add_column_if_missing( + conn, + "inventory_backfill_state", + "high_watermark_id", + "INTEGER", + )?; + Ok(()) +} + +pub fn backfill_inventory_stats(pool: &DbPool) -> Result<()> { + const CHUNK_SIZE: i64 = 25_000; + const BETWEEN_CHUNKS: std::time::Duration = std::time::Duration::from_millis(25); + + if inventory_backfill_complete(pool)? { + return Ok(()); + } + let conn = pool.get()?; + ensure_inventory_backfill_state_columns(&conn)?; + tracing::info!( + "Inventory stats backfill starting — queries may fall back to logs until this completes" + ); + let started = std::time::Instant::now(); + + loop { + let (last_log_id, high_watermark_id): (i64, Option) = conn.query_row( + "SELECT last_log_id, high_watermark_id + FROM inventory_backfill_state + WHERE name = 'app_source_inventory'", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + )?; + let high_watermark_id = match high_watermark_id { + Some(id) => id, + None => { + let high: i64 = + conn.query_row("SELECT COALESCE(MAX(id), 0) FROM logs", [], |row| { + row.get(0) + })?; + conn.execute_batch( + "BEGIN IMMEDIATE; + DELETE FROM app_inventory_stats; + DELETE FROM app_host_inventory_stats; + DELETE FROM source_ip_inventory_stats; + DELETE FROM source_ip_host_inventory_stats;", + )?; + conn.execute( + "UPDATE inventory_backfill_state + SET last_log_id = 0, + high_watermark_id = ?1, + completed_at = NULL, + last_error = NULL + WHERE name = 'app_source_inventory'", + [high], + )?; + conn.execute_batch("COMMIT;")?; + high + } + }; + + if last_log_id >= high_watermark_id { + conn.execute( + "UPDATE inventory_backfill_state + SET completed_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), + last_error = NULL + WHERE name = 'app_source_inventory'", + [], + )?; + tracing::info!( + elapsed_ms = started.elapsed().as_millis(), + high_watermark_id, + "Inventory stats backfill completed" + ); + return Ok(()); + } + + let next_log_id = (last_log_id + CHUNK_SIZE).min(high_watermark_id); + conn.execute_batch("BEGIN IMMEDIATE;")?; + let result = (|| -> rusqlite::Result<()> { + conn.execute( + "INSERT INTO app_inventory_stats(app_name, log_count, first_seen, last_seen) + SELECT app_name, COUNT(*), MIN(received_at), MAX(received_at) + FROM logs + WHERE id > ?1 + AND id <= ?2 + AND app_name IS NOT NULL + AND app_name != '' + GROUP BY app_name + ON CONFLICT(app_name) DO UPDATE SET + log_count = log_count + excluded.log_count, + first_seen = min(first_seen, excluded.first_seen), + last_seen = max(last_seen, excluded.last_seen)", + (last_log_id, next_log_id), + )?; + conn.execute( + "INSERT INTO app_host_inventory_stats(app_name, hostname, log_count, first_seen, last_seen) + SELECT app_name, hostname, COUNT(*), MIN(received_at), MAX(received_at) + FROM logs + WHERE id > ?1 + AND id <= ?2 + AND app_name IS NOT NULL + AND app_name != '' + GROUP BY app_name, hostname + ON CONFLICT(app_name, hostname) DO UPDATE SET + log_count = log_count + excluded.log_count, + first_seen = min(first_seen, excluded.first_seen), + last_seen = max(last_seen, excluded.last_seen)", + (last_log_id, next_log_id), + )?; + conn.execute( + "INSERT INTO source_ip_inventory_stats(source_ip, log_count, first_seen, last_seen) + SELECT source_ip, COUNT(*), MIN(received_at), MAX(received_at) + FROM logs + WHERE id > ?1 + AND id <= ?2 + AND source_ip != '' + GROUP BY source_ip + ON CONFLICT(source_ip) DO UPDATE SET + log_count = log_count + excluded.log_count, + first_seen = min(first_seen, excluded.first_seen), + last_seen = max(last_seen, excluded.last_seen)", + (last_log_id, next_log_id), + )?; + conn.execute( + "INSERT INTO source_ip_host_inventory_stats(source_ip, hostname, log_count, first_seen, last_seen) + SELECT source_ip, hostname, COUNT(*), MIN(received_at), MAX(received_at) + FROM logs + WHERE id > ?1 + AND id <= ?2 + AND source_ip != '' + GROUP BY source_ip, hostname + ON CONFLICT(source_ip, hostname) DO UPDATE SET + log_count = log_count + excluded.log_count, + first_seen = min(first_seen, excluded.first_seen), + last_seen = max(last_seen, excluded.last_seen)", + (last_log_id, next_log_id), + )?; + conn.execute( + "UPDATE inventory_backfill_state + SET last_log_id = ?1, + last_error = NULL + WHERE name = 'app_source_inventory'", + [next_log_id], + )?; + Ok(()) + })(); + match result { + Ok(()) => conn.execute_batch("COMMIT;")?, + Err(error) => { + let _ = conn.execute_batch("ROLLBACK;"); + let _ = conn.execute( + "UPDATE inventory_backfill_state + SET last_error = ?1 + WHERE name = 'app_source_inventory'", + [error.to_string()], + ); + return Err(error.into()); + } + } + tracing::debug!( + last_log_id = next_log_id, + high_watermark_id, + "Inventory stats backfill chunk completed" + ); + std::thread::sleep(BETWEEN_CHUNKS); + } +} + +fn apply_migration_13(conn: &Connection) -> rusqlite::Result<()> { + // Explicit transaction keeps index/version updates atomic, while each ALTER is + // guarded so manually repaired or partially migrated DBs can converge instead + // of failing on duplicate columns with no version row. + conn.execute_batch("BEGIN IMMEDIATE;")?; + let result = (|| { + add_column_if_missing(conn, "logs", "http_status", "INTEGER")?; + add_column_if_missing(conn, "logs", "auth_outcome", "TEXT")?; + add_column_if_missing(conn, "logs", "dns_blocked", "INTEGER")?; + add_column_if_missing(conn, "logs", "event_action", "TEXT")?; + add_column_if_missing(conn, "logs", "parse_error", "TEXT")?; + conn.execute_batch( + "CREATE INDEX IF NOT EXISTS idx_logs_http_status_time + ON logs(http_status, timestamp) WHERE http_status IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_logs_auth_outcome_time + ON logs(auth_outcome, timestamp) WHERE auth_outcome IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_logs_dns_blocked_time + ON logs(dns_blocked, timestamp) WHERE dns_blocked IS NOT NULL; + CREATE INDEX IF NOT EXISTS idx_logs_event_action_time + ON logs(event_action, timestamp) WHERE event_action IS NOT NULL; + INSERT OR IGNORE INTO schema_migrations (version) VALUES (13);", + ) + })(); + + match result { + Ok(()) => conn.execute_batch("COMMIT;"), + Err(error) => { + let _ = conn.execute_batch("ROLLBACK;"); + Err(error) + } + } +} + +// Migration 22: source watermark for the AI session rollup (bead cortex-g33v). +// The background refresh recomputed the full GROUP-BY over `logs` every cadence +// even when no AI rows had changed. These two columns record the source-side +// `(COUNT(*), MAX(id))` of AI rows captured by the last refresh; the refresh +// task compares the live watermark against them and skips the recompute when +// nothing changed. Both default to 0 so the first post-migration refresh always +// runs (live watermark > 0 whenever AI rows exist, and `refreshed_at` is still +// NULL regardless). +// +// Wrapped in an explicit BEGIN IMMEDIATE / COMMIT-or-ROLLBACK transaction so a +// crash between the two ALTERs and the version marker rolls back BOTH columns +// and the marker atomically — the previous bare `execute_batch` auto-committed +// each statement, leaving a half-applied DB that bricked `init_pool` on restart +// with "duplicate column name". Each ALTER is guarded with `add_column_if_missing` +// so a partially-applied DB (columns present, marker absent) converges on retry. +fn apply_migration_22(conn: &Connection) -> rusqlite::Result<()> { + conn.execute_batch("BEGIN IMMEDIATE;")?; + let result = (|| { + add_column_if_missing( + conn, + "ai_session_rollup_meta", + "source_row_count", + "INTEGER NOT NULL DEFAULT 0", + )?; + add_column_if_missing( + conn, + "ai_session_rollup_meta", + "source_max_id", + "INTEGER NOT NULL DEFAULT 0", + )?; + conn.execute_batch("INSERT OR IGNORE INTO schema_migrations (version) VALUES (22);") + })(); + + match result { + Ok(()) => conn.execute_batch("COMMIT;"), + Err(error) => { + let _ = conn.execute_batch("ROLLBACK;"); + Err(error) + } + } +} + +fn apply_migration_15_heartbeat(conn: &Connection) -> rusqlite::Result<()> { + conn.execute_batch("BEGIN IMMEDIATE;")?; + let result = conn.execute_batch( + " + CREATE TABLE IF NOT EXISTS host_heartbeats ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + host_id TEXT NOT NULL, + hostname TEXT NOT NULL, + source_ip TEXT NOT NULL DEFAULT '', + sampled_at TEXT NOT NULL, + received_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + boot_id TEXT NOT NULL, + uptime_secs INTEGER NOT NULL, + sequence INTEGER NOT NULL, + collection_ms INTEGER NOT NULL, + push_latency_ms INTEGER, + partial INTEGER NOT NULL DEFAULT 0, + agent_version TEXT NOT NULL, + os TEXT NOT NULL, + kernel TEXT, + architecture TEXT NOT NULL, + metadata_json TEXT, + UNIQUE(host_id, boot_id, sequence) + ); + + CREATE INDEX IF NOT EXISTS idx_host_heartbeats_host_sampled + ON host_heartbeats(host_id, sampled_at); + CREATE INDEX IF NOT EXISTS idx_host_heartbeats_received + ON host_heartbeats(received_at); + CREATE INDEX IF NOT EXISTS idx_host_heartbeats_hostname_sampled + ON host_heartbeats(hostname, sampled_at); + + CREATE TABLE IF NOT EXISTS heartbeat_cpu ( + heartbeat_id INTEGER NOT NULL, + load1 REAL, + load5 REAL, + load15 REAL, + usage_percent REAL, + steal_percent REAL, + io_wait_percent REAL + ); + CREATE INDEX IF NOT EXISTS idx_heartbeat_cpu_heartbeat_id + ON heartbeat_cpu(heartbeat_id); + + CREATE TABLE IF NOT EXISTS heartbeat_memory ( + heartbeat_id INTEGER NOT NULL, + total_bytes INTEGER, + available_bytes INTEGER, + used_percent REAL, + swap_total_bytes INTEGER, + swap_used_bytes INTEGER + ); + CREATE INDEX IF NOT EXISTS idx_heartbeat_memory_heartbeat_id + ON heartbeat_memory(heartbeat_id); + + CREATE TABLE IF NOT EXISTS heartbeat_disks ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + heartbeat_id INTEGER NOT NULL, + mountpoint TEXT, + filesystem TEXT, + total_bytes INTEGER, + available_bytes INTEGER, + used_percent REAL, + read_bytes_per_sec REAL, + write_bytes_per_sec REAL + ); + CREATE INDEX IF NOT EXISTS idx_heartbeat_disks_heartbeat_id + ON heartbeat_disks(heartbeat_id); + + CREATE TABLE IF NOT EXISTS heartbeat_network ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + heartbeat_id INTEGER NOT NULL, + interface TEXT NOT NULL, + rx_bytes_per_sec REAL, + tx_bytes_per_sec REAL, + rx_errors INTEGER, + tx_errors INTEGER + ); + CREATE INDEX IF NOT EXISTS idx_heartbeat_network_heartbeat_id + ON heartbeat_network(heartbeat_id); + + CREATE TABLE IF NOT EXISTS heartbeat_processes ( + heartbeat_id INTEGER NOT NULL, + total INTEGER, + running INTEGER, + sleeping INTEGER, + zombie INTEGER, + top_cpu_json TEXT, + top_memory_json TEXT + ); + CREATE INDEX IF NOT EXISTS idx_heartbeat_processes_heartbeat_id + ON heartbeat_processes(heartbeat_id); + + CREATE TABLE IF NOT EXISTS heartbeat_containers ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + heartbeat_id INTEGER NOT NULL, + runtime TEXT, + running INTEGER, + stopped INTEGER, + restarting INTEGER, + unhealthy INTEGER, + summary_json TEXT + ); + CREATE INDEX IF NOT EXISTS idx_heartbeat_containers_heartbeat_id + ON heartbeat_containers(heartbeat_id); + + INSERT OR IGNORE INTO schema_migrations (version) VALUES (15); + ", + ); + + match result { + Ok(()) => conn.execute_batch("COMMIT;"), + Err(error) => { + let _ = conn.execute_batch("ROLLBACK;"); + Err(error) + } + } +} + +fn apply_migration_18_heartbeat_restarting(conn: &Connection) -> rusqlite::Result<()> { + conn.execute_batch("BEGIN IMMEDIATE;")?; + let result = (|| { + add_column_if_missing(conn, "heartbeat_containers", "restarting", "INTEGER")?; + conn.execute_batch("INSERT OR IGNORE INTO schema_migrations (version) VALUES (18);") + })(); + match result { + Ok(()) => conn.execute_batch("COMMIT;"), + Err(error) => { + let _ = conn.execute_batch("ROLLBACK;"); + Err(error) + } + } +} + +/// Migration 19: `host_heartbeats_latest` — one row per host_id, updated on +/// every new accepted heartbeat. This is the foundation for `fleet_state` +/// queries: instead of scanning `host_heartbeats` for the latest row per host +/// (O(heartbeats)), fleet queries scan this small table (O(hosts)). +/// +/// Backfill on first apply: for each distinct `host_id`, find the row with the +/// highest `id` (proxy for latest, since `id` is AUTOINCREMENT) and seed the +/// cache. The GROUP BY scan happens once at migration time, not per query. +fn apply_migration_19_heartbeat_latest(conn: &Connection) -> rusqlite::Result<()> { + conn.execute_batch("BEGIN IMMEDIATE;")?; + let result = (|| { + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS host_heartbeats_latest ( + host_id TEXT PRIMARY KEY, + heartbeat_id INTEGER NOT NULL, + hostname TEXT NOT NULL, + sampled_at TEXT NOT NULL, + received_at TEXT NOT NULL, + partial INTEGER NOT NULL DEFAULT 0, + agent_version TEXT NOT NULL DEFAULT '', + os TEXT NOT NULL DEFAULT '', + architecture TEXT NOT NULL DEFAULT '', + metadata_json TEXT + ); + INSERT OR IGNORE INTO host_heartbeats_latest + (host_id, heartbeat_id, hostname, sampled_at, received_at, + partial, agent_version, os, architecture, metadata_json) + SELECT h.host_id, h.id, h.hostname, h.sampled_at, h.received_at, + h.partial, h.agent_version, h.os, h.architecture, h.metadata_json + FROM host_heartbeats h + INNER JOIN ( + SELECT host_id, MAX(id) AS max_id + FROM host_heartbeats + GROUP BY host_id + ) latest ON h.id = latest.max_id;", + )?; + conn.execute_batch("INSERT OR IGNORE INTO schema_migrations (version) VALUES (19);") + })(); + match result { + Ok(()) => conn.execute_batch("COMMIT;"), + Err(error) => { + let _ = conn.execute_batch("ROLLBACK;"); + Err(error) + } + } +} + +fn configure_connection_pragmas( + conn: &mut Connection, + storage: &StorageConfig, +) -> rusqlite::Result<()> { + if storage.wal_mode { + conn.execute_batch("PRAGMA journal_mode=WAL;")?; + } + conn.pragma_update(None, "synchronous", "NORMAL")?; + conn.pragma_update(None, "busy_timeout", 5000_i64)?; + let cache_size = storage + .sqlite_page_cache_kib_per_connection() + .map_err(|error| { + rusqlite::Error::ToSqlConversionFailure(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + error.to_string(), + ))) + })?; + conn.pragma_update(None, "cache_size", cache_size)?; + let mmap_size = storage.sqlite_mmap_bytes_i64().map_err(|error| { + rusqlite::Error::ToSqlConversionFailure(Box::new(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + error.to_string(), + ))) + })?; + conn.pragma_update(None, "mmap_size", mmap_size)?; + conn.pragma_update(None, "analysis_limit", 400_i64)?; + Ok(()) +} + +#[cfg(test)] +#[path = "pool_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/pool_tests.rs b/crates/shared/cortex/storage-sqlite/src/pool_tests.rs new file mode 100644 index 00000000..2c79a4d6 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/pool_tests.rs @@ -0,0 +1,5126 @@ +use super::*; +use crate::config::StorageConfig; +use crate::{ + ENTITY_TYPES, EVIDENCE_SOURCE_KINDS, LogBatchEntry, REASON_CODES, RELATIONSHIP_TYPES, + TRUST_LEVELS, insert_logs_batch, is_known_entity_type, is_known_evidence_source_kind, + is_known_reason_code, is_known_relationship_type, is_known_trust_level, +}; +use rusqlite::OptionalExtension; + +fn test_storage_config(db_path: std::path::PathBuf) -> StorageConfig { + StorageConfig::for_test(db_path) +} + +#[test] +fn test_init_pool_enables_incremental_auto_vacuum() { + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("autovac.db")); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + let mode: i64 = conn + .query_row("PRAGMA auto_vacuum", [], |r| r.get(0)) + .unwrap(); + assert_eq!(mode, 2); +} + +#[test] +fn test_init_pool_migrates_existing_db_to_incremental_auto_vacuum() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("legacy.db"); + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute_batch( + "PRAGMA auto_vacuum=NONE; + VACUUM; + CREATE TABLE legacy_probe(id INTEGER PRIMARY KEY);", + ) + .unwrap(); + drop(conn); + + let config = test_storage_config(db_path); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + let mode: i64 = conn + .query_row("PRAGMA auto_vacuum", [], |r| r.get(0)) + .unwrap(); + assert_eq!(mode, 2); +} + +#[test] +fn test_init_pool_applies_busy_timeout_to_each_pooled_connection() { + let dir = tempfile::tempdir().unwrap(); + let mut config = test_storage_config(dir.path().join("busy-timeout.db")); + config.pool_size = 2; + let pool = init_pool(&config).unwrap(); + + let conn1 = pool.get().unwrap(); + let conn2 = pool.get().unwrap(); + + let busy_timeout_1: i64 = conn1 + .query_row("PRAGMA busy_timeout", [], |r| r.get(0)) + .unwrap(); + let busy_timeout_2: i64 = conn2 + .query_row("PRAGMA busy_timeout", [], |r| r.get(0)) + .unwrap(); + + assert_eq!(busy_timeout_1, 5000); + assert_eq!(busy_timeout_2, 5000); +} + +#[test] +fn init_pool_applies_sqlite_cache_budget_to_each_pooled_connection() { + let dir = tempfile::tempdir().unwrap(); + let mut config = test_storage_config(dir.path().join("cache-budget.db")); + config.pool_size = 2; + config.sqlite_page_cache_mb = 128; + + let pool = init_pool(&config).unwrap(); + let conn1 = pool.get().unwrap(); + let conn2 = pool.get().unwrap(); + + let cache_1: i64 = conn1 + .query_row("PRAGMA cache_size", [], |row| row.get(0)) + .unwrap(); + let cache_2: i64 = conn2 + .query_row("PRAGMA cache_size", [], |row| row.get(0)) + .unwrap(); + + assert_eq!(cache_1, -65_536); + assert_eq!(cache_2, -65_536); +} + +#[test] +fn init_pool_applies_sqlite_mmap_to_each_pooled_connection() { + let dir = tempfile::tempdir().unwrap(); + let mut config = test_storage_config(dir.path().join("mmap.db")); + config.pool_size = 2; + config.sqlite_mmap_mb = 32; + + let pool = init_pool(&config).unwrap(); + let conn1 = pool.get().unwrap(); + let conn2 = pool.get().unwrap(); + + let mmap_1: i64 = conn1 + .query_row("PRAGMA mmap_size", [], |row| row.get(0)) + .unwrap(); + let mmap_2: i64 = conn2 + .query_row("PRAGMA mmap_size", [], |row| row.get(0)) + .unwrap(); + + assert_eq!(mmap_1, 32 * 1024 * 1024); + assert_eq!(mmap_2, 32 * 1024 * 1024); +} + +#[test] +fn init_db_creates_heartbeat_schema_migration_15() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("heartbeat.db"); + let config = test_storage_config(db_path); + + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + let applied: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 15", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(applied, 1); + + for table in [ + "host_heartbeats", + "heartbeat_cpu", + "heartbeat_memory", + "heartbeat_disks", + "heartbeat_network", + "heartbeat_processes", + "heartbeat_containers", + ] { + let exists: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1", + [table], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(exists, 1, "missing heartbeat table {table}"); + } + + for index in [ + "idx_host_heartbeats_host_sampled", + "idx_host_heartbeats_received", + "idx_host_heartbeats_hostname_sampled", + "idx_heartbeat_cpu_heartbeat_id", + "idx_heartbeat_memory_heartbeat_id", + "idx_heartbeat_disks_heartbeat_id", + "idx_heartbeat_network_heartbeat_id", + "idx_heartbeat_processes_heartbeat_id", + "idx_heartbeat_containers_heartbeat_id", + ] { + let exists: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = ?1", + [index], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(exists, 1, "missing heartbeat index {index}"); + } +} + +#[test] +fn init_db_creates_timeline_and_jobs_schema_migrations_25_26() { + // Validate migrations 25 + 26 on a CLEAN temp DB (never touch prod). + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("mig25_26.db")); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + for version in [25, 26] { + let applied: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = ?1", + [version], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(applied, 1, "migration {version} not recorded"); + } + + for table in [ + "timeline_hourly", + "timeline_hourly_meta", + "maintenance_jobs", + ] { + let exists: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1", + [table], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(exists, 1, "missing table {table}"); + } + + // Meta row is seeded with watermark 0 / never-refreshed on a fresh DB. + let (refreshed, max_id): (Option, i64) = conn + .query_row( + "SELECT refreshed_at, source_max_id FROM timeline_hourly_meta WHERE id = 1", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + ) + .unwrap(); + assert!(refreshed.is_none()); + assert_eq!(max_id, 0); + + // Empty DB => backfill skipped => rollup empty. + let rollup_rows: i64 = conn + .query_row("SELECT COUNT(*) FROM timeline_hourly", [], |r| r.get(0)) + .unwrap(); + assert_eq!(rollup_rows, 0); +} + +#[test] +fn init_db_creates_graph_schema_migration_27() { + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("graph.db")); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + let applied: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 27", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(applied, 1, "migration 27 not recorded"); + + for table in [ + "graph_entities", + "graph_entity_aliases", + "graph_relationships", + "graph_relationship_evidence", + "graph_projection_meta", + ] { + let exists: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1", + [table], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(exists, 1, "missing graph table {table}"); + } + + let (status, degraded): (String, i64) = conn + .query_row( + "SELECT projection_status, is_degraded FROM graph_projection_meta WHERE id = 1", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(status, "never_built"); + assert_eq!(degraded, 0); +} + +#[test] +fn graph_migration_is_idempotent_and_preserves_raw_logs() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("graph-idempotent.db"); + let config = test_storage_config(db_path); + let pool = init_pool(&config).unwrap(); + + let inserted = insert_logs_batch( + &pool, + &[LogBatchEntry { + timestamp: "2026-01-01T00:00:00Z".to_string(), + hostname: "claimed-host".to_string(), + facility: None, + severity: "info".to_string(), + app_name: Some("sshd".to_string()), + process_id: None, + message: "accepted publickey".to_string(), + raw: "accepted publickey".to_string(), + source_ip: "10.0.0.1:514".to_string(), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + }], + ) + .unwrap(); + assert_eq!(inserted, 1); + drop(pool); + + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + let log_count: i64 = conn + .query_row("SELECT COUNT(*) FROM logs", [], |row| row.get(0)) + .unwrap(); + assert_eq!(log_count, 1, "graph migration must not mutate raw logs"); + + let migration_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 27", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + migration_count, 1, + "graph migration marker must remain idempotent" + ); +} + +#[test] +fn graph_migration_converges_after_schema_exists_without_marker() { + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("graph-partial.db")); + let pool = init_pool(&config).unwrap(); + { + let conn = pool.get().unwrap(); + conn.execute("DELETE FROM schema_migrations WHERE version = 27", []) + .unwrap(); + } + drop(pool); + + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + let migration_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 27", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + migration_count, 1, + "migration 27 must converge when DDL already exists" + ); +} + +#[test] +fn known_schema_version_matches_migration_head() { + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("schema-head.db")); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + let max_version: i64 = conn + .query_row("SELECT MAX(version) FROM schema_migrations", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(max_version, KNOWN_SCHEMA_VERSION); + drop(conn); + + let info = read_schema_version_info(&pool).unwrap(); + assert_eq!(info.version, KNOWN_SCHEMA_VERSION); + assert_eq!(info.known_version, KNOWN_SCHEMA_VERSION); +} + +#[test] +fn init_pool_creates_agent_observatory_repository_schema_scaffold() { + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("observatory-repositories.db")); + + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + let columns: Vec = conn + .prepare("PRAGMA table_info(repositories)") + .unwrap() + .query_map([], |row| row.get(1)) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!( + columns, + vec![ + "id", + "repository_key", + "hostname", + "common_git_dir", + "primary_path", + "display_name", + "remote_url_hash", + "first_seen_at", + "last_seen_at", + "removed_at", + "metadata_json", + "created_at", + "updated_at", + ] + ); + + let indexes: Vec = conn + .prepare( + "SELECT name FROM sqlite_master + WHERE type = 'index' AND tbl_name = 'repositories' + ORDER BY name", + ) + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + assert!( + indexes + .iter() + .any(|name| name == "idx_repositories_display") + ); + assert!( + indexes + .iter() + .any(|name| name == "idx_repositories_host_seen") + ); + + conn.execute( + "INSERT INTO repositories + (repository_key, hostname, common_git_dir, primary_path, display_name, + first_seen_at, last_seen_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?6)", + rusqlite::params![ + "v1|6:devhost|20:/workspace/cortex/.git", + "devhost", + "/workspace/cortex/.git", + "/workspace/cortex", + "cortex", + "2026-07-31T23:00:00.000Z", + ], + ) + .unwrap(); + assert!( + conn.execute( + "INSERT INTO repositories + (repository_key, hostname, common_git_dir, primary_path, display_name, + first_seen_at, last_seen_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?6)", + rusqlite::params![ + "v1|6:devhost|20:/workspace/cortex/.git", + "other-host", + "/workspace/other/.git", + "/workspace/other", + "other", + "2026-07-31T23:00:00.000Z", + ], + ) + .is_err(), + "repository_key must be globally unique" + ); + assert!( + conn.execute( + "INSERT INTO repositories + (repository_key, hostname, common_git_dir, primary_path, display_name, + first_seen_at, last_seen_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?6)", + rusqlite::params![ + "different-key", + "devhost", + "/workspace/cortex/.git", + "/workspace/cortex-copy", + "cortex-copy", + "2026-07-31T23:00:00.000Z", + ], + ) + .is_err(), + "hostname/common_git_dir must identify one repository" + ); + drop(conn); + drop(pool); + + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + let row_count: i64 = conn + .query_row("SELECT COUNT(*) FROM repositories", [], |row| row.get(0)) + .unwrap(); + assert_eq!(row_count, 1, "reopening must preserve repository rows"); + let migration_44_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 44", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + migration_44_count, 1, + "completed migration 44 must remain marked exactly once" + ); +} + +#[test] +fn init_pool_creates_agent_observatory_worktree_schema_scaffold() { + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("observatory-worktrees.db")); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + conn.execute( + "INSERT INTO repositories + (repository_key, hostname, common_git_dir, primary_path, display_name, + first_seen_at, last_seen_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?6)", + rusqlite::params![ + "repo-key", + "devhost", + "/workspace/cortex/.git", + "/workspace/cortex", + "cortex", + "2026-08-01T01:00:00.000Z", + ], + ) + .unwrap(); + let repository_id = conn.last_insert_rowid(); + + let columns: Vec = conn + .prepare("PRAGMA table_info(repository_worktrees)") + .unwrap() + .query_map([], |row| row.get(1)) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!( + columns, + vec![ + "id", + "worktree_key", + "repository_id", + "hostname", + "path", + "git_dir", + "branch_ref", + "branch_name", + "head_sha", + "upstream_ref", + "detached", + "bare", + "locked", + "lock_reason", + "prunable", + "prune_reason", + "dirty", + "staged_count", + "unstaged_count", + "untracked_count", + "ahead", + "behind", + "status_hash", + "first_seen_at", + "last_seen_at", + "removed_at", + "created_at", + "updated_at", + ] + ); + + conn.execute( + "INSERT INTO repository_worktrees + (worktree_key, repository_id, hostname, path, git_dir, branch_ref, + branch_name, head_sha, upstream_ref, dirty, staged_count, + unstaged_count, untracked_count, ahead, behind, first_seen_at, last_seen_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, 1, 2, 3, 4, 5, 6, ?10, ?10)", + rusqlite::params![ + "worktree-key", + repository_id, + "devhost", + "/workspace/cortex", + "/workspace/cortex/.git", + "refs/heads/feat/agent-observatory", + "feat/agent-observatory", + "0123456789012345678901234567890123456789", + "refs/remotes/origin/feat/agent-observatory", + "2026-08-01T01:00:00.000Z", + ], + ) + .unwrap(); + + assert!( + conn.execute( + "INSERT INTO repository_worktrees + (worktree_key, repository_id, hostname, path, git_dir, first_seen_at, last_seen_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?6)", + rusqlite::params![ + "different-key", + repository_id, + "devhost", + "/workspace/cortex", + "/workspace/cortex/.git/worktrees/duplicate", + "2026-08-01T01:00:00.000Z", + ], + ) + .is_err(), + "hostname/path must identify one worktree" + ); + + let state: (String, String, i64, i64, i64, i64, i64) = conn + .query_row( + "SELECT branch_name, head_sha, dirty, staged_count, unstaged_count, + untracked_count, ahead + FROM repository_worktrees WHERE worktree_key = 'worktree-key'", + [], + |row| { + Ok(( + row.get(0)?, + row.get(1)?, + row.get(2)?, + row.get(3)?, + row.get(4)?, + row.get(5)?, + row.get(6)?, + )) + }, + ) + .unwrap(); + assert_eq!( + state, + ( + "feat/agent-observatory".to_string(), + "0123456789012345678901234567890123456789".to_string(), + 1, + 2, + 3, + 4, + 5, + ) + ); + + conn.execute("DELETE FROM repositories WHERE id = ?1", [repository_id]) + .unwrap(); + let remaining: i64 = conn + .query_row("SELECT COUNT(*) FROM repository_worktrees", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!( + remaining, 0, + "repository deletion must cascade to worktrees" + ); + + let foreign_key_violation: Option = conn + .query_row("PRAGMA foreign_key_check", [], |row| row.get(0)) + .optional() + .unwrap(); + assert_eq!(foreign_key_violation, None); +} + +#[test] +fn init_pool_creates_agent_observatory_observation_schema_scaffold() { + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("observatory-observations.db")); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + conn.execute( + "INSERT INTO repositories + (repository_key, hostname, common_git_dir, primary_path, display_name, + first_seen_at, last_seen_at) + VALUES ('repo-key', 'devhost', '/workspace/cortex/.git', + '/workspace/cortex', 'cortex', ?1, ?1)", + ["2026-08-01T01:00:00.000Z"], + ) + .unwrap(); + let repository_id = conn.last_insert_rowid(); + conn.execute( + "INSERT INTO repository_worktrees + (worktree_key, repository_id, hostname, path, git_dir, first_seen_at, last_seen_at) + VALUES ('worktree-key', ?1, 'devhost', '/workspace/cortex', + '/workspace/cortex/.git', ?2, ?2)", + rusqlite::params![repository_id, "2026-08-01T01:00:00.000Z"], + ) + .unwrap(); + let worktree_id = conn.last_insert_rowid(); + + let columns: Vec = conn + .prepare("PRAGMA table_info(repository_observations)") + .unwrap() + .query_map([], |row| row.get(1)) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!( + columns, + vec![ + "id", + "observation_key", + "repository_id", + "worktree_id", + "observed_at", + "observation_kind", + "old_head_sha", + "new_head_sha", + "summary", + "payload_json", + "created_at", + ] + ); + + let insert = |key: &str, observed_at: &str, kind: &str| { + conn.execute( + "INSERT INTO repository_observations + (observation_key, repository_id, worktree_id, observed_at, + observation_kind, summary, payload_json) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, '{}')", + rusqlite::params![key, repository_id, worktree_id, observed_at, kind, key], + ) + }; + insert("obs-1", "2026-08-01T01:00:00.000Z", "discovered").unwrap(); + insert("obs-2", "2026-08-01T01:00:01.000Z", "status").unwrap(); + insert("obs-3", "2026-08-01T01:00:01.000Z", "head").unwrap(); + + assert!( + insert("obs-1", "2026-08-01T01:00:02.000Z", "status").is_err(), + "observation_key must be globally unique" + ); + assert!( + conn.execute( + "INSERT INTO repository_observations + (observation_key, repository_id, observed_at, observation_kind, payload_json) + VALUES ('bad-json', ?1, ?2, 'error', '{')", + rusqlite::params![repository_id, "2026-08-01T01:00:03.000Z"], + ) + .is_err(), + "payload_json must be valid JSON" + ); + + let ordered: Vec = conn + .prepare( + "SELECT observation_key FROM repository_observations + WHERE repository_id = ?1 + ORDER BY observed_at DESC, id DESC", + ) + .unwrap() + .query_map([repository_id], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!(ordered, vec!["obs-3", "obs-2", "obs-1"]); + + let repo_plan: Vec = conn + .prepare( + "EXPLAIN QUERY PLAN + SELECT id FROM repository_observations + WHERE repository_id = ?1 + ORDER BY observed_at DESC, id DESC LIMIT 10", + ) + .unwrap() + .query_map([repository_id], |row| row.get(3)) + .unwrap() + .collect::>() + .unwrap(); + assert!( + repo_plan + .iter() + .any(|detail| detail.contains("idx_repository_observations_repo_time")), + "repository timeline query must use its chronological index: {repo_plan:?}" + ); + + let indexes: Vec = conn + .prepare( + "SELECT name FROM sqlite_master + WHERE type = 'index' AND tbl_name = 'repository_observations' + ORDER BY name", + ) + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + assert!( + indexes + .iter() + .any(|name| name == "idx_repository_observations_repo_time") + ); + assert!( + indexes + .iter() + .any(|name| name == "idx_repository_observations_worktree_time") + ); +} + +#[test] +fn init_pool_creates_agent_observatory_git_commit_schema_scaffold() { + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("observatory-commits.db")); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + for (key, common_dir, path, name) in [ + ("repo-1", "/workspace/one/.git", "/workspace/one", "one"), + ("repo-2", "/workspace/two/.git", "/workspace/two", "two"), + ] { + conn.execute( + "INSERT INTO repositories + (repository_key, hostname, common_git_dir, primary_path, display_name, + first_seen_at, last_seen_at) + VALUES (?1, 'devhost', ?2, ?3, ?4, ?5, ?5)", + rusqlite::params![key, common_dir, path, name, "2026-08-01T01:00:00.000Z"], + ) + .unwrap(); + } + let repo_one: i64 = conn + .query_row( + "SELECT id FROM repositories WHERE repository_key = 'repo-1'", + [], + |row| row.get(0), + ) + .unwrap(); + let repo_two: i64 = conn + .query_row( + "SELECT id FROM repositories WHERE repository_key = 'repo-2'", + [], + |row| row.get(0), + ) + .unwrap(); + + let columns: Vec = conn + .prepare("PRAGMA table_info(git_commits)") + .unwrap() + .query_map([], |row| row.get(1)) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!( + columns, + vec![ + "id", + "repository_id", + "sha", + "parent_shas_json", + "author_name", + "author_email_hash", + "authored_at", + "committed_at", + "subject", + "changed_files", + "insertions", + "deletions", + "changed_paths_json", + "first_observed_at", + "last_observed_at", + "reachable", + "metadata_json", + ] + ); + assert!( + !columns + .iter() + .any(|name| matches!(name.as_str(), "diff" | "patch" | "blob" | "author_email")) + ); + + let sha = "0123456789012345678901234567890123456789"; + let insert_commit = |repository_id: i64| { + conn.execute( + "INSERT INTO git_commits + (repository_id, sha, parent_shas_json, author_name, author_email_hash, + authored_at, committed_at, subject, changed_files, insertions, + deletions, changed_paths_json, first_observed_at, last_observed_at, + metadata_json) + VALUES (?1, ?2, '[]', 'Cortex Test', 'sha256:test', ?3, ?3, + 'test commit', 2, 10, 3, '[\"src/lib.rs\"]', ?3, ?3, '{}')", + rusqlite::params![repository_id, sha, "2026-08-01T01:00:00.000Z"], + ) + }; + insert_commit(repo_one).unwrap(); + assert!( + insert_commit(repo_one).is_err(), + "same SHA must dedupe within a repository" + ); + insert_commit(repo_two).unwrap(); + + assert!( + conn.execute( + "INSERT INTO git_commits + (repository_id, sha, parent_shas_json, changed_paths_json, + first_observed_at, last_observed_at) + VALUES (?1, 'bad-json', '{', '[]', ?2, ?2)", + rusqlite::params![repo_one, "2026-08-01T01:00:00.000Z"], + ) + .is_err(), + "commit JSON columns must reject invalid JSON" + ); + + conn.execute( + "UPDATE git_commits + SET reachable = 0, last_observed_at = ?1 + WHERE repository_id = ?2 AND sha = ?3", + rusqlite::params!["2026-08-01T02:00:00.000Z", repo_one, sha], + ) + .unwrap(); + let state: (i64, String, String) = conn + .query_row( + "SELECT reachable, subject, last_observed_at FROM git_commits + WHERE repository_id = ?1 AND sha = ?2", + rusqlite::params![repo_one, sha], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!( + state, + ( + 0, + "test commit".to_string(), + "2026-08-01T02:00:00.000Z".to_string() + ) + ); + + let repo_one_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM git_commits WHERE repository_id = ?1", + [repo_one], + |row| row.get(0), + ) + .unwrap(); + let repo_two_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM git_commits WHERE repository_id = ?1", + [repo_two], + |row| row.get(0), + ) + .unwrap(); + assert_eq!((repo_one_count, repo_two_count), (1, 1)); + + let index_exists: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master + WHERE type = 'index' AND name = 'idx_git_commits_repo_time'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(index_exists, 1); +} + +#[test] +fn migration_44_applies_from_schema_43_and_is_idempotent() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("observatory-migration-44.db"); + let config = test_storage_config(db_path.clone()); + + { + let pool = init_pool(&config).unwrap(); + drop(pool); + } + + { + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute_batch( + "PRAGMA foreign_keys = OFF; + DROP TABLE IF EXISTS git_commits; + DROP TABLE IF EXISTS repository_observations; + DROP TABLE IF EXISTS repository_worktrees; + DROP TABLE IF EXISTS repositories; + DELETE FROM schema_migrations WHERE version = 44; + INSERT OR REPLACE INTO stream_last_seen + (hostname, source_kind, last_seen_at) + VALUES ('legacy-host', 'syslog-tcp', '2026-08-01T01:00:00.000Z'); + PRAGMA foreign_keys = ON;", + ) + .unwrap(); + } + + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + let max_version: i64 = conn + .query_row("SELECT MAX(version) FROM schema_migrations", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!( + max_version, 47, + "schema 43 should upgrade to schema 47 (applying 44, 45, 46, 47)" + ); + let marker_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 44", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(marker_count, 1); + + for table in [ + "repositories", + "repository_worktrees", + "repository_observations", + "git_commits", + ] { + let exists: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1", + [table], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(exists, 1, "migration 44 must create {table}"); + } + + let legacy_rows: i64 = conn + .query_row( + "SELECT COUNT(*) FROM stream_last_seen + WHERE hostname = 'legacy-host' AND source_kind = 'syslog-tcp'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(legacy_rows, 1, "migration must preserve schema-43 data"); + let foreign_key_violation: Option = conn + .query_row("PRAGMA foreign_key_check", [], |row| row.get(0)) + .optional() + .unwrap(); + assert_eq!(foreign_key_violation, None); + let integrity: String = conn + .query_row("PRAGMA integrity_check", [], |row| row.get(0)) + .unwrap(); + assert_eq!(integrity, "ok"); + drop(conn); + drop(pool); + + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + let marker_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 44", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(marker_count, 1, "reopening must not duplicate migration 44"); +} + +#[test] +fn init_pool_creates_agent_observatory_run_schema_scaffold() { + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("observatory-runs.db")); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + let columns: Vec = conn + .prepare("PRAGMA table_info(agent_runs)") + .unwrap() + .query_map([], |row| row.get(1)) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!( + columns, + vec![ + "id", + "run_key", + "native_session_id", + "tool", + "provider_tool", + "hostname", + "parent_run_id", + "previous_run_id", + "primary_worktree_id", + "transcript_path", + "process_id", + "status", + "status_reason", + "status_observed_at", + "started_at", + "last_activity_at", + "ended_at", + "first_source_log_id", + "last_source_log_id", + "last_event_id", + "event_count", + "error_count", + "primary_branch", + "start_head_sha", + "current_head_sha", + "projection_version", + "freshness_json", + "metadata_json", + "created_at", + "updated_at", + ] + ); + + conn.execute( + "INSERT INTO agent_runs + (run_key, native_session_id, tool, hostname, status, + status_observed_at, started_at, last_activity_at) + VALUES (?1, ?2, ?3, ?4, 'active', ?5, ?5, ?5)", + rusqlite::params![ + "v1|6:devhost|6:claude|9:session-1", + "session-1", + "claude", + "devhost", + "2026-08-01T02:00:00.000Z", + ], + ) + .unwrap(); + + let primary_worktree_id: Option = conn + .query_row( + "SELECT primary_worktree_id FROM agent_runs WHERE native_session_id = 'session-1'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(primary_worktree_id, None); + + assert!( + conn.execute( + "INSERT INTO agent_runs + (run_key, native_session_id, tool, hostname, status, + status_observed_at, started_at, last_activity_at) + VALUES ('bad-status', 'session-2', 'claude', 'devhost', + 'running-ish', ?1, ?1, ?1)", + ["2026-08-01T02:00:01.000Z"], + ) + .is_err(), + "unknown lifecycle status must be rejected" + ); + + assert!( + conn.execute( + "INSERT INTO agent_runs + (run_key, native_session_id, tool, hostname, status, + status_observed_at, started_at, last_activity_at) + VALUES ('different-run-key', 'session-1', 'claude', 'devhost', + 'idle', ?1, ?1, ?1)", + ["2026-08-01T02:00:02.000Z"], + ) + .is_err(), + "host/tool/native-session identity must be unique" + ); + + let indexes: Vec = conn + .prepare( + "SELECT name FROM sqlite_master + WHERE type = 'index' AND tbl_name = 'agent_runs' + ORDER BY name", + ) + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + for expected in [ + "idx_agent_runs_activity", + "idx_agent_runs_status_activity", + "idx_agent_runs_worktree_activity", + "idx_agent_runs_tool_host", + ] { + assert!( + indexes.iter().any(|name| name == expected), + "missing {expected}" + ); + } + + let query_plan: Vec = conn + .prepare( + "EXPLAIN QUERY PLAN + SELECT id FROM agent_runs + WHERE status = 'active' + ORDER BY last_activity_at DESC, id DESC + LIMIT 50", + ) + .unwrap() + .query_map([], |row| row.get(3)) + .unwrap() + .collect::>() + .unwrap(); + assert!( + query_plan + .iter() + .any(|detail| detail.contains("idx_agent_runs_status_activity")), + "active-run query must use status/activity index: {query_plan:?}" + ); + + // Verify migration 47 is applied (schema includes OTLP tables) + let migration_47_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 47", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + migration_47_count, 1, + "migration 47 should be applied (OTLP metric points)" + ); +} + +#[test] +fn init_pool_creates_agent_observatory_actor_and_worktree_evidence_schema() { + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("observatory-run-evidence.db")); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + conn.execute( + "INSERT INTO repositories + (repository_key, hostname, common_git_dir, primary_path, display_name, + first_seen_at, last_seen_at) + VALUES ('repo-evidence', 'devhost', '/workspace/cortex/.git', + '/workspace/cortex', 'cortex', ?1, ?1)", + ["2026-08-01T02:30:00.000Z"], + ) + .unwrap(); + let repository_id = conn.last_insert_rowid(); + for (key, path) in [ + ("wt-main", "/workspace/cortex"), + ("wt-feature", "/workspace/cortex/.worktrees/feature"), + ] { + conn.execute( + "INSERT INTO repository_worktrees + (worktree_key, repository_id, hostname, path, git_dir, + first_seen_at, last_seen_at) + VALUES (?1, ?2, 'devhost', ?3, ?4, ?5, ?5)", + rusqlite::params![ + key, + repository_id, + path, + format!("{path}/.git"), + "2026-08-01T02:30:00.000Z", + ], + ) + .unwrap(); + } + let main_worktree: i64 = conn + .query_row( + "SELECT id FROM repository_worktrees WHERE worktree_key = 'wt-main'", + [], + |row| row.get(0), + ) + .unwrap(); + let feature_worktree: i64 = conn + .query_row( + "SELECT id FROM repository_worktrees WHERE worktree_key = 'wt-feature'", + [], + |row| row.get(0), + ) + .unwrap(); + + conn.execute( + "INSERT INTO agent_runs + (run_key, native_session_id, tool, hostname, status, + status_observed_at, started_at, last_activity_at) + VALUES ('run-evidence', 'session-evidence', 'claude', 'devhost', + 'active', ?1, ?1, ?1)", + ["2026-08-01T02:30:00.000Z"], + ) + .unwrap(); + let run_id = conn.last_insert_rowid(); + + let actor_columns: Vec = conn + .prepare("PRAGMA table_info(agent_run_actors)") + .unwrap() + .query_map([], |row| row.get(1)) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!( + actor_columns, + vec![ + "id", + "actor_key", + "run_id", + "native_actor_id", + "actor_type", + "display_name", + "started_at", + "last_activity_at", + "ended_at", + "metadata_json", + ] + ); + let evidence_columns: Vec = conn + .prepare("PRAGMA table_info(agent_run_worktrees)") + .unwrap() + .query_map([], |row| row.get(1)) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!( + evidence_columns, + vec![ + "id", + "relation_key", + "run_id", + "worktree_id", + "evidence_kind", + "evidence_source", + "trust_level", + "confidence", + "is_primary", + "first_seen_at", + "last_seen_at", + "metadata_json", + ] + ); + + conn.execute( + "INSERT INTO agent_run_actors + (actor_key, run_id, native_actor_id, actor_type, started_at, metadata_json) + VALUES ('actor-key-1', ?1, 'subagent-1', 'subagent', ?2, '{}')", + rusqlite::params![run_id, "2026-08-01T02:30:01.000Z"], + ) + .unwrap(); + assert!( + conn.execute( + "INSERT INTO agent_run_actors + (actor_key, run_id, native_actor_id, metadata_json) + VALUES ('actor-key-2', ?1, 'subagent-1', '{}')", + [run_id], + ) + .is_err(), + "native actor identity must dedupe within one run" + ); + assert!( + conn.execute( + "INSERT INTO agent_run_actors + (actor_key, run_id, native_actor_id, metadata_json) + VALUES ('actor-bad-json', ?1, 'subagent-2', '{')", + [run_id], + ) + .is_err(), + "actor metadata must be valid JSON" + ); + + let insert_relation = |relation_key: &str, + worktree_id: i64, + evidence_kind: &str, + evidence_source: &str, + trust: &str, + confidence: f64, + is_primary: i64, + last_seen: &str| { + conn.execute( + "INSERT INTO agent_run_worktrees + (relation_key, run_id, worktree_id, evidence_kind, evidence_source, + trust_level, confidence, is_primary, first_seen_at, last_seen_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?9)", + rusqlite::params![ + relation_key, + run_id, + worktree_id, + evidence_kind, + evidence_source, + trust, + confidence, + is_primary, + last_seen, + ], + ) + }; + insert_relation( + "rel-verified", + main_worktree, + "hook_cwd", + "hook:1", + "verified", + 1.0, + 1, + "2026-08-01T02:30:02.000Z", + ) + .unwrap(); + insert_relation( + "rel-claimed", + feature_worktree, + "transcript_project_path", + "log:2", + "claimed", + 0.8, + 0, + "2026-08-01T02:30:03.000Z", + ) + .unwrap(); + + assert!( + insert_relation( + "rel-duplicate", + main_worktree, + "hook_cwd", + "hook:1", + "verified", + 0.9, + 0, + "2026-08-01T02:30:04.000Z", + ) + .is_err(), + "the same evidence tuple must not create a second relation" + ); + assert!( + insert_relation( + "rel-confidence-high", + main_worktree, + "other", + "source:high", + "inferred", + 1.01, + 0, + "2026-08-01T02:30:04.000Z", + ) + .is_err(), + "confidence above one must be rejected" + ); + assert!( + insert_relation( + "rel-confidence-low", + main_worktree, + "other", + "source:low", + "inferred", + -0.01, + 0, + "2026-08-01T02:30:04.000Z", + ) + .is_err(), + "negative confidence must be rejected" + ); + assert!( + insert_relation( + "rel-bad-trust", + main_worktree, + "other", + "source:trust", + "magical", + 0.5, + 0, + "2026-08-01T02:30:04.000Z", + ) + .is_err(), + "unknown trust levels must be rejected" + ); + + let ordered: Vec = conn + .prepare( + "SELECT relation_key FROM agent_run_worktrees + WHERE run_id = ?1 + ORDER BY is_primary DESC, confidence DESC, last_seen_at DESC, id", + ) + .unwrap() + .query_map([run_id], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!(ordered, vec!["rel-verified", "rel-claimed"]); + let distinct_worktrees: i64 = conn + .query_row( + "SELECT COUNT(DISTINCT worktree_id) FROM agent_run_worktrees WHERE run_id = ?1", + [run_id], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(distinct_worktrees, 2, "one run may have worktree history"); + + for expected in [ + "idx_agent_run_actors_run", + "idx_agent_run_worktrees_run", + "idx_agent_run_worktrees_worktree", + ] { + let exists: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = ?1", + [expected], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(exists, 1, "missing {expected}"); + } +} + +#[test] +fn schema_43_fixture_upgrades_to_47_and_preserves_legacy_rows() { + const FIXTURE: &str = include_str!("../tests/fixtures/schema-43.sql"); + assert!(!FIXTURE.contains("jmagar")); + assert!(!FIXTURE.contains("/home/")); + + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("schema-43-upgrade.db"); + { + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute_batch(FIXTURE).unwrap(); + let version: i64 = conn + .query_row("SELECT MAX(version) FROM schema_migrations", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(version, 43); + } + + let config = test_storage_config(db_path); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + let version: i64 = conn + .query_row("SELECT MAX(version) FROM schema_migrations", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(version, 47); + + let legacy_log: (String, String, String) = conn + .query_row( + "SELECT hostname, message, ai_session_id FROM logs WHERE id = 1", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!( + legacy_log, + ( + "fixture-host".to_string(), + "synthetic legacy log".to_string(), + "fixture-session".to_string(), + ) + ); + + let rollup_count: i64 = conn + .query_row( + "SELECT event_count FROM ai_session_rollup + WHERE ai_project = 'fixture-project' + AND ai_tool = 'fixture-tool' + AND ai_session_id = 'fixture-session'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(rollup_count, 1); + + let integrity: String = conn + .query_row("PRAGMA integrity_check", [], |row| row.get(0)) + .unwrap(); + assert_eq!(integrity, "ok"); + let foreign_key_violation: Option = conn + .query_row("PRAGMA foreign_key_check", [], |row| row.get(0)) + .optional() + .unwrap(); + assert_eq!(foreign_key_violation, None); +} + +#[test] +fn graph_schema_enforces_vocabulary_and_dedup_keys() { + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("graph-dedup.db")); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, trust_level) + VALUES ('source_ip', '10.0.0.1:514', '10.0.0.1:514', 'verified')", + [], + ) + .unwrap(); + let duplicate = conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, trust_level) + VALUES ('source_ip', '10.0.0.1:514', 'duplicate', 'verified')", + [], + ); + assert!(duplicate.is_err(), "canonical entity identity must dedupe"); + + let bad_type = conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, trust_level) + VALUES ('same_window', 'bad', 'bad', 'verified')", + [], + ); + assert!(bad_type.is_err(), "unknown entity types must be rejected"); + + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, trust_level) + VALUES ('host', 'claimed-host', 'claimed-host', 'claimed')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, source_kind, source_id, trust_level) + VALUES ('reverse_proxy', 'proxy:example.test', 'example.test', + 'app_inventory', 'proxy:example.test', 'verified')", + [], + ) + .unwrap(); + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, source_kind, source_id, trust_level) + VALUES ('domain', 'example.test', 'example.test', + 'app_inventory', 'example.test', 'verified')", + [], + ) + .unwrap(); + let source_id: i64 = conn + .query_row( + "SELECT id FROM graph_entities WHERE entity_type = 'source_ip'", + [], + |row| row.get(0), + ) + .unwrap(); + let host_id: i64 = conn + .query_row( + "SELECT id FROM graph_entities WHERE entity_type = 'host'", + [], + |row| row.get(0), + ) + .unwrap(); + let proxy_id: i64 = conn + .query_row( + "SELECT id FROM graph_entities WHERE entity_type = 'reverse_proxy'", + [], + |row| row.get(0), + ) + .unwrap(); + let domain_id: i64 = conn + .query_row( + "SELECT id FROM graph_entities WHERE entity_type = 'domain'", + [], + |row| row.get(0), + ) + .unwrap(); + + conn.execute( + "INSERT INTO graph_entity_aliases + (entity_id, alias_type, alias_key, alias_value, source_kind, trust_level) + VALUES (?1, 'hostname', 'claimed-host', 'claimed-host', 'log', 'claimed')", + [host_id], + ) + .unwrap(); + let duplicate_alias = conn.execute( + "INSERT INTO graph_entity_aliases + (entity_id, alias_type, alias_key, alias_value, source_kind, trust_level) + VALUES (?1, 'hostname', 'claimed-host', 'claimed-host', 'log', 'claimed')", + [host_id], + ); + assert!(duplicate_alias.is_err(), "alias identity must dedupe"); + + conn.execute( + "INSERT INTO graph_relationships + (relationship_key, src_entity_id, dst_entity_id, relationship_type, + reason_code, trust_level, confidence, evidence_count) + VALUES ('source_ip:10.0.0.1:514->host:claimed-host', ?1, ?2, + 'observed_as', 'syslog_claimed_hostname', 'claimed', 0.60, 1)", + rusqlite::params![source_id, host_id], + ) + .unwrap(); + conn.execute( + "INSERT INTO graph_relationships + (relationship_key, src_entity_id, dst_entity_id, relationship_type, + reason_code, trust_level, confidence, evidence_count) + VALUES ('reverse_proxy:example.test->domain:example.test', + ?1, ?2, 'exposes_domain', 'reverse_proxy_config', + 'verified', 0.90, 1)", + rusqlite::params![proxy_id, domain_id], + ) + .unwrap(); + let duplicate_rel = conn.execute( + "INSERT INTO graph_relationships + (relationship_key, src_entity_id, dst_entity_id, relationship_type, + reason_code, trust_level, confidence, evidence_count) + VALUES ('source_ip:10.0.0.1:514->host:claimed-host', ?1, ?2, + 'observed_as', 'syslog_claimed_hostname', 'claimed', 0.60, 1)", + rusqlite::params![source_id, host_id], + ); + assert!(duplicate_rel.is_err(), "relationship key must dedupe"); + + let rel_id: i64 = conn + .query_row( + "SELECT id FROM graph_relationships + WHERE relationship_key = 'source_ip:10.0.0.1:514->host:claimed-host'", + [], + |row| row.get(0), + ) + .unwrap(); + conn.execute( + "INSERT INTO graph_relationship_evidence + (relationship_id, evidence_key, source_kind, source_id, observed_at, + reason_code, trust_level, safe_excerpt, evidence_count) + VALUES (?1, 'log:1:hostname:2026-01-01T00', 'log', '1', + '2026-01-01T00:00:00Z', 'syslog_claimed_hostname', + 'claimed', 'claimed-host', 3)", + [rel_id], + ) + .unwrap(); + let proxy_rel_id: i64 = conn + .query_row( + "SELECT id FROM graph_relationships + WHERE relationship_key = 'reverse_proxy:example.test->domain:example.test'", + [], + |row| row.get(0), + ) + .unwrap(); + conn.execute( + "INSERT INTO graph_relationship_evidence + (relationship_id, evidence_key, source_kind, source_id, observed_at, + reason_code, trust_level, safe_excerpt, evidence_count) + VALUES (?1, 'proxy:example.test:route', + 'app_inventory', 'proxy:example.test', + '2026-01-01T00:00:00Z', 'reverse_proxy_config', + 'verified', 'example.test routes through proxy config', 1)", + [proxy_rel_id], + ) + .unwrap(); + let duplicate_evidence = conn.execute( + "INSERT INTO graph_relationship_evidence + (relationship_id, evidence_key, source_kind, source_id, observed_at, + reason_code, trust_level, safe_excerpt, evidence_count) + VALUES (?1, 'log:1:hostname:2026-01-01T00', 'log', '1', + '2026-01-01T00:00:00Z', 'syslog_claimed_hostname', + 'claimed', 'claimed-host', 3)", + [rel_id], + ); + assert!( + duplicate_evidence.is_err(), + "evidence key must dedupe repeated samples" + ); + + let bad_same_window = conn.execute( + "INSERT INTO graph_relationships + (relationship_key, src_entity_id, dst_entity_id, relationship_type, + reason_code, trust_level) + VALUES ('bad-same-window', ?1, ?2, 'same_window', + 'syslog_claimed_hostname', 'correlated')", + rusqlite::params![source_id, host_id], + ); + assert!( + bad_same_window.is_err(), + "same_window must not be a persisted v1 relationship type" + ); +} + +#[test] +fn migration_30_widens_old_graph_constraints_and_preserves_rows() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("graph-migration-30.db"); + { + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute_batch( + "CREATE TABLE schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) + ); + WITH RECURSIVE versions(version) AS ( + SELECT 1 UNION ALL SELECT version + 1 FROM versions WHERE version < 29 + ) + INSERT INTO schema_migrations(version) SELECT version FROM versions; + CREATE TABLE maintenance_jobs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + kind TEXT NOT NULL, + status TEXT NOT NULL, + started_at TEXT NOT NULL, + finished_at TEXT, + result_json TEXT + ); + CREATE TABLE graph_entities ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entity_type TEXT NOT NULL CHECK (entity_type IN ( + 'host', 'container', 'service', 'app', 'source_ip', + 'ai_project', 'ai_session', 'error_signature' + )), + canonical_key TEXT NOT NULL, + display_label TEXT NOT NULL, + source_kind TEXT NOT NULL DEFAULT '', + source_id TEXT NOT NULL DEFAULT '', + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated' + )), + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(entity_type, canonical_key) + ); + CREATE TABLE graph_entity_aliases ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + entity_id INTEGER NOT NULL, + alias_type TEXT NOT NULL, + alias_key TEXT NOT NULL, + alias_value TEXT NOT NULL, + source_kind TEXT NOT NULL DEFAULT '', + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated' + )), + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(entity_id, alias_type, alias_key, source_kind) + ); + CREATE TABLE graph_relationships ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relationship_key TEXT NOT NULL UNIQUE, + src_entity_id INTEGER NOT NULL, + dst_entity_id INTEGER NOT NULL, + relationship_type TEXT NOT NULL CHECK (relationship_type IN ( + 'observed_as', 'runs_on', 'emitted_by', 'worked_on', + 'matches_signature' + )), + reason_code TEXT NOT NULL CHECK (reason_code IN ( + 'syslog_claimed_hostname', 'log_app_name', + 'docker_container_id', 'docker_service_label', + 'ai_session_project', 'heartbeat_host_state', + 'error_signature_match' + )), + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated' + )), + confidence REAL NOT NULL DEFAULT 0.0 CHECK (confidence >= 0.0 AND confidence <= 1.0), + evidence_count INTEGER NOT NULL DEFAULT 0 CHECK (evidence_count >= 0), + first_seen_at TEXT, + last_seen_at TEXT, + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(src_entity_id, dst_entity_id, relationship_type, relationship_key) + ); + CREATE TABLE graph_relationship_evidence ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relationship_id INTEGER NOT NULL, + evidence_key TEXT NOT NULL, + source_kind TEXT NOT NULL CHECK (source_kind IN ( + 'log', 'heartbeat', 'ai_session_rollup', 'error_signature' + )), + source_id TEXT NOT NULL DEFAULT '', + source_log_id INTEGER, + source_heartbeat_id INTEGER, + source_signature_hash TEXT, + observed_at TEXT NOT NULL, + reason_code TEXT NOT NULL CHECK (reason_code IN ( + 'syslog_claimed_hostname', 'log_app_name', + 'docker_container_id', 'docker_service_label', + 'ai_session_project', 'heartbeat_host_state', + 'error_signature_match' + )), + reason_text TEXT, + confidence_delta REAL NOT NULL DEFAULT 0.0 CHECK (confidence_delta >= 0.0 AND confidence_delta <= 1.0), + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'inferred', 'correlated' + )), + safe_excerpt TEXT, + metadata_path TEXT, + evidence_count INTEGER NOT NULL DEFAULT 1 CHECK (evidence_count >= 1), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(relationship_id, evidence_key) + ); + CREATE TABLE graph_projection_meta ( + id INTEGER PRIMARY KEY CHECK (id = 1), + projection_status TEXT NOT NULL DEFAULT 'pending', + last_started_at TEXT, + last_completed_at TEXT, + source_watermark TEXT NOT NULL DEFAULT '', + source_row_count INTEGER NOT NULL DEFAULT 0 CHECK (source_row_count >= 0), + entity_count INTEGER NOT NULL DEFAULT 0 CHECK (entity_count >= 0), + relationship_count INTEGER NOT NULL DEFAULT 0 CHECK (relationship_count >= 0), + evidence_count INTEGER NOT NULL DEFAULT 0 CHECK (evidence_count >= 0), + is_degraded INTEGER NOT NULL DEFAULT 0 CHECK (is_degraded IN (0, 1)), + last_error TEXT, + last_runtime_ms INTEGER NOT NULL DEFAULT 0 CHECK (last_runtime_ms >= 0), + last_chunk_count INTEGER NOT NULL DEFAULT 0 CHECK (last_chunk_count >= 0), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) + ); + INSERT INTO graph_projection_meta(id) VALUES (1); + INSERT INTO graph_entities + (id, entity_type, canonical_key, display_label, source_kind, source_id, trust_level) + VALUES + (1, 'source_ip', '10.0.0.1:514', '10.0.0.1:514', 'log', '1', 'verified'), + (2, 'host', 'claimed-host', 'claimed-host', 'log', '1', 'claimed'); + INSERT INTO graph_entity_aliases + (id, entity_id, alias_type, alias_key, alias_value, source_kind, trust_level) + VALUES (1, 2, 'hostname', 'claimed-host', 'claimed-host', 'log', 'claimed'); + INSERT INTO graph_relationships + (id, relationship_key, src_entity_id, dst_entity_id, relationship_type, + reason_code, trust_level, confidence, evidence_count) + VALUES (1, 'source_ip:10.0.0.1:514->host:claimed-host', 1, 2, + 'observed_as', 'syslog_claimed_hostname', 'claimed', 0.60, 1); + INSERT INTO graph_relationship_evidence + (id, relationship_id, evidence_key, source_kind, source_id, observed_at, + reason_code, trust_level, safe_excerpt, evidence_count) + VALUES (1, 1, 'log:1:hostname', 'log', '1', '2026-01-01T00:00:00Z', + 'syslog_claimed_hostname', 'claimed', 'claimed-host', 1);", + ) + .unwrap(); + } + + let pool = init_pool(&test_storage_config(db_path)).unwrap(); + let conn = pool.get().unwrap(); + assert_eq!( + conn.query_row( + "SELECT COUNT(*) FROM graph_relationship_evidence WHERE evidence_key = 'log:1:hostname'", + [], + |row| row.get::<_, i64>(0) + ) + .unwrap(), + 1 + ); + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, source_kind, source_id, trust_level) + VALUES ('compose_project', 'edgehost:edge', 'edge', + 'app_inventory', 'compose:edgehost', 'verified')", + [], + ) + .unwrap(); + let relationship_id = conn + .query_row( + "SELECT id FROM graph_relationships + WHERE relationship_key = 'source_ip:10.0.0.1:514->host:claimed-host'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(); + conn.execute( + "INSERT INTO graph_relationship_evidence + (relationship_id, evidence_key, source_kind, source_id, observed_at, + reason_code, trust_level, safe_excerpt, evidence_count) + VALUES (?1, 'inventory:route', 'app_inventory', 'proxy:edgehost', + '2026-01-01T00:00:00Z', 'reverse_proxy_config', + 'verified', 'proxy route', 1)", + rusqlite::params![relationship_id], + ) + .unwrap(); +} + +#[test] +fn graph_vocabulary_helpers_cover_schema_values() { + for value in ENTITY_TYPES { + assert!(is_known_entity_type(value), "missing entity type {value}"); + } + for value in RELATIONSHIP_TYPES { + assert!( + is_known_relationship_type(value), + "missing relationship type {value}" + ); + } + for value in REASON_CODES { + assert!(is_known_reason_code(value), "missing reason code {value}"); + } + for value in TRUST_LEVELS { + assert!(is_known_trust_level(value), "missing trust level {value}"); + } + for value in EVIDENCE_SOURCE_KINDS { + assert!( + is_known_evidence_source_kind(value), + "missing evidence source kind {value}" + ); + } + + assert!(!is_known_relationship_type("same_window")); + assert!(!is_known_entity_type("unknown")); + assert!(!is_known_evidence_source_kind("source_table")); +} + +#[test] +fn graph_lookup_indexes_support_expected_query_plans() { + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("graph-query-plan.db")); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + let plan_details = |sql: &str| -> Vec { + let mut stmt = conn.prepare(sql).unwrap(); + stmt.query_map([], |row| row.get::<_, String>(3)) + .unwrap() + .collect::>>() + .unwrap() + }; + + let entity_plan = plan_details( + "EXPLAIN QUERY PLAN + SELECT id FROM graph_entities + WHERE entity_type = 'host' AND canonical_key = 'devhost'", + ); + assert!( + entity_plan + .iter() + .any(|p| p.contains("SEARCH graph_entities")), + "entity lookup must use an indexed search: {entity_plan:?}" + ); + + let alias_plan = plan_details( + "EXPLAIN QUERY PLAN + SELECT entity_id FROM graph_entity_aliases + WHERE alias_type = 'hostname' AND alias_key = 'devhost'", + ); + assert!( + alias_plan + .iter() + .any(|p| p.contains("SEARCH graph_entity_aliases")), + "alias lookup must use an indexed search: {alias_plan:?}" + ); + + let outgoing_plan = plan_details( + "EXPLAIN QUERY PLAN + SELECT id FROM graph_relationships + WHERE src_entity_id = 1 AND relationship_type = 'observed_as' + ORDER BY last_seen_at DESC LIMIT 50", + ); + assert!( + outgoing_plan + .iter() + .any(|p| p.contains("SEARCH graph_relationships")), + "outgoing relationship lookup must use an indexed search: {outgoing_plan:?}" + ); + assert!( + !outgoing_plan + .iter() + .any(|p| p == "SCAN graph_relationships"), + "outgoing relationship lookup must not full-scan relationship table: {outgoing_plan:?}" + ); + + let incoming_plan = plan_details( + "EXPLAIN QUERY PLAN + SELECT id FROM graph_relationships + WHERE dst_entity_id = 2 AND relationship_type = 'observed_as' + ORDER BY last_seen_at DESC LIMIT 50", + ); + assert!( + incoming_plan + .iter() + .any(|p| p.contains("SEARCH graph_relationships")), + "incoming relationship lookup must use an indexed search: {incoming_plan:?}" + ); + assert!( + !incoming_plan + .iter() + .any(|p| p == "SCAN graph_relationships"), + "incoming relationship lookup must not full-scan relationship table: {incoming_plan:?}" + ); + + let evidence_plan = plan_details( + "EXPLAIN QUERY PLAN + SELECT id FROM graph_relationship_evidence + WHERE relationship_id = 1 + ORDER BY observed_at DESC LIMIT 3", + ); + assert!( + evidence_plan + .iter() + .any(|p| p.contains("SEARCH graph_relationship_evidence")), + "evidence lookup must use an indexed search: {evidence_plan:?}" + ); + assert!( + !evidence_plan + .iter() + .any(|p| p == "SCAN graph_relationship_evidence"), + "evidence lookup must not full-scan evidence table: {evidence_plan:?}" + ); + + let source_cleanup_plan = plan_details( + "EXPLAIN QUERY PLAN + SELECT id FROM graph_relationship_evidence + WHERE source_kind = 'log' AND source_id = '1'", + ); + assert!( + source_cleanup_plan + .iter() + .any(|p| p.contains("SEARCH graph_relationship_evidence")), + "source cleanup lookup must use an indexed search: {source_cleanup_plan:?}" + ); +} + +#[test] +fn heartbeat_schema_enforces_idempotency_key() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("heartbeat-unique.db"); + let config = test_storage_config(db_path); + + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + let insert = "INSERT INTO host_heartbeats ( + host_id, hostname, source_ip, sampled_at, received_at, boot_id, + uptime_secs, sequence, collection_ms, partial, agent_version, os, architecture + ) VALUES ( + 'host-1', 'box-a', '127.0.0.1:3100', '2026-05-25T00:00:00Z', + '2026-05-25T00:00:01Z', 'boot-a', 60, 1, 12, 0, '0.1.0', 'linux', 'x86_64' + )"; + conn.execute(insert, []).unwrap(); + let duplicate = conn.execute(insert, []); + assert!( + duplicate.is_err(), + "duplicate heartbeat key must be rejected" + ); +} + +#[test] +fn init_db_adds_ai_session_metadata_columns() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let config = crate::config::StorageConfig { + db_path, + ..Default::default() + }; + + let _pool = init_pool(&config).unwrap(); + let conn = rusqlite::Connection::open(&config.db_path).unwrap(); + for column in [ + "ai_tool", + "ai_project", + "ai_session_id", + "ai_transcript_path", + "metadata_json", + ] { + let exists: i64 = conn + .query_row( + "SELECT COUNT(*) FROM pragma_table_info('logs') WHERE name = ?1", + [column], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(exists, 1, "missing column {column}"); + } +} + +#[test] +fn init_db_creates_partial_ai_metadata_indexes() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let config = crate::config::StorageConfig { + db_path, + ..Default::default() + }; + + let _pool = init_pool(&config).unwrap(); + let conn = rusqlite::Connection::open(&config.db_path).unwrap(); + let indexes: Vec<(String, String)> = { + let mut stmt = conn + .prepare( + "SELECT name, sql FROM sqlite_schema + WHERE type = 'index' + AND name IN ( + 'idx_logs_ai_project_time', + 'idx_logs_ai_session', + 'idx_logs_ai_transcript_path' + ) + ORDER BY name", + ) + .unwrap(); + stmt.query_map([], |row| Ok((row.get(0)?, row.get(1)?))) + .unwrap() + .collect::>>() + .unwrap() + }; + + assert_eq!(indexes.len(), 3); + for (_, sql) in indexes { + assert!(sql.contains("WHERE")); + assert!(sql.contains("IS NOT NULL")); + } +} + +#[test] +fn migrations_23_24_yield_final_covering_index_set() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let config = crate::config::StorageConfig { + db_path, + ..Default::default() + }; + + let _pool = init_pool(&config).unwrap(); + let conn = rusqlite::Connection::open(&config.db_path).unwrap(); + + let index_sql = |name: &str| -> Option { + conn.query_row( + "SELECT sql FROM sqlite_schema WHERE type = 'index' AND name = ?1", + [name], + |row| row.get::<_, String>(0), + ) + .ok() + }; + + // Migration 23's interim AI index is superseded and DROPped by migration 24. + assert!( + index_sql("idx_logs_ai_project_cover").is_none(), + "migration 24 must drop the superseded idx_logs_ai_project_cover" + ); + + // errors covering index (migration 23) survives. + let sev_cover = index_sql("idx_logs_sev_host_time").expect("severity/host covering index"); + assert!(sev_cover.contains("severity")); + assert!(sev_cover.contains("hostname")); + assert!(sev_cover.contains("timestamp")); + + // Timestamp-positioned AI covering index (migration 24) serves ai projects + ai blocks. + let ts_cover = index_sql("idx_logs_ai_project_ts_cover").expect("ai project ts-covering index"); + // Column order matters: ai_project, THEN timestamp (seekable), then the covered cols. + let p = ts_cover.find("ai_project").unwrap(); + let t = ts_cover.find("timestamp").unwrap(); + let tool = ts_cover.find("ai_tool").unwrap(); + assert!( + p < t && t < tool, + "order must be ai_project, timestamp, ai_tool, ..." + ); + assert!(ts_cover.contains("ai_session_id")); + assert!(ts_cover.contains("ai_project IS NOT NULL")); + + // ai tools covering index (migration 24). + let tool_cover = index_sql("idx_logs_ai_tool_cover").expect("ai tool covering index"); + assert!(tool_cover.contains("ai_tool")); + assert!(tool_cover.contains("ai_session_id")); + assert!(tool_cover.contains("timestamp")); + + // Migration 24 only ANALYZEs when `logs` already has rows, so this empty + // fresh DB writes no `sqlite_stat1` (by design — empty-table stats mislead + // the planner). The populated-DB ANALYZE path is covered by live validation. + + for v in [23, 24] { + let applied: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = ?1", + [v], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(applied, 1, "migration {v} must be recorded"); + } +} + +#[test] +fn migration_32_covers_graph_to_log_join() { + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("graph-log-cover.db")); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + // Index DDL is present and carries the expected column order. + let index_sql = |name: &str| -> Option { + conn.query_row( + "SELECT sql FROM sqlite_schema WHERE type = 'index' AND name = ?1", + [name], + |row| row.get::<_, String>(0), + ) + .ok() + }; + + let cover = index_sql("idx_logs_hostname_appname_time") + .expect("graph→log covering index must exist after migration 32"); + let h = cover.find("hostname").unwrap(); + let a = cover.find("app_name").unwrap(); + let t = cover.find("timestamp").unwrap(); + assert!( + h < a && a < t, + "column order must be hostname, app_name, timestamp: {cover}" + ); + + let session_cover = index_sql("idx_logs_ai_session_time") + .expect("session-anchored covering index must exist after migration 32"); + assert!(session_cover.contains("ai_session_id")); + assert!(session_cover.contains("timestamp")); + assert!(session_cover.contains("ai_session_id IS NOT NULL")); + + // The planner must pick the covering index for the topic_correlate join shape: + // hostname IN (...) AND timestamp BETWEEN ... AND app_name = ... + let plan_details = |sql: &str| -> Vec { + let mut stmt = conn.prepare(sql).unwrap(); + stmt.query_map([], |row| row.get::<_, String>(3)) + .unwrap() + .collect::>>() + .unwrap() + }; + let join_plan = plan_details( + "EXPLAIN QUERY PLAN + SELECT id FROM logs + WHERE hostname IN ('devhost', 'edgehost') + AND app_name = 'swag' + AND timestamp BETWEEN '2026-06-18T00:00:00Z' AND '2026-06-18T01:00:00Z'", + ); + assert!( + join_plan + .iter() + .any(|p| p.contains("idx_logs_hostname_appname_time")), + "graph→log join must use idx_logs_hostname_appname_time: {join_plan:?}" + ); + + let applied: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 32", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(applied, 1, "migration 32 must be recorded"); +} + +#[test] +fn init_db_creates_inventory_stats_tables_and_triggers() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let config = crate::config::StorageConfig { + db_path, + ..Default::default() + }; + + let _pool = init_pool(&config).unwrap(); + let conn = rusqlite::Connection::open(&config.db_path).unwrap(); + for table in [ + "app_inventory_stats", + "app_host_inventory_stats", + "source_ip_inventory_stats", + "source_ip_host_inventory_stats", + ] { + let exists: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1", + [table], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(exists, 1, "missing table {table}"); + } + for trigger in [ + "logs_inventory_app_ai", + "logs_inventory_app_ad", + "logs_inventory_source_ip_ai", + "logs_inventory_source_ip_ad", + ] { + let exists: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'trigger' AND name = ?1", + [trigger], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(exists, 1, "missing trigger {trigger}"); + } +} + +#[test] +fn inventory_backfill_processes_existing_logs_in_chunks() { + let dir = tempfile::tempdir().unwrap(); + let config = crate::config::StorageConfig { + db_path: dir.path().join("test.db"), + ..Default::default() + }; + let pool = init_pool(&config).unwrap(); + let mut entries = Vec::new(); + for i in 0..3 { + entries.push(LogBatchEntry { + timestamp: format!("2026-01-01T00:00:0{i}Z"), + hostname: format!("host-{i}"), + facility: None, + severity: "info".to_string(), + app_name: Some("nginx".to_string()), + process_id: None, + message: "hello".to_string(), + raw: "hello".to_string(), + source_ip: "10.0.0.1:514".to_string(), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + }); + } + insert_logs_batch(&pool, &entries).unwrap(); + + let conn = pool.get().unwrap(); + conn.execute("DELETE FROM app_inventory_stats", []).unwrap(); + conn.execute("DELETE FROM app_host_inventory_stats", []) + .unwrap(); + conn.execute("DELETE FROM source_ip_inventory_stats", []) + .unwrap(); + conn.execute("DELETE FROM source_ip_host_inventory_stats", []) + .unwrap(); + drop(conn); + + backfill_inventory_stats(&pool).unwrap(); + + let conn = pool.get().unwrap(); + let complete: bool = conn + .query_row( + "SELECT completed_at IS NOT NULL + FROM inventory_backfill_state + WHERE name = 'app_source_inventory'", + [], + |row| row.get(0), + ) + .unwrap(); + assert!(complete); + let app_count: i64 = conn + .query_row( + "SELECT log_count FROM app_inventory_stats WHERE app_name = 'nginx'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(app_count, 3); + let source_count: i64 = conn + .query_row( + "SELECT log_count FROM source_ip_inventory_stats WHERE source_ip = '10.0.0.1:514'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(source_count, 3); +} + +#[test] +fn init_db_adds_transcript_checkpoint_tables() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let config = crate::config::StorageConfig { + db_path, + ..Default::default() + }; + + let _pool = init_pool(&config).unwrap(); + let conn = rusqlite::Connection::open(&config.db_path).unwrap(); + for table in [ + "transcript_sources", + "transcript_import_records", + "transcript_parse_errors", + ] { + let exists: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = ?1", + [table], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(exists, 1, "missing table {table}"); + } + let preview_not_null: i64 = conn + .query_row( + "SELECT [notnull] FROM pragma_table_info('transcript_parse_errors') WHERE name = 'record_preview'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(preview_not_null, 1); +} + +#[test] +fn init_db_migrates_legacy_ai_schema_without_losing_logs() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("legacy-ai.db"); + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute_batch( + " + CREATE TABLE logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + hostname TEXT NOT NULL, + facility TEXT, + severity TEXT NOT NULL, + app_name TEXT, + process_id TEXT, + message TEXT NOT NULL, + raw TEXT NOT NULL, + received_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + source_ip TEXT NOT NULL DEFAULT '' + ); + CREATE VIRTUAL TABLE logs_fts USING fts5( + message, + content='logs', + content_rowid='id', + tokenize='porter unicode61' + ); + CREATE TABLE hosts ( + hostname TEXT PRIMARY KEY, + first_seen TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + last_seen TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + log_count INTEGER NOT NULL DEFAULT 0 + ); + CREATE TABLE schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) + ); + INSERT INTO schema_migrations(version) VALUES (1), (2), (3); + INSERT INTO logs(timestamp, hostname, facility, severity, app_name, process_id, message, raw, source_ip) + VALUES ('2026-05-11T00:00:00Z', 'legacy-host', 'local0', 'info', 'legacy', NULL, 'legacy preserved', 'legacy preserved', '127.0.0.1:514'); + INSERT INTO logs_fts(rowid, message) VALUES (1, 'legacy preserved'); + INSERT INTO hosts(hostname, log_count) VALUES ('legacy-host', 1); + ", + ) + .unwrap(); + drop(conn); + + let pool = init_pool(&test_storage_config(db_path)).unwrap(); + let conn = pool.get().unwrap(); + for column in [ + "ai_tool", + "ai_project", + "ai_session_id", + "ai_transcript_path", + "metadata_json", + ] { + let exists: i64 = conn + .query_row( + "SELECT COUNT(*) FROM pragma_table_info('logs') WHERE name = ?1", + [column], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(exists, 1, "missing migrated column {column}"); + } + for version in [4, 5, 6] { + let applied: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = ?1", + [version], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(applied, 1, "missing migration {version}"); + } + let preserved: String = conn + .query_row( + "SELECT message FROM logs WHERE hostname = 'legacy-host'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(preserved, "legacy preserved"); +} + +#[test] +fn migration_13_adds_enrichment_columns() { + let dir = tempfile::tempdir().unwrap(); + let config = crate::config::StorageConfig { + db_path: dir.path().join("test.db"), + wal_mode: true, + pool_size: 1, + ..Default::default() + }; + let pool = init_pool(&config).expect("init_pool ok"); + let conn = pool.get().unwrap(); + + let cols: Vec = conn + .prepare("PRAGMA table_info(logs)") + .unwrap() + .query_map([], |r| r.get::<_, String>(1)) + .unwrap() + .filter_map(Result::ok) + .collect(); + + for expected in [ + "http_status", + "auth_outcome", + "dns_blocked", + "event_action", + "parse_error", + ] { + assert!( + cols.contains(&expected.to_string()), + "missing column {expected}" + ); + } + + let indices: Vec = conn + .prepare("SELECT name FROM sqlite_master WHERE type='index' AND tbl_name='logs'") + .unwrap() + .query_map([], |r| r.get::<_, String>(0)) + .unwrap() + .filter_map(Result::ok) + .collect(); + + for expected in [ + "idx_logs_http_status_time", + "idx_logs_auth_outcome_time", + "idx_logs_dns_blocked_time", + "idx_logs_event_action_time", + ] { + assert!( + indices.contains(&expected.to_string()), + "missing index {expected}" + ); + } + + let version_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 13", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(version_count, 1, "migration 13 row not recorded"); +} + +#[test] +fn migration_13_tolerates_existing_columns_without_version_row() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("migration-13-drift.db"); + let config = crate::config::StorageConfig { + db_path: db_path.clone(), + wal_mode: true, + pool_size: 1, + ..Default::default() + }; + let pool = init_pool(&config).expect("initial init_pool ok"); + drop(pool); + + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute("DELETE FROM schema_migrations WHERE version = 13", []) + .unwrap(); + conn.execute("DROP INDEX idx_logs_event_action_time", []) + .unwrap(); + drop(conn); + + let pool = init_pool(&config).expect("re-init should repair migration drift"); + let conn = pool.get().unwrap(); + let version_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 13", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(version_count, 1, "migration 13 row not restored"); + + let index_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'idx_logs_event_action_time'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(index_count, 1, "migration 13 index not restored"); +} + +#[test] +fn transcript_import_identity_enforces_uniqueness() { + let dir = tempfile::tempdir().unwrap(); + let config = crate::config::StorageConfig { + db_path: dir.path().join("test.db"), + ..Default::default() + }; + + let _pool = init_pool(&config).unwrap(); + let conn = rusqlite::Connection::open(&config.db_path).unwrap(); + conn.execute( + "INSERT INTO transcript_sources (canonical_path, source_kind) VALUES (?1, ?2)", + rusqlite::params!["/tmp/session.jsonl", "explicit_file"], + ) + .unwrap(); + let source_id = conn.last_insert_rowid(); + conn.execute( + "INSERT INTO transcript_import_records (source_id, record_key) VALUES (?1, ?2)", + rusqlite::params![source_id, "record-1"], + ) + .unwrap(); + let err = conn + .execute( + "INSERT INTO transcript_import_records (source_id, record_key) VALUES (?1, ?2)", + rusqlite::params![source_id, "record-1"], + ) + .unwrap_err(); + assert!(matches!(err, rusqlite::Error::SqliteFailure(_, _))); +} + +/// Reproduces the post-crash state of Migration 22 (bead syslog-mcp-tfr0): a +/// crash between the `ALTER TABLE ... ADD COLUMN` statements and the version +/// marker leaves the watermark columns present but version 22 absent from +/// `schema_migrations`. We reach that identical on-disk state cheaply by +/// migrating clean to head, then deleting only the version-22 marker row. +/// +/// On the pre-fix (bare `execute_batch`) code this FAILS: re-running `init_pool` +/// re-issues the unguarded ALTERs and aborts with "duplicate column name". The +/// Style-C rewrite guards each ALTER with `add_column_if_missing` and stamps the +/// version with `INSERT OR IGNORE`, so `init_pool` converges (reentrant) and the +/// partial state becomes crash-impossible (a real mid-tx crash now rolls back +/// both columns and the marker atomically). +#[test] +fn migration_22_converges_from_partial_apply() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("partial_m22.db"); + let config = test_storage_config(db_path.clone()); + + // 1. Migrate a clean DB to head (version 22, both columns present). + let pool = init_pool(&config).unwrap(); + { + let conn = pool.get().unwrap(); + // Sanity: migration 22 specifically is applied, with the columns present. + // Assert on version 22 directly (not MAX(version)) so a future migration 23 + // cannot break this test even though migration 22 is correctly applied. + let m22_applied: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 22", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(m22_applied, 1, "fixture must reach migration 22"); + for column in ["source_row_count", "source_max_id"] { + assert!( + column_exists(&conn, "ai_session_rollup_meta", column).unwrap(), + "fixture must have column {column}" + ); + } + // 2. Recreate the post-crash state: columns present, marker absent. + conn.execute("DELETE FROM schema_migrations WHERE version = 22", []) + .unwrap(); + } + drop(pool); // release the pooled connections / file handles + + // 3. Re-running init_pool must converge, not brick on "duplicate column name". + let pool = + init_pool(&config).expect("init_pool must be reentrant after a partial migration 22 apply"); + let conn = pool.get().unwrap(); + + // Assert migration 22 specifically was re-stamped (not MAX(version)) so a + // future migration 23 cannot mask a missing 22 marker / break this test. + let m22_applied: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 22", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(m22_applied, 1, "version marker must be re-stamped to 22"); + + for column in ["source_row_count", "source_max_id"] { + assert!( + column_exists(&conn, "ai_session_rollup_meta", column).unwrap(), + "watermark column {column} must remain present after convergence" + ); + } +} + +/// Regression guard (bead syslog-mcp-tfr0): running `init_pool` twice against the +/// same file must both succeed. This passes on the pre-fix code too — it is NOT +/// the bug-prover (`migration_22_converges_from_partial_apply` is) — it just pins +/// the idempotent-on-clean-reopen behaviour so a future migration change can't +/// silently break it. +#[test] +fn init_pool_is_idempotent_when_run_twice() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("idempotent.db"); + let config = test_storage_config(db_path); + + let pool = init_pool(&config).expect("first init_pool must succeed"); + drop(pool); + + let pool = init_pool(&config).expect("second init_pool on same file must succeed"); + let conn = pool.get().unwrap(); + // Assert migration 22 specifically is applied (not MAX(version)) so a future + // migration 23 cannot break this test even though 22 is correctly applied. + let m22_applied: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 22", + [], + |r| r.get(0), + ) + .unwrap(); + assert_eq!(m22_applied, 1); +} + +#[test] +fn migration_18_re_stamps_when_restarting_column_already_exists() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("partial-m18.db"); + let config = test_storage_config(db_path.clone()); + + let pool = init_pool(&config).unwrap(); + { + let conn = pool.get().unwrap(); + assert!( + column_exists(&conn, "heartbeat_containers", "restarting").unwrap(), + "fixture must have heartbeat_containers.restarting" + ); + conn.execute("DELETE FROM schema_migrations WHERE version = 18", []) + .unwrap(); + } + drop(pool); + + let pool = init_pool(&config).expect("migration 18 must converge with existing column"); + let conn = pool.get().unwrap(); + assert!( + column_exists(&conn, "heartbeat_containers", "restarting").unwrap(), + "restarting column must remain present" + ); + let version_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 18", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(version_count, 1, "migration 18 marker must be restored"); +} + +#[test] +fn migration_28_repairs_missing_runtime_metric_column_without_duplicate_marker() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("partial-m28.db"); + let config = test_storage_config(db_path.clone()); + + let pool = init_pool(&config).unwrap(); + { + let conn = pool.get().unwrap(); + conn.execute( + "UPDATE graph_projection_meta + SET last_runtime_ms = 4242, + last_chunk_count = 7 + WHERE id = 1", + [], + ) + .unwrap(); + conn.execute("DELETE FROM schema_migrations WHERE version = 28", []) + .unwrap(); + conn.execute( + "ALTER TABLE graph_projection_meta DROP COLUMN last_chunk_count", + [], + ) + .unwrap(); + } + drop(pool); + + let pool = init_pool(&config).expect("migration 28 must repair a missing runtime column"); + let conn = pool.get().unwrap(); + assert!( + column_exists(&conn, "graph_projection_meta", "last_runtime_ms").unwrap(), + "existing metric column must remain present" + ); + assert!( + column_exists(&conn, "graph_projection_meta", "last_chunk_count").unwrap(), + "missing metric column must be restored" + ); + let version_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 28", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + version_count, 1, + "migration 28 marker must be restored exactly once" + ); + let runtime_ms: i64 = conn + .query_row( + "SELECT last_runtime_ms FROM graph_projection_meta WHERE id = 1", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(runtime_ms, 4242, "existing metric data must survive repair"); +} + +/// Golden old-schema fixture: the exact v0.2.6 schema (pre-migration-framework +/// — no schema_migrations table, no ai_* columns, no metadata_json). Frozen +/// from `git show v0.2.6:src/db.rs`; do not "modernize" it — its purpose is to +/// represent a real old installation. +const V0_2_6_SCHEMA: &str = " + CREATE TABLE IF NOT EXISTS logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + hostname TEXT NOT NULL, + facility TEXT, + severity TEXT NOT NULL, + app_name TEXT, + process_id TEXT, + message TEXT NOT NULL, + raw TEXT NOT NULL, + received_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + source_ip TEXT NOT NULL DEFAULT '' + ); + + CREATE INDEX IF NOT EXISTS idx_logs_timestamp ON logs(timestamp); + CREATE INDEX IF NOT EXISTS idx_logs_hostname ON logs(hostname); + CREATE INDEX IF NOT EXISTS idx_logs_severity ON logs(severity); + CREATE INDEX IF NOT EXISTS idx_logs_app_name ON logs(app_name); + CREATE INDEX IF NOT EXISTS idx_logs_host_time ON logs(hostname, timestamp); + CREATE INDEX IF NOT EXISTS idx_logs_sev_time ON logs(severity, timestamp); + CREATE INDEX IF NOT EXISTS idx_logs_received_at ON logs(received_at); + + CREATE VIRTUAL TABLE IF NOT EXISTS logs_fts USING fts5( + message, + content='logs', + content_rowid='id', + tokenize='porter unicode61' + ); + + CREATE TRIGGER IF NOT EXISTS logs_ai AFTER INSERT ON logs BEGIN + INSERT INTO logs_fts(rowid, message) VALUES (new.id, new.message); + END; + + CREATE TABLE IF NOT EXISTS hosts ( + hostname TEXT PRIMARY KEY, + first_seen TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + last_seen TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + log_count INTEGER NOT NULL DEFAULT 0 + ); +"; + +/// full-review TH2: every migration was previously tested only from CLEAN +/// temp DBs, so a migration that works against `CREATE`-fresh state but +/// breaks against populated old-shape tables would pass CI and brick real +/// upgrades. This walks the ENTIRE chain against a populated v0.2.6 database +/// and asserts: head version reached, pre-existing rows survive and remain +/// FTS-searchable, and a second run is a no-op. +#[test] +fn full_migration_chain_upgrades_populated_v0_2_6_database() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("v0_2_6-upgrade.db"); + + { + let conn = rusqlite::Connection::open(&db_path).unwrap(); + conn.execute_batch(V0_2_6_SCHEMA).unwrap(); + for (ts, host, msg) in [ + ( + "2025-06-01T00:00:00Z", + "old-host-a", + "legacy kernel panic message", + ), + ( + "2025-06-02T00:00:00Z", + "old-host-b", + "legacy nginx upstream error", + ), + ] { + conn.execute( + "INSERT INTO logs (timestamp, hostname, severity, message, raw, received_at, source_ip) + VALUES (?1, ?2, 'err', ?3, ?3, ?1, '192.168.1.50:514')", + rusqlite::params![ts, host, msg], + ) + .unwrap(); + conn.execute( + "INSERT INTO hosts (hostname, first_seen, last_seen, log_count) + VALUES (?1, ?2, ?2, 1) + ON CONFLICT(hostname) DO NOTHING", + rusqlite::params![host, ts], + ) + .unwrap(); + } + } + + // Walk the full migration chain (plus the auto_vacuum conversion VACUUM). + let config = test_storage_config(db_path.clone()); + let pool = init_pool(&config).expect("full migration chain must apply to a populated old DB"); + + let head_version: i64 = { + let conn = pool.get().unwrap(); + conn.query_row("SELECT MAX(version) FROM schema_migrations", [], |r| { + r.get(0) + }) + .unwrap() + }; + assert!( + head_version >= 31, + "expected migration head >= 31, got {head_version}" + ); + + // Pre-existing rows survived and the FTS index still finds them. + { + let conn = pool.get().unwrap(); + let count: i64 = conn + .query_row("SELECT COUNT(*) FROM logs", [], |r| r.get(0)) + .unwrap(); + assert_eq!(count, 2, "old rows must survive the migration chain"); + } + let results = crate::search_logs( + &pool, + &crate::SearchParams { + query: Some("legacy".to_string()), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(results.len(), 2, "migrated rows must stay FTS-searchable"); + + // New-schema columns are live: a current-shape insert works. + insert_logs_batch( + &pool, + &[LogBatchEntry { + timestamp: "2026-01-01T00:00:00Z".to_string(), + hostname: "new-host".to_string(), + facility: None, + severity: "info".to_string(), + app_name: Some("upgrade-test".to_string()), + process_id: None, + message: "post-upgrade insert".to_string(), + raw: "post-upgrade insert".to_string(), + source_ip: "127.0.0.1:514".to_string(), + docker_checkpoint: None, + ai_tool: Some("claude-code".to_string()), + ai_project: Some("/tmp/project".to_string()), + ai_session_id: None, + ai_transcript_path: None, + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + }], + ) + .expect("current-shape insert must work after upgrade"); + + drop(pool); + + // Idempotency: a second init on the upgraded DB is a clean no-op. + let pool2 = init_pool(&config).expect("re-running init on an upgraded DB must succeed"); + let conn = pool2.get().unwrap(); + let head_again: i64 = conn + .query_row("SELECT MAX(version) FROM schema_migrations", [], |r| { + r.get(0) + }) + .unwrap(); + assert_eq!(head_again, head_version); + let count: i64 = conn + .query_row("SELECT COUNT(*) FROM logs", [], |r| r.get(0)) + .unwrap(); + assert_eq!(count, 3); +} + +#[test] +fn migration_37_creates_llm_invocations_table() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let config = test_storage_config(db_path); + let pool = init_pool(&config).expect("init_pool should succeed"); + let conn = pool.get().unwrap(); + + // Table exists with the exact locked column set. + let mut stmt = conn + .prepare( + "SELECT COUNT(*) FROM pragma_table_info('llm_invocations') WHERE name IN ( + 'id','started_at','finished_at','duration_ms','caller_surface','action', + 'provider','model','program','incident_id','ai_tool','ai_project', + 'ai_session_id','evidence_counts_json','prompt_bytes','output_bytes', + 'status','error','metadata_json' + )", + ) + .unwrap(); + let count: i64 = stmt.query_row([], |row| row.get(0)).unwrap(); + assert_eq!(count, 19, "llm_invocations must have all 19 locked columns"); + drop(stmt); + + // Migration is recorded and idempotent. + let version: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 37", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(version, 1); + + // Re-running init_pool (simulating a restart) must not error or duplicate the row. + drop(conn); + drop(pool); + let pool2 = init_pool(&config).expect("second init_pool should succeed"); + let conn2 = pool2.get().unwrap(); + let version2: i64 = conn2 + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 37", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + version2, 1, + "migration 37 must be idempotent across restarts" + ); +} + +#[test] +fn migration_37_indexes_exist() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let config = test_storage_config(db_path); + let pool = init_pool(&config).expect("init_pool should succeed"); + let conn = pool.get().unwrap(); + for idx in [ + "idx_llm_invocations_started", + "idx_llm_invocations_action_started", + "idx_llm_invocations_status_started", + ] { + let count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type='index' AND name = ?1", + [idx], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(count, 1, "expected index {idx} to exist"); + } +} + +// PR #106 reconciliation fix (code-reviewer): if the process is killed +// between `LlmRunner::write_start_row` (status='running') and the matching +// finish-row write, the audit row is orphaned in 'running' forever — no +// process is left to finish it. Authoritative server startup reconciles +// orphaned rows after opening the pool. Query-only CLI processes deliberately +// skip this step because they can coexist with live server-owned work. +#[test] +fn server_start_reconciles_orphaned_running_work_without_pool_side_effects() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let config = test_storage_config(db_path); + let pool = init_pool(&config).expect("init_pool should succeed"); + let conn = pool.get().unwrap(); + + conn.execute( + "INSERT INTO llm_invocations + (id, started_at, caller_surface, action, provider, status) + VALUES ('llm-orphan', strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), 'test', 'ai_assess', 'gemini-cli', 'running')", + [], + ) + .expect("seed orphaned running row"); + + // A concurrently-'success' row (as if it finished cleanly before the + // crash) must be left untouched by the reconciliation. + conn.execute( + "INSERT INTO llm_invocations + (id, started_at, finished_at, caller_surface, action, provider, status) + VALUES ('llm-clean', strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), 'test', 'ai_assess', 'gemini-cli', 'success')", + [], + ) + .expect("seed clean success row"); + + conn.execute( + "INSERT INTO maintenance_jobs (kind, status, started_at) + VALUES ('db_integrity', 'running', strftime('%Y-%m-%dT%H:%M:%fZ', 'now'))", + [], + ) + .expect("seed running maintenance job"); + + drop(conn); + drop(pool); + + // Merely opening another pool, as a local CLI does, must not alter work + // still owned by the running server. + let pool2 = init_pool(&config).expect("second init_pool should succeed"); + let conn2 = pool2.get().unwrap(); + + let untouched: String = conn2 + .query_row( + "SELECT status FROM llm_invocations WHERE id = 'llm-orphan'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(untouched, "running"); + let maintenance_untouched: String = conn2 + .query_row( + "SELECT status FROM maintenance_jobs WHERE kind = 'db_integrity'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(maintenance_untouched, "running"); + drop(conn2); + + reconcile_interrupted_server_work(&pool2).unwrap(); + let conn2 = pool2.get().unwrap(); + + let (status, finished_at, error): (String, Option, Option) = conn2 + .query_row( + "SELECT status, finished_at, error FROM llm_invocations WHERE id = 'llm-orphan'", + [], + |row| Ok((row.get(0)?, row.get(1)?, row.get(2)?)), + ) + .unwrap(); + assert_eq!(status, "interrupted"); + assert!( + finished_at.is_some(), + "reconciled row must get a finished_at timestamp" + ); + assert_eq!(error.as_deref(), Some("interrupted by server restart")); + + let (maintenance_status, maintenance_error): (String, String) = conn2 + .query_row( + "SELECT status, json_extract(result_json, '$.error') + FROM maintenance_jobs WHERE kind = 'db_integrity'", + [], + |row| Ok((row.get(0)?, row.get(1)?)), + ) + .unwrap(); + assert_eq!(maintenance_status, "failed"); + assert_eq!(maintenance_error, "interrupted by server restart"); + + let clean_status: String = conn2 + .query_row( + "SELECT status FROM llm_invocations WHERE id = 'llm-clean'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + clean_status, "success", + "reconciliation must not touch rows that already reached a terminal status" + ); +} + +#[test] +fn migration_38_creates_ai_skill_events_table() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let config = test_storage_config(db_path); + let pool = init_pool(&config).expect("init_pool should succeed"); + let conn = pool.get().unwrap(); + + let table_exists: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'table' AND name = 'ai_skill_events'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(table_exists, 1); + + let indexes: Vec = { + let mut stmt = conn + .prepare( + "SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'ai_skill_events' ORDER BY name", + ) + .unwrap(); + stmt.query_map([], |row| row.get::<_, String>(0)) + .unwrap() + .collect::>>() + .unwrap() + }; + assert!(indexes.contains(&"idx_ai_skill_events_timestamp".to_string())); + assert!(indexes.contains(&"idx_ai_skill_events_skill_time".to_string())); + assert!(indexes.contains(&"idx_ai_skill_events_plugin_time".to_string())); + assert!(indexes.contains(&"idx_ai_skill_events_hostname_time".to_string())); + assert!(indexes.contains(&"idx_ai_skill_events_session_time".to_string())); + assert!(indexes.contains(&"idx_ai_skill_events_project_skill_time".to_string())); + + // Eng review Fix 5: idx_logs_ai_tool_id lives on the EXISTING `logs` + // table (backfill keyset-pagination support), not `ai_skill_events`. + let logs_index_exists: i64 = conn + .query_row( + "SELECT COUNT(*) FROM sqlite_master WHERE type = 'index' AND name = 'idx_logs_ai_tool_id'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(logs_index_exists, 1); + + // UNIQUE constraint + idempotent re-run of the whole insert on identical + // (log_id, skill_name, event_kind, evidence_kind) is exercised in Task 6; + // here we only assert the migration ran and the schema is fully caught up + // (later migrations, e.g. 39/40, run in the same init_db pass). + let version = crate::read_schema_version_info_conn(&conn).unwrap().version; + assert_eq!(version, KNOWN_SCHEMA_VERSION); +} + +#[test] +fn graph_schema_accepts_entity_resolution_vocabulary() { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test( + dir.path().join("resolver-vocab.db"), + )) + .unwrap(); + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, source_kind, source_id, trust_level) + VALUES + ('logical_service', 'plex', 'plex', 'resolver', 'fixture', 'verified'), + ('service_instance', 'nashost/plex', 'nashost/plex', 'resolver', 'fixture', 'verified')", + [], + ) + .unwrap(); + let service = conn + .query_row( + "SELECT id FROM graph_entities WHERE entity_type = 'logical_service'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(); + let instance = conn + .query_row( + "SELECT id FROM graph_entities WHERE entity_type = 'service_instance'", + [], + |row| row.get::<_, i64>(0), + ) + .unwrap(); + conn.execute( + "INSERT INTO graph_relationships + (relationship_key, src_entity_id, dst_entity_id, relationship_type, + reason_code, trust_level, confidence) + VALUES (?1, ?2, ?3, 'instance_of', 'resolver_instance_of', 'verified', 1.0)", + rusqlite::params![ + format!("{instance}:instance_of:{service}"), + instance, + service + ], + ) + .unwrap(); +} + +#[test] +fn migration_41_cleans_legacy_service_rows_from_populated_db() { + // Simulate a populated pre-41 DB: run all migrations, then re-insert the + // old-shaped rows a v40 DB could contain and re-run the 41 cleanup SQL by + // reverting the migration marker before a second init_pool pass. + // + // NOTE: this replay runs on a post-41 schema (the CHECK constraints + // already include the v41 vocabulary), not a byte-faithful v40 schema. + // The migration's INSERT…SELECT filters are what is under test. + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("migration-41-cutover.db"); + { + let pool = init_pool(&StorageConfig::for_test(db_path.clone())).unwrap(); + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, source_kind, source_id, trust_level) + VALUES + ('service', 'nashost:plex', 'plex', 'log', 'fixture', 'inferred'), + ('service', 'nashost:plex:plex', 'nashost/plex/plex', 'log', 'fixture', 'inferred'), + ('app', 'plex/plex/plex', 'plex/plex/plex', 'log', 'fixture', 'claimed'), + ('app', 'kernel', 'kernel', 'log', 'fixture', 'claimed')", + [], + ) + .unwrap(); + conn.execute("DELETE FROM schema_migrations WHERE version = 41", []) + .unwrap(); + // Drop the 41-added column so the ALTER TABLE in the replayed + // migration does not collide. + conn.execute_batch("ALTER TABLE graph_projection_meta DROP COLUMN projection_contract;") + .unwrap(); + } + let pool = init_pool(&StorageConfig::for_test(db_path)).unwrap(); + let conn = pool.get().unwrap(); + let stale: i64 = conn + .query_row( + "SELECT COUNT(*) FROM graph_entities + WHERE entity_type = 'service' + OR (entity_type = 'app' AND canonical_key LIKE '%/%/%')", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(stale, 0); + // Plain app labels survive the cutover; only nested defect shapes go. + let plain_app: i64 = conn + .query_row( + "SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'app' AND canonical_key = 'kernel'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(plain_app, 1); + let contract: String = conn + .query_row( + "SELECT projection_contract FROM graph_projection_meta WHERE id = 1", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + contract, + crate::entity_resolution::vocab::GRAPH_PROJECTION_CONTRACT_V2 + ); +} + +#[test] +fn migration_41_prunes_relationships_evidence_and_aliases_touching_legacy_entities() { + // Same replay technique as the cleanup test above (post-41 schema, see + // its NOTE): seed a legacy `service` entity wired to a surviving host + // via a relationship with evidence plus an alias, and an unrelated + // surviving app→host relationship with evidence. Migration 41 must + // prune everything touching the legacy entity and nothing else, leaving + // no evidence row pointing at a dead relationship id, and flip a ready + // projection to stale. + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("migration-41-prune.db"); + { + let pool = init_pool(&StorageConfig::for_test(db_path.clone())).unwrap(); + let conn = pool.get().unwrap(); + let insert_entity = |entity_type: &str, key: &str| -> i64 { + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, source_kind, + source_id, trust_level) + VALUES (?1, ?2, ?2, 'log', 'fixture', 'inferred')", + rusqlite::params![entity_type, key], + ) + .unwrap(); + conn.last_insert_rowid() + }; + let insert_rel = |key: &str, src: i64, dst: i64| -> i64 { + conn.execute( + "INSERT INTO graph_relationships + (relationship_key, src_entity_id, dst_entity_id, + relationship_type, reason_code, trust_level, confidence, + evidence_count) + VALUES (?1, ?2, ?3, 'runs_on', 'log_app_name', 'inferred', + 0.5, 1)", + rusqlite::params![key, src, dst], + ) + .unwrap(); + conn.last_insert_rowid() + }; + let insert_evidence = |rel_id: i64, evidence_key: &str| { + conn.execute( + "INSERT INTO graph_relationship_evidence + (relationship_id, evidence_key, source_kind, source_id, + observed_at, reason_code, trust_level, evidence_count) + VALUES (?1, ?2, 'log', 'fixture', '2026-01-01T00:00:00Z', + 'log_app_name', 'inferred', 1)", + rusqlite::params![rel_id, evidence_key], + ) + .unwrap(); + }; + + let legacy = insert_entity("service", "nashost:plex"); + let host = insert_entity("host", "nashost"); + let app = insert_entity("app", "kernel"); + let legacy_rel = insert_rel("legacy:runs_on:host", legacy, host); + insert_evidence(legacy_rel, "legacy-evidence"); + conn.execute( + "INSERT INTO graph_entity_aliases + (entity_id, alias_type, alias_key, alias_value, trust_level) + VALUES (?1, 'service_name', 'plex-legacy', 'plex-legacy', + 'inferred')", + [legacy], + ) + .unwrap(); + let surviving_rel = insert_rel("app:runs_on:host", app, host); + insert_evidence(surviving_rel, "surviving-evidence"); + + conn.execute( + "UPDATE graph_projection_meta SET projection_status = 'ready' WHERE id = 1", + [], + ) + .unwrap(); + + conn.execute("DELETE FROM schema_migrations WHERE version = 41", []) + .unwrap(); + conn.execute_batch("ALTER TABLE graph_projection_meta DROP COLUMN projection_contract;") + .unwrap(); + } + let pool = init_pool(&StorageConfig::for_test(db_path)).unwrap(); + let conn = pool.get().unwrap(); + let count = |sql: &str| -> i64 { conn.query_row(sql, [], |row| row.get(0)).unwrap() }; + + // Legacy entity and everything touching it are gone. + assert_eq!( + count("SELECT COUNT(*) FROM graph_entities WHERE entity_type = 'service'"), + 0 + ); + assert_eq!( + count( + "SELECT COUNT(*) FROM graph_relationships + WHERE relationship_key = 'legacy:runs_on:host'" + ), + 0 + ); + assert_eq!( + count( + "SELECT COUNT(*) FROM graph_relationship_evidence + WHERE evidence_key = 'legacy-evidence'" + ), + 0 + ); + assert_eq!( + count("SELECT COUNT(*) FROM graph_entity_aliases WHERE alias_key = 'plex-legacy'"), + 0 + ); + + // The unrelated app→host relationship and its evidence survive. + assert_eq!( + count( + "SELECT COUNT(*) FROM graph_relationships + WHERE relationship_key = 'app:runs_on:host'" + ), + 1 + ); + assert_eq!( + count( + "SELECT COUNT(*) FROM graph_relationship_evidence + WHERE evidence_key = 'surviving-evidence'" + ), + 1 + ); + + // Referential integrity: no evidence row references a dead relationship. + assert_eq!( + count( + "SELECT COUNT(*) FROM graph_relationship_evidence e + WHERE e.relationship_id NOT IN (SELECT id FROM graph_relationships)" + ), + 0 + ); + + // A previously-ready projection is flipped to stale by the migration. + let status: String = conn + .query_row( + "SELECT projection_status FROM graph_projection_meta WHERE id = 1", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(status, "stale"); +} + +#[test] +fn migration_42_allows_refuted_alias_trust_level() { + // Migrations 35/41 added 'refuted' to graph_entities, graph_relationships, + // and graph_relationship_evidence but missed graph_entity_aliases. + // Migration 42 widens that CHECK too; assert a fresh DB accepts an alias + // write at 'refuted' trust without violating the constraint. + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("migration-42-refuted-alias.db")); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, source_kind, source_id, trust_level) + VALUES ('host', 'refuted-alias-host', 'refuted-alias-host', 'log', 'fixture', 'verified')", + [], + ) + .unwrap(); + let entity_id = conn.last_insert_rowid(); + + conn.execute( + "INSERT INTO graph_entity_aliases + (entity_id, alias_type, alias_key, alias_value, source_kind, trust_level) + VALUES (?1, 'hostname', 'refuted-alias-host', 'refuted-alias-host', 'log', 'refuted')", + rusqlite::params![entity_id], + ) + .unwrap(); + + let stored_trust: String = conn + .query_row( + "SELECT trust_level FROM graph_entity_aliases WHERE entity_id = ?1", + rusqlite::params![entity_id], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(stored_trust, "refuted"); +} + +#[test] +fn migration_42_widens_old_aliases_constraint_and_preserves_rows() { + // Simulate a populated pre-42 DB: run all migrations, seed an alias row + // at a pre-refuted trust level, revert the migration 42 marker, then + // re-run init_pool. The rebuilt table must preserve the existing row and + // accept a subsequent 'refuted' write. + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("migration-42-widen.db"); + let entity_id; + { + let pool = init_pool(&StorageConfig::for_test(db_path.clone())).unwrap(); + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, source_kind, source_id, trust_level) + VALUES ('host', 'pre42-host', 'pre42-host', 'log', 'fixture', 'verified')", + [], + ) + .unwrap(); + entity_id = conn.last_insert_rowid(); + conn.execute( + "INSERT INTO graph_entity_aliases + (entity_id, alias_type, alias_key, alias_value, source_kind, trust_level) + VALUES (?1, 'hostname', 'pre42-host', 'pre42-host', 'log', 'claimed')", + rusqlite::params![entity_id], + ) + .unwrap(); + conn.execute("DELETE FROM schema_migrations WHERE version = 42", []) + .unwrap(); + } + + let pool = init_pool(&StorageConfig::for_test(db_path)).unwrap(); + let conn = pool.get().unwrap(); + + let preserved: String = conn + .query_row( + "SELECT trust_level FROM graph_entity_aliases WHERE entity_id = ?1", + rusqlite::params![entity_id], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(preserved, "claimed"); + + conn.execute( + "INSERT INTO graph_entity_aliases + (entity_id, alias_type, alias_key, alias_value, source_kind, trust_level) + VALUES (?1, 'service_name', 'pre42-host-refuted', 'pre42-host-refuted', 'log', 'refuted')", + rusqlite::params![entity_id], + ) + .unwrap(); +} + +#[test] +fn init_pool_creates_agent_observatory_run_events_schema() { + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("observatory-events.db")); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + // RED: table does not exist yet + let columns: Vec = conn + .prepare("PRAGMA table_info(agent_run_events)") + .unwrap() + .query_map([], |row| row.get(1)) + .unwrap() + .collect::>() + .unwrap(); + + assert_eq!( + columns, + vec![ + "id", + "event_key", + "run_id", + "actor_id", + "worktree_id", + "commit_id", + "observed_at", + "ingested_at", + "event_kind", + "source_kind", + "source_id", + "source_log_id", + "provider_sequence", + "trace_id", + "span_id", + "severity", + "title", + "summary", + "payload_json", + "content_scrubbed", + "created_at", + ] + ); + + // Insert a test run first for foreign key constraint + conn.execute( + "INSERT INTO agent_runs + (run_key, native_session_id, tool, hostname, status, + status_observed_at, started_at, last_activity_at) + VALUES ('run-events-test', 'session-events', 'claude', 'devhost', + 'active', ?1, ?1, ?1)", + ["2026-08-01T02:40:00.000Z"], + ) + .unwrap(); + let run_id = conn.last_insert_rowid(); + + // Test unique event key constraint + conn.execute( + "INSERT INTO agent_run_events + (event_key, run_id, observed_at, ingested_at, event_kind, + source_kind, source_id, payload_json) + VALUES ('evt-1', ?1, ?2, ?2, 'lifecycle', 'test', 'src-1', '{}')", + rusqlite::params![run_id, "2026-08-01T02:40:01.000Z"], + ) + .unwrap(); + + assert!( + conn.execute( + "INSERT INTO agent_run_events + (event_key, run_id, observed_at, ingested_at, event_kind, + source_kind, source_id, payload_json) + VALUES ('evt-1', ?1, ?2, ?2, 'command', 'test', 'src-2', '{}')", + rusqlite::params![run_id, "2026-08-01T02:40:02.000Z"], + ) + .is_err(), + "duplicate event key must be rejected" + ); + + // Test invalid event kind rejection + assert!( + conn.execute( + "INSERT INTO agent_run_events + (event_key, run_id, observed_at, ingested_at, event_kind, + source_kind, source_id, payload_json) + VALUES ('evt-2', ?1, ?2, ?2, 'invalid_kind', 'test', 'src-3', '{}')", + rusqlite::params![run_id, "2026-08-01T02:40:03.000Z"], + ) + .is_err(), + "invalid event kind must be rejected" + ); + + // Test JSON validation on payload + assert!( + conn.execute( + "INSERT INTO agent_run_events + (event_key, run_id, observed_at, ingested_at, event_kind, + source_kind, source_id, payload_json) + VALUES ('evt-3', ?1, ?2, ?2, 'command', 'test', 'src-4', '{invalid')", + rusqlite::params![run_id, "2026-08-01T02:40:04.000Z"], + ) + .is_err(), + "invalid payload JSON must be rejected" + ); + + // Test 1000-event fixture for query plan and ordering + let mut events = Vec::new(); + for i in 0..1000 { + let event_key = format!("evt-batch-{}", i); + events.push((event_key, run_id)); + } + + for (event_key, run_id) in &events { + conn.execute( + "INSERT INTO agent_run_events + (event_key, run_id, observed_at, ingested_at, event_kind, + source_kind, source_id, payload_json) + VALUES (?1, ?2, datetime('now'), datetime('now'), 'command', 'test', ?3, '{}')", + rusqlite::params![event_key, run_id, format!("src-{}", event_key)], + ) + .unwrap(); + } + + // Verify query uses index and returns stable ordering + let query_plan: Vec = conn + .prepare( + "EXPLAIN QUERY PLAN + SELECT id, observed_at FROM agent_run_events + WHERE run_id = ?1 + ORDER BY observed_at DESC, id DESC + LIMIT 10", + ) + .unwrap() + .query_map(rusqlite::params![run_id], |row| row.get(3)) + .unwrap() + .collect::>() + .unwrap(); + + assert!( + query_plan + .iter() + .any(|detail| detail.contains("idx_agent_run_events_run_order")), + "query plan should use idx_agent_run_events_run_order index" + ); + + // Verify stable ordering + let mut prev_observed_at: Option = None; + let mut prev_id: Option = None; + + let results: Vec<(i64, String)> = conn + .prepare( + "SELECT id, observed_at FROM agent_run_events + WHERE run_id = ?1 + ORDER BY observed_at DESC, id DESC + LIMIT 100", + ) + .unwrap() + .query_map(rusqlite::params![run_id], |row| { + Ok((row.get(0)?, row.get(1)?)) + }) + .unwrap() + .collect::>() + .unwrap(); + + for (id, observed_at) in results { + if let (Some(prev_id), Some(prev_observed)) = (prev_id, prev_observed_at) { + assert!( + observed_at <= prev_observed || (observed_at == prev_observed && id < prev_id), + "results should be ordered by observed_at DESC, id DESC" + ); + } + prev_observed_at = Some(observed_at); + prev_id = Some(id); + } + + // Verify indexes exist (excluding autoindexes created by UNIQUE constraints) + let indexes: Vec = conn + .prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'agent_run_events' AND name NOT LIKE 'sqlite_autoindex_%' ORDER BY name") + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + + assert_eq!( + indexes, + vec![ + "idx_agent_run_events_run_kind", + "idx_agent_run_events_run_order", + "idx_agent_run_events_source_log", + "idx_agent_run_events_trace", + ] + ); +} + +#[test] +fn init_pool_creates_agent_stream_outbox() { + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("observatory-outbox.db")); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + // RED: table does not exist yet + let columns: Vec = conn + .prepare("PRAGMA table_info(agent_stream_outbox)") + .unwrap() + .query_map([], |row| row.get(1)) + .unwrap() + .collect::>() + .unwrap(); + + assert_eq!( + columns, + vec![ + "id", + "outbox_key", + "run_id", + "stream_event_type", + "expires_at", + "payload_json", + "created_at", + ] + ); + + // Insert test data for foreign key constraint + conn.execute( + "INSERT INTO agent_runs + (run_key, native_session_id, tool, hostname, status, + status_observed_at, started_at, last_activity_at) + VALUES ('run-outbox-test', 'session-outbox', 'claude', 'devhost', + 'active', ?1, ?1, ?1)", + ["2026-08-01T03:00:00.000Z"], + ) + .unwrap(); + let run_id = conn.last_insert_rowid(); + + // Test unique outbox_key constraint + conn.execute( + "INSERT INTO agent_stream_outbox + (outbox_key, run_id, stream_event_type, expires_at, payload_json) + VALUES ('outbox-1', ?1, 'lifecycle', ?2, '{}')", + rusqlite::params![run_id, "2026-08-01T03:01:00.000Z"], + ) + .unwrap(); + + assert!( + conn.execute( + "INSERT INTO agent_stream_outbox + (outbox_key, run_id, stream_event_type, expires_at, payload_json) + VALUES ('outbox-1', ?1, 'command', ?2, '{}')", + rusqlite::params![run_id, "2026-08-01T03:02:00.000Z"], + ) + .is_err(), + "duplicate outbox key must be rejected" + ); + + // Test JSON validation on payload + assert!( + conn.execute( + "INSERT INTO agent_stream_outbox + (outbox_key, run_id, stream_event_type, expires_at, payload_json) + VALUES ('outbox-2', ?1, 'command', ?2, '{invalid')", + rusqlite::params![run_id, "2026-08-01T03:03:00.000Z"], + ) + .is_err(), + "invalid payload JSON must be rejected" + ); + + // Test cascade delete: when run is deleted, outbox rows are removed + conn.execute( + "INSERT INTO agent_stream_outbox + (outbox_key, run_id, stream_event_type, expires_at, payload_json) + VALUES ('outbox-3', ?1, 'skill', ?2, '{}')", + rusqlite::params![run_id, "2026-08-01T03:04:00.000Z"], + ) + .unwrap(); + + let outbox_count_before: i64 = conn + .query_row( + "SELECT COUNT(*) FROM agent_stream_outbox WHERE run_id = ?1", + rusqlite::params![run_id], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + outbox_count_before, 2, + "should have 2 outbox rows (one duplicate failed)" + ); + + // Delete the run + conn.execute( + "DELETE FROM agent_runs WHERE id = ?1", + rusqlite::params![run_id], + ) + .unwrap(); + + let outbox_count_after: i64 = conn + .query_row( + "SELECT COUNT(*) FROM agent_stream_outbox WHERE run_id = ?1", + rusqlite::params![run_id], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + outbox_count_after, 0, + "all outbox rows should be cascade deleted" + ); + + // Verify query uses index and returns ascending order + conn.execute( + "INSERT INTO agent_runs + (run_key, native_session_id, tool, hostname, status, + status_observed_at, started_at, last_activity_at) + VALUES ('run-outbox-query', 'session-outbox-query', 'claude', 'devhost', + 'active', ?1, ?1, ?1)", + ["2026-08-01T03:05:00.000Z"], + ) + .unwrap(); + let run_id = conn.last_insert_rowid(); + + // Insert 100 outbox events + for i in 0..100 { + let outbox_key = format!("outbox-query-{}", i); + conn.execute( + "INSERT INTO agent_stream_outbox + (outbox_key, run_id, stream_event_type, expires_at, payload_json) + VALUES (?1, ?2, 'command', datetime('now', '+1 hour'), '{}')", + rusqlite::params![outbox_key, run_id], + ) + .unwrap(); + } + + let query_plan: Vec = conn + .prepare( + "EXPLAIN QUERY PLAN + SELECT id FROM agent_stream_outbox + WHERE run_id = ?1 + ORDER BY id ASC + LIMIT 10", + ) + .unwrap() + .query_map(rusqlite::params![run_id], |row| row.get(3)) + .unwrap() + .collect::>() + .unwrap(); + + assert!( + query_plan + .iter() + .any(|detail| detail.contains("idx_agent_stream_outbox_run")), + "query plan should use idx_agent_stream_outbox_run index" + ); + + // Verify indexes exist + let indexes: Vec = conn + .prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'agent_stream_outbox' AND name NOT LIKE 'sqlite_autoindex_%' ORDER BY name") + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + + assert_eq!( + indexes, + vec![ + "idx_agent_stream_outbox_expiry", + "idx_agent_stream_outbox_run", + ] + ); +} + +#[test] +fn init_pool_creates_agent_run_commits_and_projection_cursors() { + let dir = tempfile::tempdir().unwrap(); + let config = test_storage_config(dir.path().join("observatory-commits.db")); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + // RED: tables do not exist yet + let commit_columns: Vec = conn + .prepare("PRAGMA table_info(agent_run_commits)") + .unwrap() + .query_map([], |row| row.get(1)) + .unwrap() + .collect::>() + .unwrap(); + + assert_eq!( + commit_columns, + vec![ + "id", + "relation_key", + "run_id", + "commit_id", + "worktree_id", + "evidence_kind", + "evidence_source", + "trust_level", + "confidence", + "first_seen_at", + "last_seen_at", + "metadata_json", + ] + ); + + let cursor_columns: Vec = conn + .prepare("PRAGMA table_info(agent_projection_cursors)") + .unwrap() + .query_map([], |row| row.get(1)) + .unwrap() + .collect::>() + .unwrap(); + + assert_eq!( + cursor_columns, + vec![ + "id", + "cursor_type", + "source_name", + "cursor_value", + "updated_at", + ] + ); + + // Insert test data for foreign key constraints + conn.execute( + "INSERT INTO repositories (repository_key, hostname, common_git_dir, primary_path, display_name, first_seen_at, last_seen_at) + VALUES ('repo-test', 'devhost', '/tmp/repo', '/tmp/repo', 'Test Repo', ?1, ?2)", + ["2026-08-01T02:50:00.000Z", "2026-08-01T02:50:00.000Z"], + ) + .unwrap(); + let repo_id = conn.last_insert_rowid(); + + conn.execute( + "INSERT INTO repository_worktrees + (worktree_key, repository_id, hostname, path, git_dir, first_seen_at, last_seen_at) + VALUES ('wt-main', ?1, 'devhost', '/tmp/repo', '/tmp/repo/.git', ?2, ?2)", + rusqlite::params![repo_id, "2026-08-01T02:50:01.000Z"], + ) + .unwrap(); + let worktree_id = conn.last_insert_rowid(); + + conn.execute( + "INSERT INTO git_commits + (repository_id, sha, parent_shas_json, subject, author_name, + first_observed_at, last_observed_at) + VALUES (?1, 'abc123', '[]', 'Test commit', 'Test Author', ?2, ?2)", + rusqlite::params![repo_id, "2026-08-01T02:50:02.000Z"], + ) + .unwrap(); + let commit_id = conn.last_insert_rowid(); + + conn.execute( + "INSERT INTO agent_runs + (run_key, native_session_id, tool, hostname, status, + status_observed_at, started_at, last_activity_at) + VALUES ('run-commits-test', 'session-commits', 'claude', 'devhost', + 'active', ?1, ?1, ?1)", + ["2026-08-01T02:50:03.000Z"], + ) + .unwrap(); + let run_id = conn.last_insert_rowid(); + + // Test unique relation_key constraint + conn.execute( + "INSERT INTO agent_run_commits + (relation_key, run_id, commit_id, worktree_id, evidence_kind, + evidence_source, trust_level, confidence, first_seen_at, last_seen_at) + VALUES ('rel-1', ?1, ?2, ?3, 'git_head', 'git', 'verified', 0.95, ?4, ?4)", + rusqlite::params![run_id, commit_id, worktree_id, "2026-08-01T02:50:04.000Z"], + ) + .unwrap(); + + assert!( + conn.execute( + "INSERT INTO agent_run_commits + (relation_key, run_id, commit_id, worktree_id, evidence_kind, + evidence_source, trust_level, confidence, first_seen_at, last_seen_at) + VALUES ('rel-1', ?1, ?2, ?3, 'git_status', 'git', 'claimed', 0.5, ?4, ?4)", + rusqlite::params![run_id, commit_id, worktree_id, "2026-08-01T02:50:05.000Z"], + ) + .is_err(), + "duplicate relation key must be rejected" + ); + + // Test trust level constraint + assert!( + conn.execute( + "INSERT INTO agent_run_commits + (relation_key, run_id, commit_id, worktree_id, evidence_kind, + evidence_source, trust_level, confidence, first_seen_at, last_seen_at) + VALUES ('rel-2', ?1, ?2, ?3, 'git_head', 'git', 'invalid_trust', 0.9, ?4, ?4)", + rusqlite::params![run_id, commit_id, worktree_id, "2026-08-01T02:50:06.000Z"], + ) + .is_err(), + "invalid trust level must be rejected" + ); + + // Test confidence range constraint + assert!( + conn.execute( + "INSERT INTO agent_run_commits + (relation_key, run_id, commit_id, worktree_id, evidence_kind, + evidence_source, trust_level, confidence, first_seen_at, last_seen_at) + VALUES ('rel-3', ?1, ?2, ?3, 'git_head', 'git', 'verified', 1.5, ?4, ?4)", + rusqlite::params![run_id, commit_id, worktree_id, "2026-08-01T02:50:07.000Z"], + ) + .is_err(), + "confidence > 1.0 must be rejected" + ); + + // Verify seeded projection cursors (exactly 8 rows) + let cursor_count: i64 = conn + .query_row("SELECT COUNT(*) FROM agent_projection_cursors", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!( + cursor_count, 8, + "should have exactly 8 seeded projection cursors" + ); + + // Verify cursor types are correct + let cursor_types: Vec = conn + .prepare("SELECT DISTINCT cursor_type FROM agent_projection_cursors ORDER BY cursor_type") + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + + assert_eq!( + cursor_types, + vec![ + "agent_run_events", + "agent_runs", + "git_commits", + "otel_metric_points", + "otel_spans", + "repositories", + "repository_observations", + "repository_worktrees", + ] + ); + + // Verify INSERT OR IGNORE preserves existing cursors on repeated open + drop(conn); + drop(pool); + + let pool2 = init_pool(&config).unwrap(); + let conn2 = pool2.get().unwrap(); + + let cursor_count2: i64 = conn2 + .query_row("SELECT COUNT(*) FROM agent_projection_cursors", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!( + cursor_count2, 8, + "should still have exactly 8 cursors after reopen" + ); + + // Verify we can advance a cursor + conn2 + .execute( + "UPDATE agent_projection_cursors + SET cursor_value = 'advanced-123', updated_at = '2026-08-01T03:00:00.000Z' + WHERE cursor_type = 'agent_runs' AND source_name = 'default'", + [], + ) + .unwrap(); + + // Reopen again and verify the advanced cursor is preserved + drop(conn2); + drop(pool2); + + let pool3 = init_pool(&config).unwrap(); + let conn3 = pool3.get().unwrap(); + + let advanced_value: String = conn3 + .query_row( + "SELECT cursor_value FROM agent_projection_cursors + WHERE cursor_type = 'agent_runs' AND source_name = 'default'", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + advanced_value, "advanced-123", + "advanced cursor should be preserved" + ); + + // Verify indexes exist + let commit_indexes: Vec = conn3 + .prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'agent_run_commits' AND name NOT LIKE 'sqlite_autoindex_%' ORDER BY name") + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + + assert_eq!( + commit_indexes, + vec!["idx_agent_run_commits_commit", "idx_agent_run_commits_run",] + ); + + let cursor_indexes: Vec = conn3 + .prepare("SELECT name FROM sqlite_master WHERE type = 'index' AND tbl_name = 'agent_projection_cursors' AND name NOT LIKE 'sqlite_autoindex_%' ORDER BY name") + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + + assert_eq!(cursor_indexes, vec!["idx_agent_projection_cursors_type"]); +} + +#[test] +fn migration_45_completes_transactionally_and_is_idempotent() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("migration-45.db"); + + // Manually create a schema-44 database by stopping before migration 45 + let conn = rusqlite::Connection::open(&db_path).unwrap(); + + // Apply the base schema + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + hostname TEXT NOT NULL, + facility TEXT, + severity TEXT NOT NULL, + app_name TEXT, + process_id TEXT, + message TEXT NOT NULL, + raw TEXT NOT NULL, + received_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + source_ip TEXT NOT NULL DEFAULT '', + ai_tool TEXT, + ai_project TEXT, + ai_session_id TEXT, + ai_transcript_path TEXT, + metadata_json TEXT + ); + CREATE INDEX IF NOT EXISTS idx_logs_timestamp ON logs(timestamp); + CREATE INDEX IF NOT EXISTS idx_logs_hostname ON logs(hostname); + CREATE INDEX IF NOT EXISTS idx_logs_severity ON logs(severity); + CREATE INDEX IF NOT EXISTS idx_logs_app_name ON logs(app_name); + CREATE INDEX IF NOT EXISTS idx_logs_host_time ON logs(hostname, timestamp); + CREATE INDEX IF NOT EXISTS idx_logs_sev_time ON logs(severity, timestamp); + CREATE INDEX IF NOT EXISTS idx_logs_app_name_timestamp ON logs(app_name, timestamp); + CREATE INDEX IF NOT EXISTS idx_logs_received_at ON logs(received_at); + CREATE INDEX IF NOT EXISTS idx_logs_hostname_received_at ON logs(hostname, received_at); + CREATE INDEX IF NOT EXISTS idx_logs_source_ip_timestamp ON logs(source_ip, timestamp); + + CREATE VIRTUAL TABLE IF NOT EXISTS logs_fts USING fts5( + message, + content='logs', + content_rowid='id', + tokenize='porter unicode61' + ); + + CREATE TRIGGER IF NOT EXISTS logs_ai AFTER INSERT ON logs BEGIN + INSERT INTO logs_fts(rowid, message) VALUES (new.id, new.message); + END; + + CREATE TABLE IF NOT EXISTS hosts ( + hostname TEXT PRIMARY KEY, + first_seen TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + last_seen TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + log_count INTEGER NOT NULL DEFAULT 0 + ); + + CREATE TABLE IF NOT EXISTS schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) + );", + ) + .unwrap(); + + // Manually insert migration 44 marker (simulating migration 44 was applied) + conn.execute("INSERT INTO schema_migrations (version) VALUES (44)", []) + .unwrap(); + + // Apply migration 44 tables manually + conn.execute_batch( + "CREATE TABLE IF NOT EXISTS repositories ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + repository_key TEXT NOT NULL UNIQUE, + hostname TEXT NOT NULL, + common_git_dir TEXT NOT NULL, + primary_path TEXT NOT NULL, + display_name TEXT NOT NULL, + remote_url_hash TEXT, + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + removed_at TEXT, + metadata_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(metadata_json)), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(hostname, common_git_dir) + ); + CREATE INDEX IF NOT EXISTS idx_repositories_host_seen + ON repositories(hostname, last_seen_at DESC); + CREATE INDEX IF NOT EXISTS idx_repositories_display + ON repositories(display_name COLLATE NOCASE); + + CREATE TABLE IF NOT EXISTS repository_worktrees ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + worktree_key TEXT NOT NULL UNIQUE, + repository_id INTEGER NOT NULL REFERENCES repositories(id) ON DELETE CASCADE, + hostname TEXT NOT NULL, + path TEXT NOT NULL, + git_dir TEXT NOT NULL, + branch_ref TEXT, + branch_name TEXT, + head_sha TEXT, + upstream_ref TEXT, + detached INTEGER NOT NULL DEFAULT 0 CHECK (detached IN (0, 1)), + bare INTEGER NOT NULL DEFAULT 0 CHECK (bare IN (0, 1)), + locked INTEGER NOT NULL DEFAULT 0 CHECK (locked IN (0, 1)), + lock_reason TEXT, + prunable INTEGER NOT NULL DEFAULT 0 CHECK (prunable IN (0, 1)), + prune_reason TEXT, + dirty INTEGER NOT NULL DEFAULT 0 CHECK (dirty IN (0, 1)), + staged_count INTEGER NOT NULL DEFAULT 0 CHECK (staged_count >= 0), + unstaged_count INTEGER NOT NULL DEFAULT 0 CHECK (unstaged_count >= 0), + untracked_count INTEGER NOT NULL DEFAULT 0 CHECK (untracked_count >= 0), + ahead INTEGER CHECK (ahead IS NULL OR ahead >= 0), + behind INTEGER CHECK (behind IS NULL OR behind >= 0), + status_hash TEXT, + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + removed_at TEXT, + metadata_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(metadata_json)), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(repository_id, path), + UNIQUE(repository_id, hostname, git_dir) + ); + CREATE INDEX IF NOT EXISTS idx_repository_worktrees_repo + ON repository_worktrees(repository_id, last_seen_at DESC, id DESC); + CREATE INDEX IF NOT EXISTS idx_repository_worktrees_host + ON repository_worktrees(hostname, path); + + CREATE TABLE IF NOT EXISTS repository_observations ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + observation_key TEXT NOT NULL UNIQUE, + repository_id INTEGER NOT NULL REFERENCES repositories(id) ON DELETE CASCADE, + worktree_id INTEGER REFERENCES repository_worktrees(id) ON DELETE SET NULL, + observed_at TEXT NOT NULL, + observed_from TEXT NOT NULL, + payload_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(payload_json)), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')) + ); + CREATE INDEX IF NOT EXISTS idx_repository_observations_repo_time + ON repository_observations(repository_id, observed_at DESC, id DESC); + CREATE INDEX IF NOT EXISTS idx_repository_observations_worktree_time + ON repository_observations(worktree_id, observed_at DESC, id DESC); + + CREATE TABLE IF NOT EXISTS git_commits ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + repository_id INTEGER NOT NULL REFERENCES repositories(id) ON DELETE CASCADE, + sha TEXT NOT NULL, + parent_shas_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(parent_shas_json)), + author_name TEXT, + author_email_hash TEXT, + authored_at TEXT, + committed_at TEXT, + subject TEXT NOT NULL DEFAULT '', + changed_files INTEGER CHECK (changed_files IS NULL OR changed_files >= 0), + insertions INTEGER CHECK (insertions IS NULL OR insertions >= 0), + deletions INTEGER CHECK (deletions IS NULL OR deletions >= 0), + changed_paths_json TEXT NOT NULL DEFAULT '[]' CHECK (json_valid(changed_paths_json)), + first_observed_at TEXT NOT NULL, + last_observed_at TEXT NOT NULL, + reachable INTEGER NOT NULL DEFAULT 1 CHECK (reachable IN (0, 1)), + metadata_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(metadata_json)), + UNIQUE(repository_id, sha) + ); + CREATE INDEX IF NOT EXISTS idx_git_commits_repo_time + ON git_commits(repository_id, committed_at DESC, id DESC); + + CREATE TABLE IF NOT EXISTS agent_runs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + run_key TEXT NOT NULL UNIQUE, + native_session_id TEXT NOT NULL, + tool TEXT NOT NULL, + provider_tool TEXT, + hostname TEXT NOT NULL, + parent_run_id INTEGER REFERENCES agent_runs(id) ON DELETE SET NULL, + previous_run_id INTEGER REFERENCES agent_runs(id) ON DELETE SET NULL, + primary_worktree_id INTEGER REFERENCES repository_worktrees(id) ON DELETE SET NULL, + transcript_path TEXT, + process_id TEXT, + status TEXT NOT NULL CHECK (status IN ( + 'starting', 'active', 'waiting', 'idle', 'stale', + 'completed', 'failed', 'abandoned' + )), + status_reason TEXT NOT NULL DEFAULT '', + status_observed_at TEXT NOT NULL, + started_at TEXT NOT NULL, + last_activity_at TEXT NOT NULL, + ended_at TEXT, + first_source_log_id INTEGER, + last_source_log_id INTEGER, + last_event_id INTEGER, + event_count INTEGER NOT NULL DEFAULT 0 CHECK (event_count >= 0), + error_count INTEGER NOT NULL DEFAULT 0 CHECK (error_count >= 0), + primary_branch TEXT, + start_head_sha TEXT, + current_head_sha TEXT, + projection_version INTEGER NOT NULL DEFAULT 1, + freshness_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(freshness_json)), + metadata_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(metadata_json)), + created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + updated_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%fZ', 'now')), + UNIQUE(hostname, tool, native_session_id) + ); + CREATE INDEX IF NOT EXISTS idx_agent_runs_activity + ON agent_runs(last_activity_at DESC, id DESC); + CREATE INDEX IF NOT EXISTS idx_agent_runs_status_activity + ON agent_runs(status, last_activity_at DESC, id DESC); + CREATE INDEX IF NOT EXISTS idx_agent_runs_worktree_activity + ON agent_runs(primary_worktree_id, last_activity_at DESC, id DESC); + CREATE INDEX IF NOT EXISTS idx_agent_runs_tool_host + ON agent_runs(tool, hostname, last_activity_at DESC); + + CREATE TABLE IF NOT EXISTS agent_run_actors ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + actor_key TEXT NOT NULL UNIQUE, + run_id INTEGER NOT NULL REFERENCES agent_runs(id) ON DELETE CASCADE, + native_actor_id TEXT NOT NULL, + actor_type TEXT, + display_name TEXT, + started_at TEXT, + last_activity_at TEXT, + ended_at TEXT, + metadata_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(metadata_json)), + UNIQUE(run_id, native_actor_id) + ); + CREATE INDEX IF NOT EXISTS idx_agent_run_actors_run + ON agent_run_actors(run_id, last_activity_at DESC); + + CREATE TABLE IF NOT EXISTS agent_run_worktrees ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + relation_key TEXT NOT NULL UNIQUE, + run_id INTEGER NOT NULL REFERENCES agent_runs(id) ON DELETE CASCADE, + worktree_id INTEGER NOT NULL REFERENCES repository_worktrees(id) ON DELETE CASCADE, + evidence_kind TEXT NOT NULL, + evidence_source TEXT NOT NULL, + trust_level TEXT NOT NULL CHECK (trust_level IN ( + 'verified', 'claimed', 'correlated', 'inferred', 'refuted' + )), + confidence REAL NOT NULL CHECK (confidence >= 0.0 AND confidence <= 1.0), + is_primary INTEGER NOT NULL DEFAULT 0 CHECK (is_primary IN (0, 1)), + first_seen_at TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + metadata_json TEXT NOT NULL DEFAULT '{}' CHECK (json_valid(metadata_json)), + UNIQUE(run_id, worktree_id, evidence_kind, evidence_source) + ); + CREATE INDEX IF NOT EXISTS idx_agent_run_worktrees_run + ON agent_run_worktrees(run_id, is_primary DESC, confidence DESC, last_seen_at DESC); + CREATE INDEX IF NOT EXISTS idx_agent_run_worktrees_worktree + ON agent_run_worktrees(worktree_id, last_seen_at DESC, run_id);", + ) + .unwrap(); + + drop(conn); + + // Now reopen the database - migration 45 should apply transactionally + let pool_45 = init_pool(&StorageConfig::for_test(db_path.clone())).unwrap(); + let conn_45 = pool_45.get().unwrap(); + + // Verify we're now at schema 47 (45 + 46 + 47 are applied automatically) + let schema_version_final: i64 = conn_45 + .query_row("SELECT MAX(version) FROM schema_migrations", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!( + schema_version_final, 47, + "should upgrade from schema 44 to schema 47 (applying 45, 46, 47)" + ); + + // Verify all migration 45 tables now exist + let tables_45: Vec = conn_45 + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'agent_%' ORDER BY name") + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + + assert_eq!( + tables_45, + vec![ + "agent_projection_cursors", + "agent_run_actors", + "agent_run_commits", + "agent_run_events", + "agent_run_worktrees", + "agent_runs", + "agent_stream_outbox", + ], + "should have all migration 45 agent tables" + ); + + // Verify seeded cursors exist + let cursor_count: i64 = conn_45 + .query_row("SELECT COUNT(*) FROM agent_projection_cursors", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(cursor_count, 8, "should have 8 seeded cursors"); + + drop(conn_45); + drop(pool_45); + + // Verify idempotency: reopening should keep schema at 45 and not reapply migration + let pool_again = init_pool(&StorageConfig::for_test(db_path.clone())).unwrap(); + let conn_again = pool_again.get().unwrap(); + + let schema_version_again: i64 = conn_again + .query_row("SELECT MAX(version) FROM schema_migrations", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!( + schema_version_again, 47, + "should remain at schema 47 after migrations 45, 46, 47" + ); + + let cursor_count_again: i64 = conn_again + .query_row("SELECT COUNT(*) FROM agent_projection_cursors", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!( + cursor_count_again, 8, + "should still have 8 cursors (not duplicated)" + ); + + // Verify migration 45 marker exists only once + let migration_45_count: i64 = conn_again + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 45", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + migration_45_count, 1, + "should have exactly one migration 45 marker" + ); + + // Verify foreign key checks pass + let fk_check: String = conn_again + .query_row("PRAGMA foreign_key_check", [], |row| row.get(0)) + .unwrap_or("ok".to_string()); + assert_eq!(fk_check, "ok", "foreign key checks should pass"); + + // Verify integrity checks pass + let integrity_result: String = conn_again + .query_row("PRAGMA integrity_check", [], |row| row.get(0)) + .unwrap(); + assert_eq!(integrity_result, "ok", "integrity checks should pass"); +} + +#[test] +fn migration_45_fresh_database_applies_transactionally() { + let dir = tempfile::tempdir().unwrap(); + let config = StorageConfig::for_test(dir.path().join("migration-45-fresh.db")); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + // Fresh database should be at schema 45 + let schema_version: i64 = conn + .query_row("SELECT MAX(version) FROM schema_migrations", [], |row| { + row.get(0) + }) + .unwrap(); + assert_eq!(schema_version, 47, "fresh database should be at schema 47"); + + // Verify all migration 45 tables still exist (additive migrations preserve them) + let tables: Vec = conn + .prepare("SELECT name FROM sqlite_master WHERE type = 'table' AND name LIKE 'agent_%' ORDER BY name") + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + + assert_eq!( + tables, + vec![ + "agent_projection_cursors", + "agent_run_actors", + "agent_run_commits", + "agent_run_events", + "agent_run_worktrees", + "agent_runs", + "agent_stream_outbox", + ], + "migration 47 should preserve all migration 45 tables" + ); +} + +// AO-014: migration 46 OTLP span table contract. +#[test] +fn migration_46_creates_otel_spans_table_and_indexes() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("migration-46-otel-spans.db"); + let config = StorageConfig::for_test(db_path.clone()); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + let columns: Vec = conn + .prepare("PRAGMA table_info(otel_spans)") + .unwrap() + .query_map([], |row| row.get(1)) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!( + columns, + vec![ + "id", + "trace_id", + "span_id", + "parent_span_id", + "trace_state", + "flags", + "span_name", + "span_kind", + "start_time_unix_nano", + "end_time_unix_nano", + "duration_nano", + "status_code", + "status_message", + "hostname", + "service_name", + "service_version", + "scope_name", + "scope_version", + "ai_tool", + "ai_project", + "ai_session_id", + "run_id", + "resource_json", + "attributes_json", + "events_json", + "links_json", + "received_at", + "content_scrubbed", + ] + ); + + let indexes: Vec = conn + .prepare( + "SELECT name FROM sqlite_master + WHERE type = 'index' AND tbl_name = 'otel_spans' + ORDER BY name", + ) + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + for expected in [ + "idx_otel_spans_run_time", + "idx_otel_spans_service_time", + "idx_otel_spans_session_time", + "idx_otel_spans_trace", + ] { + assert!( + indexes.iter().any(|name| name == expected), + "missing {expected}: {indexes:?}" + ); + } + + conn.execute( + "INSERT INTO agent_runs + (run_key, native_session_id, tool, hostname, status, + status_observed_at, started_at, last_activity_at) + VALUES ('span-run', 'span-session', 'claude', 'devhost', + 'active', ?1, ?1, ?1)", + ["2026-08-01T03:00:00.000Z"], + ) + .unwrap(); + let run_id = conn.last_insert_rowid(); + + let insert_span = |span_id: &str, start: i64| { + conn.execute( + "INSERT INTO otel_spans + (trace_id, span_id, parent_span_id, span_name, span_kind, + start_time_unix_nano, end_time_unix_nano, duration_nano, + hostname, service_name, ai_tool, ai_session_id, run_id, + resource_json, attributes_json, events_json, links_json, + received_at, content_scrubbed) + VALUES (?1, ?2, ?3, ?4, 1, ?5, ?6, 100, + 'devhost', 'cortex', 'claude', 'span-session', ?7, + '{}', '{\"worktree\":\"cortex\"}', '[]', '[]', ?8, 1)", + rusqlite::params![ + "0123456789abcdef0123456789abcdef", + span_id, + "1111111111111111", + format!("span-{span_id}"), + start, + start + 100, + run_id, + "2026-08-01T03:00:00.000Z", + ], + ) + }; + insert_span("2222222222222222", 100).unwrap(); + insert_span("3333333333333333", 200).unwrap(); + + assert!( + insert_span("2222222222222222", 300).is_err(), + "trace/span identity must deduplicate" + ); + for (label, sql) in [ + ( + "trace length", + "INSERT INTO otel_spans + (trace_id, span_id, span_name, span_kind, start_time_unix_nano, + end_time_unix_nano, duration_nano, received_at) + VALUES ('short', '4444444444444444', 'bad-trace', 1, 1, 2, 1, 'now')", + ), + ( + "span length", + "INSERT INTO otel_spans + (trace_id, span_id, span_name, span_kind, start_time_unix_nano, + end_time_unix_nano, duration_nano, received_at) + VALUES ('aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa', 'short', 'bad-span', 1, 1, 2, 1, 'now')", + ), + ( + "parent length", + "INSERT INTO otel_spans + (trace_id, span_id, parent_span_id, span_name, span_kind, + start_time_unix_nano, end_time_unix_nano, duration_nano, received_at) + VALUES ('bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb', '5555555555555555', 'short', + 'bad-parent', 1, 1, 2, 1, 'now')", + ), + ( + "negative duration", + "INSERT INTO otel_spans + (trace_id, span_id, span_name, span_kind, start_time_unix_nano, + end_time_unix_nano, duration_nano, received_at) + VALUES ('cccccccccccccccccccccccccccccccc', '6666666666666666', + 'bad-duration', 1, 2, 1, -1, 'now')", + ), + ( + "invalid JSON", + "INSERT INTO otel_spans + (trace_id, span_id, span_name, span_kind, start_time_unix_nano, + end_time_unix_nano, duration_nano, resource_json, received_at) + VALUES ('dddddddddddddddddddddddddddddddd', '7777777777777777', + 'bad-json', 1, 1, 2, 1, '{', 'now')", + ), + ( + "scrub flag", + "INSERT INTO otel_spans + (trace_id, span_id, span_name, span_kind, start_time_unix_nano, + end_time_unix_nano, duration_nano, received_at, content_scrubbed) + VALUES ('eeeeeeeeeeeeeeeeeeeeeeeeeeeeeeee', '8888888888888888', + 'bad-scrub', 1, 1, 2, 1, 'now', 2)", + ), + ] { + assert!(conn.execute(sql, []).is_err(), "{label} must be rejected"); + } + + let ordered: Vec = conn + .prepare( + "SELECT span_id FROM otel_spans + WHERE run_id = ?1 + ORDER BY start_time_unix_nano DESC, id DESC", + ) + .unwrap() + .query_map([run_id], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!(ordered, vec!["3333333333333333", "2222222222222222"]); + + let run_plan: Vec = conn + .prepare( + "EXPLAIN QUERY PLAN + SELECT id FROM otel_spans + WHERE run_id = ?1 + ORDER BY start_time_unix_nano DESC, id DESC LIMIT 10", + ) + .unwrap() + .query_map([run_id], |row| row.get(3)) + .unwrap() + .collect::>() + .unwrap(); + assert!( + run_plan + .iter() + .any(|detail| detail.contains("idx_otel_spans_run_time")), + "run timeline query must use its index: {run_plan:?}" + ); + + let trace_plan: Vec = conn + .prepare( + "EXPLAIN QUERY PLAN + SELECT span_id FROM otel_spans + WHERE trace_id = ?1 + ORDER BY start_time_unix_nano, span_id", + ) + .unwrap() + .query_map(["0123456789abcdef0123456789abcdef"], |row| row.get(3)) + .unwrap() + .collect::>() + .unwrap(); + assert!( + trace_plan + .iter() + .any(|detail| detail.contains("idx_otel_spans_trace")), + "trace query must use its index: {trace_plan:?}" + ); + + let marker_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 46", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(marker_count, 1); + + conn.execute("DELETE FROM agent_runs WHERE id = ?1", [run_id]) + .unwrap(); + let null_run_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM otel_spans WHERE run_id IS NULL", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(null_run_count, 2, "run deletion must preserve spans"); + + let foreign_key_violation: Option = conn + .query_row("PRAGMA foreign_key_check", [], |row| row.get(0)) + .optional() + .unwrap(); + assert_eq!(foreign_key_violation, None); + let integrity: String = conn + .query_row("PRAGMA integrity_check", [], |row| row.get(0)) + .unwrap(); + assert_eq!(integrity, "ok"); + + drop(conn); + drop(pool); + let reopened = init_pool(&StorageConfig::for_test(db_path)).unwrap(); + let reopened_conn = reopened.get().unwrap(); + let marker_count: i64 = reopened_conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 46", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(marker_count, 1, "migration 46 must be idempotent"); +} + +// AO-015: migration 47 OTLP metric-point table contract. +#[test] +fn migration_47_creates_otel_metric_points_table_and_indexes() { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("migration-47-otel-metrics.db"); + let config = StorageConfig::for_test(db_path.clone()); + let pool = init_pool(&config).unwrap(); + let conn = pool.get().unwrap(); + + let columns: Vec = conn + .prepare("PRAGMA table_info(otel_metric_points)") + .unwrap() + .query_map([], |row| row.get(1)) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!( + columns, + vec![ + "id", + "point_key", + "metric_name", + "description", + "unit", + "instrument_kind", + "aggregation_temporality", + "monotonic", + "start_time_unix_nano", + "time_unix_nano", + "hostname", + "service_name", + "service_version", + "scope_name", + "scope_version", + "ai_tool", + "ai_project", + "ai_session_id", + "run_id", + "resource_json", + "attributes_json", + "value_json", + "exemplars_json", + "received_at", + "content_scrubbed", + ] + ); + + let indexes: Vec = conn + .prepare( + "SELECT name FROM sqlite_master + WHERE type = 'index' AND tbl_name = 'otel_metric_points' + ORDER BY name", + ) + .unwrap() + .query_map([], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + for expected in [ + "idx_otel_metric_points_name_time", + "idx_otel_metric_points_run_time", + "idx_otel_metric_points_session_time", + ] { + assert!( + indexes.iter().any(|name| name == expected), + "missing {expected}: {indexes:?}" + ); + } + + conn.execute( + "INSERT INTO agent_runs + (run_key, native_session_id, tool, hostname, status, + status_observed_at, started_at, last_activity_at) + VALUES ('metric-run', 'metric-session', 'codex', 'devhost', + 'active', ?1, ?1, ?1)", + ["2026-08-01T03:30:00.000Z"], + ) + .unwrap(); + let run_id = conn.last_insert_rowid(); + + let insert_point = |point_key: &str, metric_name: &str, kind: &str, time: i64| { + conn.execute( + "INSERT INTO otel_metric_points + (point_key, metric_name, description, unit, instrument_kind, + aggregation_temporality, monotonic, start_time_unix_nano, + time_unix_nano, hostname, service_name, ai_tool, + ai_session_id, run_id, resource_json, attributes_json, + value_json, exemplars_json, received_at, content_scrubbed) + VALUES (?1, ?2, 'fixture', 'ms', ?3, 2, 0, ?4, ?5, + 'devhost', 'cortex', 'codex', 'metric-session', ?6, + '{}', '{}', ?7, '[]', ?8, 1)", + rusqlite::params![ + point_key, + metric_name, + kind, + time - 10, + time, + run_id, + "{\"value\":42.0}", + "2026-08-01T03:30:00.000Z", + ], + ) + }; + insert_point("point-1", "agent.latency", "gauge", 100).unwrap(); + insert_point("point-2", "agent.latency", "histogram", 200).unwrap(); + + assert!( + insert_point("point-1", "agent.latency", "gauge", 300).is_err(), + "point_key must deduplicate" + ); + assert!( + insert_point("bad-kind", "agent.latency", "invalid_kind", 300).is_err(), + "unknown instrument kind must be rejected" + ); + + for (label, sql, params) in [ + ( + "resource JSON", + "INSERT INTO otel_metric_points + (point_key, metric_name, instrument_kind, time_unix_nano, + resource_json, value_json, received_at) + VALUES (?1, 'agent.bad', 'gauge', 300, ?2, '{}', 'now')", + ("bad-resource", "{"), + ), + ( + "value JSON", + "INSERT INTO otel_metric_points + (point_key, metric_name, instrument_kind, time_unix_nano, + value_json, received_at) + VALUES (?1, 'agent.bad', 'sum', 301, ?2, 'now')", + ("bad-value", "{"), + ), + ( + "exemplars JSON", + "INSERT INTO otel_metric_points + (point_key, metric_name, instrument_kind, time_unix_nano, + value_json, exemplars_json, received_at) + VALUES (?1, 'agent.bad', 'summary', 302, '{}', ?2, 'now')", + ("bad-exemplars", "{"), + ), + ] { + assert!( + conn.execute(sql, rusqlite::params![params.0, params.1]) + .is_err(), + "{label} must be rejected" + ); + } + + assert!( + conn.execute( + "INSERT INTO otel_metric_points + (point_key, metric_name, instrument_kind, monotonic, + time_unix_nano, value_json, received_at) + VALUES ('bad-monotonic', 'agent.bad', 'sum', 2, 303, '{}', 'now')", + [], + ) + .is_err(), + "monotonic must be null, zero, or one" + ); + assert!( + conn.execute( + "INSERT INTO otel_metric_points + (point_key, metric_name, instrument_kind, time_unix_nano, + value_json, received_at, content_scrubbed) + VALUES ('bad-scrub', 'agent.bad', 'gauge', 304, '{}', 'now', 2)", + [], + ) + .is_err(), + "content_scrubbed must be zero or one" + ); + + let ordered: Vec = conn + .prepare( + "SELECT point_key FROM otel_metric_points + WHERE run_id = ?1 + ORDER BY time_unix_nano DESC, id DESC", + ) + .unwrap() + .query_map([run_id], |row| row.get(0)) + .unwrap() + .collect::>() + .unwrap(); + assert_eq!(ordered, vec!["point-2", "point-1"]); + + let run_plan: Vec = conn + .prepare( + "EXPLAIN QUERY PLAN + SELECT id FROM otel_metric_points + WHERE run_id = ?1 + ORDER BY time_unix_nano DESC, id DESC LIMIT 10", + ) + .unwrap() + .query_map([run_id], |row| row.get(3)) + .unwrap() + .collect::>() + .unwrap(); + assert!( + run_plan + .iter() + .any(|detail| detail.contains("idx_otel_metric_points_run_time")), + "run metric query must use its index: {run_plan:?}" + ); + + let name_plan: Vec = conn + .prepare( + "EXPLAIN QUERY PLAN + SELECT id FROM otel_metric_points + WHERE metric_name = ?1 + ORDER BY time_unix_nano DESC, id DESC LIMIT 10", + ) + .unwrap() + .query_map(["agent.latency"], |row| row.get(3)) + .unwrap() + .collect::>() + .unwrap(); + assert!( + name_plan + .iter() + .any(|detail| detail.contains("idx_otel_metric_points_name_time")), + "metric-name query must use its index: {name_plan:?}" + ); + + let marker_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 47", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(marker_count, 1); + + conn.execute("DELETE FROM agent_runs WHERE id = ?1", [run_id]) + .unwrap(); + let null_run_count: i64 = conn + .query_row( + "SELECT COUNT(*) FROM otel_metric_points WHERE run_id IS NULL", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!( + null_run_count, 2, + "run deletion must preserve metric points" + ); + + let foreign_key_violation: Option = conn + .query_row("PRAGMA foreign_key_check", [], |row| row.get(0)) + .optional() + .unwrap(); + assert_eq!(foreign_key_violation, None); + let integrity: String = conn + .query_row("PRAGMA integrity_check", [], |row| row.get(0)) + .unwrap(); + assert_eq!(integrity, "ok"); + + drop(conn); + drop(pool); + let reopened = init_pool(&StorageConfig::for_test(db_path)).unwrap(); + let reopened_conn = reopened.get().unwrap(); + let marker_count: i64 = reopened_conn + .query_row( + "SELECT COUNT(*) FROM schema_migrations WHERE version = 47", + [], + |row| row.get(0), + ) + .unwrap(); + assert_eq!(marker_count, 1, "migration 47 must be idempotent"); +} diff --git a/crates/shared/cortex/storage-sqlite/src/queries.rs b/crates/shared/cortex/storage-sqlite/src/queries.rs new file mode 100644 index 00000000..d56cd1a7 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/queries.rs @@ -0,0 +1,3813 @@ +//! All read-path SQL for the log intelligence core lives here — every SELECT +//! behind the MCP actions, `/api/*` routes, and direct CLI queries. +//! +//! Key invariants: +//! - **All query SQL lives in this module** (deletes live in `maintenance.rs`, +//! schema in `pool.rs`). Handlers never build SQL strings. +//! - Every query uses parameterized bindings — no user input is interpolated. +//! - FTS5 searches JOIN `logs_fts` back to `logs`, which prunes phantom rows +//! left behind by retention/storage deletes at query time. +//! - The FTS fast path caps match-set materialization at the 200K most-recent +//! matches (`SEARCH_FTS_FAST_PATH_MATCH_CAP`); severity-only filtered +//! searches use the same capped candidate plan. +//! - Unbounded `sessions` reads are served from the `ai_session_rollup` +//! materialization; time-windowed reads run live against `logs`. + +use anyhow::Result; +use rusqlite::{OptionalExtension, params}; + +use crate::config::StorageConfig; +use cortex_ingest_core::SourceKind; + +use super::entity_resolution::{ + FALLBACK_EXPLICIT_DEGRADED_HOST_CONTEXT, INCLUSION_GRAPH_RELATED, INCLUSION_HOST_CONTEXT, + ResolverStatus, +}; +use super::maintenance::{exceeds_trigger, get_storage_metrics}; +use super::models::{ + AbuseIncident, AiAbuseMatch, AiAbuseParams, AiAbuseResult, AiCorrelateParams, AiIncidentParams, + AiIncidentResult, AiInvestigateParams, AiInvestigateResult, AiProjectInventoryEntry, + AiRelatedLogsForAnchor, AiRelatedLogsParams, AiSessionEntry, AiToolInventoryEntry, DbStats, + ErrorSummaryEntry, GraphRelatedLogEntry, IncidentEvidence, ListAiProjectsParams, + ListAiProjectsResult, ListAiSessionsParams, ListAiToolsParams, ListAiToolsResult, LogEntry, + ResolvedTopicEntity, SearchAiSessionsParams, SearchAiSessionsResult, SearchParams, + SearchedAiSessionEntry, SessionGraphInputs, TopicGraphInputs, +}; +use super::pool::DbPool; +use super::queries_service_instances; + +const SEARCH_FTS_CANDIDATE_CAP: usize = 10_000; +const SIMILAR_INCIDENT_FTS_CANDIDATE_CAP: usize = 5_000; +/// Cap on the FTS match-set materialization in the fast (index-led) search +/// path. The `id IN (SELECT rowid FROM logs_fts ...)` subquery is +/// non-correlated, so SQLite materializes the whole match set into an +/// ephemeral index before walking the filter's composite index — for a common +/// term on a multi-million-row DB that was unbounded memory and a full FTS +/// walk (full-review PH1). 200K most-recent matches is recency-biased and far +/// larger than any result LIMIT (search caps at 1000), so in practice results +/// are unaffected; matches older than the newest 200K are no longer +/// intersected. +const SEARCH_FTS_FAST_PATH_MATCH_CAP: usize = 200_000; + +fn push_bound_limit( + sql: &mut String, + bindings: &mut Vec, + idx: &mut usize, + keyword: &str, + limit: impl Into, +) { + let idx_value = *idx; + bindings.push(rusqlite::types::Value::Integer(limit.into())); + *idx += 1; + sql.push_str(&format!(" {keyword} ?{idx_value}")); +} + +/// Detect common FTS5 foot-guns and return a fix-it error. Runs before the +/// generic length/term-count checks in [`validate_fts_query`]. +/// +/// - A whitespace-separated term with a non-leading hyphen (e.g. `smoke-test`) +/// is parsed by FTS5 as `smoke NOT test`, which surprises users searching a +/// hyphenated word. The check is per-term: a leading-hyphen term (`-nginx`) +/// is an intentional NOT and is left alone, and a term that is part of a +/// quoted phrase (contains a `"`) is skipped — so `"disk full" smoke-test` +/// still flags the unquoted `smoke-test`. +/// - An odd number of double-quotes is an unterminated phrase. +fn lint_fts_query(query: &str) -> Result<()> { + let has_unquoted_hyphen = query + .split_whitespace() + .any(|t| !t.contains('"') && t.len() > 1 && t.contains('-') && !t.starts_with('-')); + if has_unquoted_hyphen { + return Err(anyhow::Error::new( + cortex_domain::DomainError::InvalidInput( + "hyphen is the FTS5 NOT operator; quote hyphenated terms as a phrase \ + (e.g. \"smoke-test\") or use --grep for literal text" + .to_string(), + ), + )); + } + if !query.matches('"').count().is_multiple_of(2) { + return Err(anyhow::Error::new( + cortex_domain::DomainError::InvalidInput( + "unbalanced quote in search query; wrap phrases in matching double quotes" + .to_string(), + ), + )); + } + Ok(()) +} + +/// Validate a user-supplied FTS5 query before execution. +/// +/// Limits: +/// - Max 512 characters (prevents very long queries from taxing the FTS tokenizer) +/// - Max 16 whitespace-separated terms (prevents 28+ wildcard term DoS) +/// +/// Returns a user-friendly error; the caller logs the details server-side. +pub fn validate_fts_query(query: &str) -> Result<()> { + lint_fts_query(query)?; + if query.len() > 512 { + return Err(anyhow::Error::new( + cortex_domain::DomainError::InvalidInput(format!( + "Search query too long ({} chars); maximum is 512 characters", + query.len() + )), + )); + } + let term_count = query.split_whitespace().count(); + if term_count > 16 { + return Err(anyhow::Error::new( + cortex_domain::DomainError::InvalidInput(format!( + "Search query has too many terms ({term_count}); maximum is 16 terms" + )), + )); + } + Ok(()) +} + +/// Column list for the FTS result projection (must match `map_row`'s order). +pub(super) const FTS_SELECT_COLS: &str = "l.id, l.timestamp, l.hostname, l.facility, l.severity, \ + l.app_name, l.process_id, l.message, l.received_at, l.source_ip, \ + l.ai_tool, l.ai_project, l.ai_session_id, l.ai_transcript_path, l.metadata_json"; + +fn search_logs_fts_sql( + query: &str, + params: &SearchParams, + limit: u32, +) -> (String, Vec) { + let mut bindings: Vec = + vec![rusqlite::types::Value::Text(query.to_string())]; + let mut idx = 2; + + if params.has_indexed_equality_filter() { + // Fast path: a selective indexed equality filter (hostname / source_ip + // / app_name / event_action / ai_project — NOT severity, see + // `has_indexed_equality_filter`) is present. Lead with that filter's + // composite `(, timestamp)` index and intersect against the FTS + // match set via a bloom-filtered `id IN (...)` subquery. SQLite walks + // the filtered partition newest-first and stops at LIMIT, so a + // host-scoped search of a common term drops from ~200s (full FTS scan) + // to sub-second. + // + // The match-set subquery is capped at the most-recent + // SEARCH_FTS_FAST_PATH_MATCH_CAP rowids: the non-correlated IN + // subquery is materialized in full before the index walk, which was + // unbounded memory for common terms (full-review PH1). Results for + // matches older than the newest 200K are dropped — callers needing + // deeper history should narrow the time range. + let mut sql = format!( + "SELECT {FTS_SELECT_COLS} + FROM logs l + WHERE l.id IN (SELECT rowid FROM logs_fts WHERE logs_fts MATCH ?1 + ORDER BY rowid DESC LIMIT {SEARCH_FTS_FAST_PATH_MATCH_CAP})" + ); + append_filters(&mut sql, &mut bindings, &mut idx, params); + sql.push_str(" ORDER BY l.timestamp DESC, l.id DESC"); + push_bound_limit(&mut sql, &mut bindings, &mut idx, "LIMIT", limit); + return (sql, bindings); + } + + // Default path (no indexed equality filter): materialize the most-recent + // FTS candidates by rowid, capped, then project. Fast when unfiltered + // because it never sorts the full match set; the cap bounds the work. + let mut sql = String::from( + "WITH fts_candidates(id, ts) AS MATERIALIZED ( + SELECT l.id, l.timestamp + FROM logs_fts + JOIN logs l ON l.id = logs_fts.rowid + WHERE logs_fts MATCH ?1", + ); + append_filters(&mut sql, &mut bindings, &mut idx, params); + sql.push_str(" ORDER BY logs_fts.rowid DESC"); + push_bound_limit( + &mut sql, + &mut bindings, + &mut idx, + "LIMIT", + SEARCH_FTS_CANDIDATE_CAP as i64, + ); + sql.push_str(&format!( + " + ) + SELECT {FTS_SELECT_COLS} + FROM fts_candidates c + JOIN logs l ON l.id = c.id + ORDER BY c.ts DESC, l.id DESC" + )); + push_bound_limit(&mut sql, &mut bindings, &mut idx, "LIMIT", limit); + (sql, bindings) +} + +#[derive(Debug, Default)] +struct SqlParams { + bindings: Vec, + next_idx: usize, +} + +impl SqlParams { + fn new(next_idx: usize) -> Self { + Self { + bindings: Vec::new(), + next_idx, + } + } + + fn push_text(&mut self, value: String) -> usize { + let idx = self.next_idx; + self.bindings.push(rusqlite::types::Value::Text(value)); + self.next_idx += 1; + idx + } +} + +fn push_required_ai_filters(sql: &mut String, alias: &str) { + sql.push_str(&format!( + " AND {alias}.ai_project IS NOT NULL AND {alias}.ai_project != '' + AND {alias}.ai_tool IS NOT NULL AND {alias}.ai_tool != '' + AND {alias}.ai_session_id IS NOT NULL AND {alias}.ai_session_id != ''" + )); +} + +fn push_ai_scope_filters( + sql: &mut String, + params: &mut SqlParams, + alias: &str, + project: &Option, + tool: &Option, + from: &Option, + to: &Option, +) { + if let Some(project) = project { + let idx = params.push_text(project.clone()); + sql.push_str(&format!(" AND {alias}.ai_project = ?{idx}")); + } + if let Some(tool) = tool { + let idx = params.push_text(tool.clone()); + sql.push_str(&format!(" AND {alias}.ai_tool = ?{idx}")); + } + if let Some(from) = from { + let idx = params.push_text(from.clone()); + sql.push_str(&format!(" AND {alias}.timestamp >= ?{idx}")); + } + if let Some(to) = to { + let idx = params.push_text(to.clone()); + sql.push_str(&format!(" AND {alias}.timestamp <= ?{idx}")); + } +} + +/// Search logs with flexible filtering + FTS +pub fn search_logs(pool: &DbPool, params: &SearchParams) -> Result> { + let conn = pool.get()?; + let limit = params.limit.unwrap_or(100).min(1000); + + // If we have a full-text query, use FTS5 join + if let Some(ref query) = params.query { + validate_fts_query(query)?; + + let (sql, bindings) = search_logs_fts_sql(query, params, limit); + + let mut stmt = conn.prepare(&sql)?; + let rows = stmt + .query_map(rusqlite::params_from_iter(bindings.iter()), map_row) + .map_err(|e| { + tracing::error!(error = %e, query = %query, "FTS5 MATCH query failed"); + anyhow::anyhow!("Search query failed") + })?; + rows.collect::>>().map_err(|e| { + tracing::error!(error = %e, query = %query, "FTS5 row mapping failed"); + anyhow::anyhow!("Search query failed") + }) + } else { + let mut sql = String::from( + "SELECT l.id, l.timestamp, l.hostname, l.facility, l.severity, + l.app_name, l.process_id, l.message, l.received_at, l.source_ip, + l.ai_tool, l.ai_project, l.ai_session_id, l.ai_transcript_path, l.metadata_json + FROM logs l WHERE 1=1", + ); + let mut bindings: Vec = vec![]; + let mut idx = 1; + + append_filters(&mut sql, &mut bindings, &mut idx, params); + sql.push_str(" ORDER BY l.timestamp DESC"); + push_bound_limit(&mut sql, &mut bindings, &mut idx, "LIMIT", limit); + + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map(rusqlite::params_from_iter(bindings.iter()), map_row)?; + Ok(rows.collect::>>()?) + } +} + +/// Get the N most recent logs for a host/service +pub fn tail_logs( + pool: &DbPool, + hostname: Option<&str>, + source_ip: Option<&str>, + app_name: Option<&str>, + severity_in: Option<&[String]>, + n: u32, +) -> Result> { + let conn = pool.get()?; + let (sql, bindings) = tail_logs_sql(hostname, source_ip, app_name, severity_in, n); + + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map(rusqlite::params_from_iter(bindings.iter()), map_row)?; + Ok(rows.collect::>>()?) +} + +pub fn page_agent_projection_logs( + pool: &DbPool, + after_id: i64, + limit: usize, +) -> Result> { + if after_id < 0 || !(1..=500).contains(&limit) { + anyhow::bail!("projection log cursor/limit out of bounds"); + } + let conn = pool.get()?; + let mut statement = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, + app_name, process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs WHERE id > ?1 ORDER BY id LIMIT ?2", + )?; + Ok(statement + .query_map(rusqlite::params![after_id, limit as i64], map_row)? + .collect::>>()?) +} + +fn tail_logs_sql( + hostname: Option<&str>, + source_ip: Option<&str>, + app_name: Option<&str>, + severity_in: Option<&[String]>, + n: u32, +) -> (String, Vec) { + let n = n.min(500); + + // Severity-only fast path: with no other filter, the generic plan walks + // idx_logs_timestamp newest-first and filters — O(table) when the + // requested severities are rare (e.g. `tail severity_in=[emerg,alert]`, + // full-review PM6). Instead probe `idx_logs_sev_time (severity, + // timestamp)` once per severity with its own LIMIT and merge: each arm is + // a bounded index walk, and the outer sort covers at most + // severities × n ≤ 8 × 500 rows. `n` is server-clamped, so interpolating + // it is safe. + if hostname.is_none() + && source_ip.is_none() + && app_name.is_none() + && let Some(levels) = severity_in.filter(|levels| !levels.is_empty()) + { + const COLS: &str = "id, timestamp, hostname, facility, severity, \ + app_name, process_id, message, received_at, source_ip, \ + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json"; + let mut bindings: Vec = Vec::with_capacity(levels.len()); + let arms = levels + .iter() + .enumerate() + .map(|(i, lvl)| { + bindings.push(rusqlite::types::Value::Text(lvl.clone())); + format!( + "SELECT * FROM (SELECT {COLS} FROM logs WHERE severity = ?{} \ + ORDER BY timestamp DESC LIMIT {n})", + i + 1 + ) + }) + .collect::>() + .join(" UNION ALL "); + let sql = format!("{arms} ORDER BY timestamp DESC LIMIT {n}"); + return (sql, bindings); + } + + let mut sql = String::from( + "SELECT id, timestamp, hostname, facility, severity, + app_name, process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs WHERE 1=1", + ); + let mut bindings: Vec = vec![]; + let mut idx = 1; + + if let Some(h) = hostname { + append_host_selector(&mut sql, &mut bindings, &mut idx, "hostname", h); + } + if let Some(source_ip) = source_ip { + sql.push_str(&format!(" AND source_ip = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(source_ip.to_string())); + idx += 1; + } + if let Some(a) = app_name { + sql.push_str(&format!(" AND app_name = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(a.to_string())); + idx += 1; + } + if let Some(levels) = severity_in + && !levels.is_empty() + { + let placeholders: Vec = + (0..levels.len()).map(|i| format!("?{}", idx + i)).collect(); + sql.push_str(&format!(" AND severity IN ({})", placeholders.join(", "))); + for lvl in levels { + bindings.push(rusqlite::types::Value::Text(lvl.clone())); + idx += 1; + } + debug_assert_eq!(bindings.len() + 1, idx); + } + + sql.push_str(" ORDER BY timestamp DESC"); + push_bound_limit(&mut sql, &mut bindings, &mut idx, "LIMIT", n); + (sql, bindings) +} + +/// Get error/warning summary per host in a time window. When `group_by_app` is +/// true, results also include `app_name` as a secondary grouping key. +pub fn get_error_summary( + pool: &DbPool, + from: Option<&str>, + to: Option<&str>, + group_by_app: bool, + limit: Option, +) -> Result> { + let conn = pool.get()?; + let (sql, bindings) = get_error_summary_sql(from, to, group_by_app, limit); + + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + Ok(ErrorSummaryEntry { + hostname: row.get(0)?, + app_name: row.get::<_, Option>(1)?, + severity: row.get(2)?, + count: row.get(3)?, + }) + })?; + Ok(rows.collect::>>()?) +} + +fn get_error_summary_sql( + from: Option<&str>, + to: Option<&str>, + group_by_app: bool, + limit: Option, +) -> (String, Vec) { + let from = from.unwrap_or("1970-01-01T00:00:00Z"); + // Upper sentinel: any valid RFC 3339 timestamp will sort before this. + let to = to.unwrap_or("9999-12-31T23:59:59Z"); + + let mut bindings = vec![ + rusqlite::types::Value::Text(from.to_string()), + rusqlite::types::Value::Text(to.to_string()), + ]; + let mut idx = 3usize; + + let mut sql = if group_by_app { + "SELECT hostname, app_name, severity, COUNT(*) as count + FROM logs + WHERE severity IN ('emerg', 'alert', 'crit', 'err', 'warning') + AND timestamp BETWEEN ?1 AND ?2 + GROUP BY hostname, app_name, severity + ORDER BY hostname, app_name, count DESC" + .to_string() + } else { + "SELECT hostname, NULL AS app_name, severity, COUNT(*) as count + FROM logs + WHERE severity IN ('emerg', 'alert', 'crit', 'err', 'warning') + AND timestamp BETWEEN ?1 AND ?2 + GROUP BY hostname, severity + ORDER BY hostname, count DESC" + .to_string() + }; + if let Some(limit) = limit { + push_bound_limit(&mut sql, &mut bindings, &mut idx, "LIMIT", limit.max(1)); + } + (sql, bindings) +} + +pub(crate) use super::queries_hosts::canonical_host_keys; +pub use super::queries_hosts::list_hosts; + +/// List AI transcript sessions ordered by recency. +/// +/// Fast path (bead cortex-2vre): when the caller supplies NO time window +/// (`from`/`to` both unset) the result is served from the periodically-refreshed +/// `ai_session_rollup` materialization — an O(#sessions) indexed read instead of +/// the O(#AI-rows) GROUP-BY + temp-btree sort that grew to ~4s at 10M rows. The +/// rollup is refreshed on a background cadence, so unbounded results reflect data +/// as of the last refresh; reach for [`ai_session_rollup_status`] to surface +/// staleness. If the rollup has never been refreshed (e.g. immediately after a +/// migration, before the background task runs) the fast path transparently falls +/// back to the live aggregation, so correctness never depends on the rollup being +/// warm. +/// +/// Slow/exact path: when a time window IS supplied, the query is bounded by the +/// timestamp index and runs live against `logs` (the rollup pre-aggregates across +/// all time and cannot answer a windowed `event_count`/`first_seen`/`last_seen`). +pub fn list_ai_sessions( + pool: &DbPool, + params: &ListAiSessionsParams, +) -> Result> { + let time_filtered = params.since.is_some() || params.until.is_some(); + if !time_filtered && ai_session_rollup_is_populated(pool)? { + return list_ai_sessions_from_rollup(pool, params); + } + list_ai_sessions_live(pool, params) +} + +/// Live aggregation over `logs`. This is the ground-truth implementation used +/// for time-windowed queries and to (re)compute the rollup. +pub fn list_ai_sessions_live( + pool: &DbPool, + params: &ListAiSessionsParams, +) -> Result> { + let conn = pool.get()?; + let limit = params.limit.unwrap_or(100).min(1000); + let mut sql = String::from( + "SELECT ai_project, ai_tool, ai_session_id, + MIN(ai_transcript_path) AS ai_transcript_path, + hostname, + MIN(timestamp) AS first_seen, + MAX(timestamp) AS last_seen, + COUNT(*) AS event_count + FROM logs + WHERE ai_project IS NOT NULL + AND ai_project != '' + AND ai_tool IS NOT NULL + AND ai_tool != '' + AND ai_session_id IS NOT NULL + AND ai_session_id != ''", + ); + let mut bindings: Vec = vec![]; + let mut idx = 1; + + if let Some(project) = ¶ms.ai_project { + sql.push_str(&format!(" AND ai_project = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(project.clone())); + idx += 1; + } + if let Some(tool) = ¶ms.ai_tool { + sql.push_str(&format!(" AND ai_tool = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(tool.clone())); + idx += 1; + } + if let Some(hostname) = ¶ms.host { + sql.push_str(&format!(" AND hostname = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(hostname.clone())); + idx += 1; + } + if let Some(from) = ¶ms.since { + sql.push_str(&format!(" AND timestamp >= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(from.clone())); + idx += 1; + } + if let Some(to) = ¶ms.until { + sql.push_str(&format!(" AND timestamp <= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(to.clone())); + } + + sql.push_str(&format!( + " GROUP BY ai_project, ai_tool, ai_session_id, hostname + ORDER BY last_seen DESC + LIMIT {limit}" + )); + + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + Ok(AiSessionEntry { + ai_project: row.get(0)?, + ai_tool: row.get(1)?, + ai_session_id: row.get(2)?, + ai_transcript_path: row.get(3)?, + hostname: row.get(4)?, + first_seen: row.get(5)?, + last_seen: row.get(6)?, + event_count: row.get(7)?, + }) + })?; + Ok(rows.collect::>>()?) +} + +/// Indexed read from the `ai_session_rollup` materialization (no time window). +fn list_ai_sessions_from_rollup( + pool: &DbPool, + params: &ListAiSessionsParams, +) -> Result> { + let conn = pool.get()?; + let limit = params.limit.unwrap_or(100).min(1000); + let mut sql = String::from( + "SELECT ai_project, ai_tool, ai_session_id, ai_transcript_path, + hostname, first_seen, last_seen, event_count + FROM ai_session_rollup + WHERE 1=1", + ); + let mut bindings: Vec = vec![]; + let mut idx = 1; + if let Some(project) = ¶ms.ai_project { + sql.push_str(&format!(" AND ai_project = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(project.clone())); + idx += 1; + } + if let Some(tool) = ¶ms.ai_tool { + sql.push_str(&format!(" AND ai_tool = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(tool.clone())); + idx += 1; + } + if let Some(hostname) = ¶ms.host { + sql.push_str(&format!(" AND hostname = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(hostname.clone())); + } + // Order by last_seen DESC ONLY — exactly mirroring the live path's + // `ORDER BY last_seen DESC`. A single-column order lets SQLite serve the + // sort straight from idx_ai_session_rollup_last_seen with NO temp b-tree + // (the cost that made the live aggregation slow). Adding tiebreak columns + // would reintroduce a temp b-tree, so ties stay engine-arbitrary here just + // as they are in the live query. + sql.push_str(&format!(" ORDER BY last_seen DESC LIMIT {limit}")); + + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + Ok(AiSessionEntry { + ai_project: row.get(0)?, + ai_tool: row.get(1)?, + ai_session_id: row.get(2)?, + ai_transcript_path: row.get(3)?, + hostname: row.get(4)?, + first_seen: row.get(5)?, + last_seen: row.get(6)?, + event_count: row.get(7)?, + }) + })?; + Ok(rows.collect::>>()?) +} + +/// True once the rollup has been refreshed at least once (`refreshed_at` set). +/// Before the first refresh, `list_ai_sessions` falls back to the live path. +fn ai_session_rollup_is_populated(pool: &DbPool) -> Result { + let conn = pool.get()?; + let refreshed: Option = conn + .query_row( + "SELECT refreshed_at FROM ai_session_rollup_meta WHERE id = 1", + [], + |r| r.get::<_, Option>(0), + ) + .optional()? + .flatten(); + Ok(refreshed.is_some()) +} + +/// Staleness snapshot for the AI session rollup. +#[derive(Debug, Clone)] +pub struct AiSessionRollupStatus { + /// RFC 3339 timestamp of the last successful refresh, or `None` if never. + pub refreshed_at: Option, + /// Number of session rows in the rollup as of the last refresh. + pub row_count: i64, +} + +/// Read the rollup staleness metadata (cheap single-row lookup). +pub fn ai_session_rollup_status(pool: &DbPool) -> Result { + let conn = pool.get()?; + let (refreshed_at, row_count) = conn + .query_row( + "SELECT refreshed_at, row_count FROM ai_session_rollup_meta WHERE id = 1", + [], + |r| Ok((r.get::<_, Option>(0)?, r.get::<_, i64>(1)?)), + ) + .optional()? + .unwrap_or((None, 0)); + Ok(AiSessionRollupStatus { + refreshed_at, + row_count, + }) +} + +impl AiSessionRollupStatus { + /// Human-readable staleness summary, e.g. for `db status` / diagnostics: + /// `"42 sessions, refreshed 2026-05-29T12:00:00.000Z"` or `"never refreshed"`. + pub fn summary(&self) -> String { + match &self.refreshed_at { + Some(ts) => format!("{} sessions, refreshed {ts}", self.row_count), + None => "never refreshed".to_string(), + } + } +} + +/// Cheap source-side fingerprint of the AI-row partition used to decide whether +/// the rollup is stale. `(COUNT(*), MAX(id))` over rows that *could* contribute +/// to the rollup. Computed index-only from `idx_logs_ai_project_time` +/// (partial index `WHERE ai_project IS NOT NULL`): the `!= ''` residual is on +/// the index's leading column, and `id` is the implicit rowid carried in every +/// index entry, so neither a table lookup nor a temp b-tree is needed. +/// +/// The predicate is intentionally BROADER than the rollup's contributing-row +/// filter (it omits the `ai_tool`/`ai_session_id` checks): any row that +/// contributes to the rollup necessarily has `ai_project != ''`, so it is +/// counted here too. That makes the fingerprint *conservative* — it may change +/// (forcing a refresh) for a non-contributing row, but it can never miss a +/// change to a contributing row. `id` is a monotonic AUTOINCREMENT PK, so an +/// insert always advances `MAX(id)` and a delete always changes `COUNT(*)` +/// and/or `MAX(id)`. In-place UPDATEs to a row's rollup-relevant columns would +/// be invisible to this fingerprint, but the ingest path never does them: +/// verified there is no `UPDATE ... logs` anywhere (the scanner re-indexes by +/// `DELETE FROM logs` + re-INSERT, both of which the fingerprint catches). +fn ai_rows_watermark(conn: &rusqlite::Connection) -> rusqlite::Result<(i64, i64)> { + conn.query_row( + "SELECT COUNT(*), COALESCE(MAX(id), 0) FROM logs + WHERE ai_project IS NOT NULL AND ai_project != ''", + [], + |r| Ok((r.get(0)?, r.get(1)?)), + ) +} + +/// Outcome of a conditional rollup refresh. +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum RollupRefresh { + /// The source fingerprint changed; the rollup was recomputed. + Refreshed { row_count: usize }, + /// The source fingerprint was unchanged since the last refresh; the + /// expensive re-aggregation was skipped. + Skipped, +} + +/// Refresh the rollup only if the AI-row partition changed since the last +/// refresh (bead cortex-g33v). The full re-aggregation is a temp-btree +/// `GROUP BY` over the whole AI partition (~4s at scale) and holds the +/// maintenance permit while running; the common case on the background cadence +/// is "nothing changed", so this skips that work via the cheap +/// the private `ai_rows_watermark` fingerprint. +/// +/// The skip is correct because [`refresh_ai_session_rollup`] stamps the exact +/// fingerprint of the data it aggregated; if the live fingerprint still matches +/// and we have refreshed at least once, the materialization is already current. +pub fn refresh_ai_session_rollup_if_stale(pool: &DbPool) -> Result { + { + let conn = pool.get()?; + let (cur_count, cur_max_id) = ai_rows_watermark(&conn)?; + let stored: Option<(Option, i64, i64)> = conn + .query_row( + "SELECT refreshed_at, source_row_count, source_max_id + FROM ai_session_rollup_meta WHERE id = 1", + [], + |r| { + Ok(( + r.get::<_, Option>(0)?, + r.get::<_, i64>(1)?, + r.get::<_, i64>(2)?, + )) + }, + ) + .optional()?; + if let Some((Some(_refreshed_at), src_count, src_max_id)) = stored + && src_count == cur_count + && src_max_id == cur_max_id + { + return Ok(RollupRefresh::Skipped); + } + } + let row_count = refresh_ai_session_rollup(pool)?; + Ok(RollupRefresh::Refreshed { row_count }) +} + +/// Recompute the `ai_session_rollup` materialization from `logs` using a +/// **staging + atomic swap** strategy, then stamp `refreshed_at` and the source +/// watermark. Returns the number of session rows. This is the +/// unconditional/force path; the background task uses +/// [`refresh_ai_session_rollup_if_stale`] to skip no-op refreshes. +/// +/// This is a FULL recompute (not incremental) and stays correct under retention +/// DELETEs: AI rows ingest at `info` and get NO severity exemption from the +/// purge paths (maintenance.rs), so they ARE deleted out from under the rollup. +/// A watermark-incremental refresh would corrupt `MIN(first_seen)`, leave ghost +/// rollup rows for fully-purged sessions, and drift `event_count` — MIN/MAX are +/// non-self-maintainable aggregates. See the Migration 21 note in `pool.rs` and +/// bead syslog-mcp-rvcz for the full rationale. +/// +/// ## Staging + swap (writer-starvation fix, bead syslog-mcp-rvcz) +/// The full `GROUP BY` over the AI partition costs ~4s at scale. Previously it +/// ran inside the `IMMEDIATE` write transaction, holding the single WAL writer +/// slot for that whole window and starving the ingest writer (dropped inserts) +/// and bloating the WAL. We now split it: +/// 1. **Build** the full aggregation into a connection-local TEMP staging +/// table under a READ snapshot — WAL readers do NOT block the writer, so +/// this holds ZERO write lock for the entire ~4s. +/// 2. **Swap** under a sub-millisecond `IMMEDIATE` transaction: +/// `DELETE` + `INSERT ... SELECT * FROM staging` + stamp meta + `COMMIT`. +/// +/// ### INVARIANT — the build and the swap MUST use the SAME `Connection`. +/// The staging table is a `TEMP` table, which is **connection-local**: it is +/// only visible to the rusqlite `Connection` that created it. This function +/// deliberately holds ONE `conn` (from a single `pool.get()`) across both +/// phases. A future refactor that splits the build and swap into helpers that +/// each call `pool.get()` would silently produce an EMPTY staging table and +/// wipe the rollup (data-loss regression). DO NOT split the connection. The +/// `assert`/guard before the swap (staging row count == built row count) exists +/// to catch exactly that mistake at runtime. +pub fn refresh_ai_session_rollup(pool: &DbPool) -> Result { + // ONE connection for BOTH phases — the TEMP staging table is + // connection-local (see the INVARIANT in the doc comment above). + let mut conn = pool.get()?; + + // --- Phase 1: BUILD under a read snapshot (no write lock held) --------- + // A DEFERRED transaction takes a WAL read snapshot on its first read and + // never upgrades to a writer here (we only CREATE TEMP + SELECT), so it + // does not contend for the single WAL writer slot. The watermark and the + // GROUP BY both read from this one consistent snapshot, so the stored + // fingerprint exactly describes the data we aggregate. + let (src_count, src_max_id, staged, rollup_eligible) = { + let build = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Deferred)?; + let (src_count, src_max_id) = ai_rows_watermark(&build)?; + // TEMP table: connection-local, spills to the temp store (/tmp), never + // to /data. Rebuilt every refresh, so drop any stale prior copy. + build.execute("DROP TABLE IF EXISTS _ai_rollup_staging", [])?; + build.execute( + "CREATE TEMP TABLE _ai_rollup_staging AS + SELECT ai_project, ai_tool, ai_session_id, hostname, + MIN(ai_transcript_path) AS ai_transcript_path, + MIN(timestamp) AS first_seen, + MAX(timestamp) AS last_seen, + COUNT(*) AS event_count + FROM logs + WHERE ai_project IS NOT NULL AND ai_project != '' + AND ai_tool IS NOT NULL AND ai_tool != '' + AND ai_session_id IS NOT NULL AND ai_session_id != '' + GROUP BY ai_project, ai_tool, ai_session_id, hostname", + [], + )?; + let staged: i64 = + build.query_row("SELECT COUNT(*) FROM _ai_rollup_staging", [], |r| r.get(0))?; + // Rollup-eligible row count under the SAME read snapshot, using the + // EXACT predicate as the staging INSERT above. Must be computed inside + // this transaction (not after commit / on a fresh connection): under + // one snapshot, any row matching this predicate yields >=1 GROUP BY + // group, so `staged == 0` IMPLIES `rollup_eligible == 0`. The R1 guard + // below relies on that mutual consistency; counting under a different + // snapshot would let a concurrent INSERT revive a false positive. + let rollup_eligible: i64 = build.query_row( + "SELECT COUNT(*) FROM logs + WHERE ai_project IS NOT NULL AND ai_project != '' + AND ai_tool IS NOT NULL AND ai_tool != '' + AND ai_session_id IS NOT NULL AND ai_session_id != ''", + [], + |r| r.get(0), + )?; + // Commit the read snapshot (releases the read lock). The TEMP table + // survives the commit — it is tied to the connection, not the txn. + build.commit()?; + (src_count, src_max_id, staged, rollup_eligible) + }; + + // R1 guardrail (bead syslog-mcp-rvcz security addendum): the same-connection + // requirement is NOT compile-time enforceable. If a refactor ever ran the + // build on a different pooled connection, the TEMP table would be invisible + // here and the swap would wipe the rollup. We must distinguish that + // regression from a LEGITIMATELY empty rollup: rows can have `ai_project` + // set but no recognized `ai_tool`/`ai_session_id` (e.g. OTLP logs carrying + // only project.path), which the watermark counts (`src_count > 0`) but the + // rollup GROUP BY correctly excludes (`staged == 0`). Comparing against + // `src_count` would error forever on that data shape. Instead, only bail + // when staging is empty AND rows matching the FULL rollup predicate exist — + // i.e. the build genuinely produced groups but the TEMP table is invisible. + debug_assert!(staged >= 0, "staging row count must be non-negative"); + if staged == 0 && rollup_eligible > 0 { + return Err(anyhow::anyhow!( + "ai_session_rollup staging table is empty despite {rollup_eligible} \ + rollup-eligible AI rows present — the build and swap MUST share one \ + Connection (TEMP tables are connection-local); refusing to wipe the \ + rollup" + )); + } + + // --- Phase 2: SWAP under a sub-millisecond IMMEDIATE write lock --------- + // IMMEDIATE (not DEFERRED): take the write lock up front. We read nothing + // before the DELETE here, but IMMEDIATE keeps the swap a single short + // writer that never risks an SQLITE_BUSY_SNAPSHOT upgrade failure (which + // busy_timeout does NOT retry). The GROUP BY is already done, so this lock + // is held only for the DELETE + INSERT-from-staging + meta UPDATE. + let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + tx.execute("DELETE FROM ai_session_rollup", [])?; + tx.execute( + "INSERT INTO ai_session_rollup + (ai_project, ai_tool, ai_session_id, hostname, + ai_transcript_path, first_seen, last_seen, event_count) + SELECT ai_project, ai_tool, ai_session_id, hostname, + ai_transcript_path, first_seen, last_seen, event_count + FROM _ai_rollup_staging", + [], + )?; + // The staged count is already known from Phase 1 — use it directly rather than + // running a post-INSERT COUNT(*) inside the IMMEDIATE transaction, which + // unnecessarily extends write-lock hold time. + let row_count = staged; + tx.execute( + "UPDATE ai_session_rollup_meta + SET refreshed_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), + row_count = ?1, + source_row_count = ?2, + source_max_id = ?3 + WHERE id = 1", + params![row_count, src_count, src_max_id], + )?; + tx.commit()?; + // Drop the TEMP table so a long-lived pooled connection doesn't carry it + // back into the pool. Best-effort: a failure here doesn't affect the + // already-committed swap. + let _ = conn.execute("DROP TABLE IF EXISTS _ai_rollup_staging", []); + Ok(row_count as usize) +} + +// ----------------------------------------------------------------------------- +// timeline_hourly rollup (bead syslog-mcp-kcvq) +// ----------------------------------------------------------------------------- + +/// SQLite `strftime` pattern bucketing a `timestamp` into the hour grain stored +/// in `timeline_hourly.bucket`. Kept in one place so the backfill (pool.rs), the +/// incremental refresh, and any future caller stay byte-identical. +pub const TIMELINE_HOUR_FMT: &str = "%Y-%m-%dT%H:00:00Z"; + +/// Staleness/coverage snapshot for the `timeline_hourly` rollup. +#[derive(Debug, Clone)] +pub struct TimelineRollupStatus { + /// RFC 3339 timestamp of the last successful incremental refresh, or `None`. + pub refreshed_at: Option, + /// Highest `logs.id` aggregated into the rollup so far. Reads add the live + /// delta `WHERE id > source_max_id` on top of the rollup for fresh totals. + /// Exposed for diagnostics/tests; the read paths query the meta row directly. + pub source_max_id: i64, +} + +/// Read the timeline rollup metadata (cheap single-row lookup). +pub fn timeline_rollup_status(pool: &DbPool) -> Result { + let conn = pool.get()?; + let (refreshed_at, source_max_id) = conn + .query_row( + "SELECT refreshed_at, source_max_id FROM timeline_hourly_meta WHERE id = 1", + [], + |r| Ok((r.get::<_, Option>(0)?, r.get::<_, i64>(1)?)), + ) + .optional()? + .unwrap_or((None, 0)); + Ok(TimelineRollupStatus { + refreshed_at, + source_max_id, + }) +} + +/// Incrementally fold new `logs` rows into `timeline_hourly`. +/// +/// Aggregates ONLY `logs WHERE id > source_max_id AND id <= MAX(id)` and +/// upsert-ADDS into the per-hour buckets, then advances the watermark. This is +/// self-maintainable for adds because the rollup holds only `COUNT(*)` (no +/// MIN/MAX): a new high-id row with an old timestamp correctly adds to its old +/// bucket. The `id <= new_max` upper bound is captured inside the same IMMEDIATE +/// transaction as the aggregate, so a row inserted mid-refresh is neither +/// double-counted now nor skipped next tick. +/// +/// `app_name` is normalized to `COALESCE(app_name,'')` to match the NOT NULL PK +/// column — without this, null-app rows would never hit the ON CONFLICT path and +/// would duplicate every tick. +/// +/// Per cadence this touches only the rows ingested since the last tick +/// (milliseconds), unlike the AI rollup's full re-aggregation, so a single short +/// IMMEDIATE write is correct and simpler than the staging+swap dance. +/// +/// Returns the number of source `logs` rows folded in this tick (0 when the +/// watermark was already current — the common idle-tick case). +pub fn refresh_timeline_rollup(pool: &DbPool) -> Result { + let mut conn = pool.get()?; + let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Immediate)?; + let old_max: i64 = tx.query_row( + "SELECT source_max_id FROM timeline_hourly_meta WHERE id = 1", + [], + |r| r.get(0), + )?; + let new_max: i64 = tx.query_row("SELECT COALESCE(MAX(id), 0) FROM logs", [], |r| r.get(0))?; + if new_max <= old_max { + // Watermark already current; nothing new to fold. (Deletes are handled + // out-of-band by the retention prune, not here.) + tx.commit()?; + return Ok(0); + } + let folded: i64 = tx.query_row( + "SELECT COUNT(*) FROM logs WHERE id > ?1 AND id <= ?2", + params![old_max, new_max], + |r| r.get(0), + )?; + tx.execute( + &format!( + "INSERT INTO timeline_hourly (bucket, hostname, app_name, severity, event_count) + SELECT strftime('{TIMELINE_HOUR_FMT}', timestamp) AS bucket, + hostname, + COALESCE(app_name, '') AS app_name, + severity, + COUNT(*) AS event_count + FROM logs + WHERE id > ?1 AND id <= ?2 + GROUP BY bucket, hostname, app_name, severity + ON CONFLICT(bucket, hostname, app_name, severity) + DO UPDATE SET event_count = event_count + excluded.event_count" + ), + params![old_max, new_max], + )?; + tx.execute( + "UPDATE timeline_hourly_meta + SET refreshed_at = strftime('%Y-%m-%dT%H:%M:%fZ', 'now'), + source_max_id = ?1 + WHERE id = 1", + [new_max], + )?; + tx.commit()?; + Ok(folded as usize) +} + +/// Prune `timeline_hourly` buckets that are entirely older than the oldest +/// remaining `logs` row, called after a retention purge (which deletes oldest +/// rows by `received_at`). Removes ghost buckets so `timeline`/`stats` totals do +/// not drift upward unbounded on hosts whose ingest watermark is idle while +/// retention keeps purging. +/// +/// A minor transient overcount can remain in the single boundary hour (the hour +/// straddling the purge cutoff keeps its full pre-purge count until that hour +/// itself ages out) — accepted as negligible for a volume chart. +/// +/// Returns the number of rollup rows deleted. +pub fn prune_timeline_rollup(pool: &DbPool) -> Result { + let conn = pool.get()?; + // Fetch MIN(timestamp) as a plain string and apply strftime formatting in Rust + // so SQLite can use the B-tree MIN optimization (a single leaf seek) rather than + // scanning the full index when strftime() wraps the MIN expression. + let oldest_ts: Option = conn.query_row("SELECT MIN(timestamp) FROM logs", [], |r| { + r.get::<_, Option>(0) + })?; + let Some(oldest_ts) = oldest_ts else { + // No logs at all — clear the whole rollup so it can't ghost. + let n = conn.execute("DELETE FROM timeline_hourly", [])?; + return Ok(n); + }; + // Truncate to the hour bucket format used by the rollup (e.g. "2024-01-15T14:00:00Z"). + // Timestamps are RFC 3339 / ISO 8601 strings with at least 13 chars ("YYYY-MM-DDTHH"). + let oldest_bucket = if oldest_ts.len() >= 13 { + format!("{}:00:00Z", &oldest_ts[..13]) + } else { + oldest_ts + }; + let n = conn.execute( + "DELETE FROM timeline_hourly WHERE bucket < ?1", + [oldest_bucket], + )?; + Ok(n) +} + +pub fn search_ai_sessions( + pool: &DbPool, + params: &SearchAiSessionsParams, +) -> Result { + validate_fts_query(¶ms.query)?; + + let limit = params.limit.unwrap_or(20).clamp(1, 100) as usize; + let conn = pool.get()?; + let (sql, bindings) = search_ai_sessions_sql(params, limit); + + let mut stmt = conn.prepare(&sql)?; + let mut total_candidates = 0usize; + let mut raw_candidate_count = 0usize; + let rows = stmt.query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + total_candidates = row.get::<_, i64>(9)? as usize; + raw_candidate_count = row.get::<_, i64>(10)? as usize; + Ok(SearchedAiSessionEntry { + ai_project: row.get(0)?, + ai_tool: row.get(1)?, + ai_session_id: row.get(2)?, + hostname: row.get(3)?, + first_seen: row.get(4)?, + last_seen: row.get(5)?, + event_count: row.get(6)?, + match_count: row.get(7)?, + best_snippet: row.get(8)?, + }) + })?; + let sessions = rows.collect::>>()?; + + Ok(SearchAiSessionsResult { + total_candidates, + candidate_rows: raw_candidate_count.min(CANDIDATE_CAP), + candidate_cap: CANDIDATE_CAP, + candidate_window_truncated: raw_candidate_count > CANDIDATE_CAP, + truncated: total_candidates > sessions.len() || raw_candidate_count > CANDIDATE_CAP, + sessions, + }) +} + +const CANDIDATE_CAP: usize = 5_000; + +fn search_ai_sessions_sql( + params: &SearchAiSessionsParams, + limit: usize, +) -> (String, Vec) { + let mut filters = String::new(); + push_required_ai_filters(&mut filters, "l"); + let mut query_params = SqlParams::new(2); + query_params + .bindings + .push(rusqlite::types::Value::Text(params.query.clone())); + push_ai_scope_filters( + &mut filters, + &mut query_params, + "l", + ¶ms.ai_project, + ¶ms.ai_tool, + ¶ms.since, + ¶ms.until, + ); + if let Some(hostname) = ¶ms.host { + let idx = query_params.push_text(hostname.clone()); + filters.push_str(&format!(" AND l.hostname = ?{idx}")); + } + if let Some(app_name) = ¶ms.app { + let idx = query_params.push_text(app_name.clone()); + filters.push_str(&format!(" AND l.app_name = ?{idx}")); + } + let sql = format!( + "WITH candidates AS MATERIALIZED ( + SELECT l.ai_project, + l.ai_tool, + l.ai_session_id, + l.hostname, + l.timestamp, + l.message + FROM logs_fts + JOIN logs l ON l.id = logs_fts.rowid + WHERE logs_fts MATCH ?1{filters} + ORDER BY logs_fts.rowid DESC + LIMIT {} + ), + bounded_candidates AS MATERIALIZED ( + SELECT * FROM candidates + LIMIT {CANDIDATE_CAP} + ), + grouped AS MATERIALIZED ( + SELECT ai_project, + ai_tool, + ai_session_id, + hostname, + COUNT(*) AS match_count, + MIN(timestamp) AS first_match, + MAX(timestamp) AS latest_match + FROM bounded_candidates c + GROUP BY ai_project, ai_tool, ai_session_id, hostname + ), + selected AS MATERIALIZED ( + SELECT * + FROM grouped + ORDER BY latest_match DESC + LIMIT {limit} + ), + rollup_meta AS MATERIALIZED ( + SELECT COALESCE(MAX(source_max_id), 0) AS source_max_id + FROM ai_session_rollup_meta + ), + tail_stats AS MATERIALIZED ( + SELECT l.ai_project, + l.ai_tool, + l.ai_session_id, + l.hostname, + MIN(l.timestamp) AS first_seen, + MAX(l.timestamp) AS last_seen, + COUNT(*) AS event_count + FROM selected g + CROSS JOIN rollup_meta meta + JOIN logs l INDEXED BY idx_logs_ai_session_host_time + ON g.ai_project = l.ai_project + AND g.ai_tool = l.ai_tool + AND g.ai_session_id = l.ai_session_id + AND g.hostname = l.hostname + AND l.id > meta.source_max_id + WHERE l.ai_project IS NOT NULL + AND l.ai_tool IS NOT NULL + AND l.ai_session_id IS NOT NULL + GROUP BY l.ai_project, l.ai_tool, l.ai_session_id, l.hostname + ), + totals AS MATERIALIZED ( + SELECT COUNT(*) AS total_candidates, + COALESCE(SUM(match_count), 0) AS raw_candidate_count + FROM ( + SELECT COUNT(*) AS match_count + FROM candidates + GROUP BY ai_project, ai_tool, ai_session_id, hostname + ) filtered_sessions + ) + SELECT g.ai_project, g.ai_tool, g.ai_session_id, g.hostname, + COALESCE(CASE + WHEN rollup.first_seen IS NULL THEN tail.first_seen + WHEN tail.first_seen IS NULL THEN rollup.first_seen + ELSE MIN(rollup.first_seen, tail.first_seen) + END, g.first_match) AS first_seen, + COALESCE(CASE + WHEN rollup.last_seen IS NULL THEN tail.last_seen + WHEN tail.last_seen IS NULL THEN rollup.last_seen + ELSE MAX(rollup.last_seen, tail.last_seen) + END, g.latest_match) AS last_seen, + CASE + WHEN rollup.event_count IS NULL AND tail.event_count IS NULL + THEN g.match_count + ELSE COALESCE(rollup.event_count, 0) + COALESCE(tail.event_count, 0) + END AS event_count, + g.match_count, + ( + SELECT c2.message + FROM bounded_candidates c2 + WHERE c2.ai_project = g.ai_project + AND c2.ai_tool = g.ai_tool + AND c2.ai_session_id = g.ai_session_id + AND c2.hostname = g.hostname + ORDER BY c2.timestamp DESC + LIMIT 1 + ) AS best_snippet, + totals.total_candidates, + totals.raw_candidate_count + FROM selected g + LEFT JOIN ai_session_rollup rollup + ON rollup.ai_project = g.ai_project + AND rollup.ai_tool = g.ai_tool + AND rollup.ai_session_id = g.ai_session_id + AND rollup.hostname = g.hostname + LEFT JOIN tail_stats tail + ON tail.ai_project = g.ai_project + AND tail.ai_tool = g.ai_tool + AND tail.ai_session_id = g.ai_session_id + AND tail.hostname = g.hostname + CROSS JOIN totals + -- Rank by match recency (latest matching row), not the full-session + -- last_seen computed above: a session with an old match but newer + -- non-matching activity must not jump ahead of more recent matches. + -- This mirrors `selected`'s own `latest_match DESC` pre-selection and + -- the pre-refactor match-recency ordering. + ORDER BY g.latest_match DESC + LIMIT {limit}", + CANDIDATE_CAP + 1 + ); + (sql, query_params.bindings) +} + +pub fn search_ai_anchors(pool: &DbPool, params: &AiCorrelateParams) -> Result> { + let conn = pool.get()?; + let limit = params.limit.unwrap_or(10).clamp(1, 50); + let mut bindings: Vec = vec![]; + let mut idx = 1usize; + let has_query = if let Some(query) = ¶ms.ai_query { + validate_fts_query(query)?; + bindings.push(rusqlite::types::Value::Text(query.clone())); + idx += 1; + true + } else { + false + }; + + // Shared filter clause — applied inside the FTS candidate CTE (so the cap + // operates on already-filtered rows) or directly on the plain scan. + let mut filters = String::new(); + push_required_ai_filters(&mut filters, "l"); + if let Some(project) = ¶ms.ai_project { + filters.push_str(&format!(" AND l.ai_project = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(project.clone())); + idx += 1; + } + if let Some(tool) = ¶ms.ai_tool { + filters.push_str(&format!(" AND l.ai_tool = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(tool.clone())); + idx += 1; + } + if let Some(session_id) = ¶ms.ai_session_id { + filters.push_str(&format!(" AND l.ai_session_id = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(session_id.clone())); + idx += 1; + } + if let Some(from) = ¶ms.since { + filters.push_str(&format!(" AND l.timestamp >= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(from.clone())); + idx += 1; + } + if let Some(to) = ¶ms.until { + filters.push_str(&format!(" AND l.timestamp <= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(to.clone())); + } + + let sql = if has_query { + // Capped-candidate plan: the previous `FROM logs_fts JOIN logs ... + // ORDER BY timestamp` shape fetched and sorted the ENTIRE FTS match + // set in a temp b-tree before LIMIT applied — the exact pathology + // `search_logs` fixed with its candidate cap, never applied to this + // entry point (full-review PM2). + format!( + "WITH fts_candidates(id, ts) AS MATERIALIZED ( + SELECT l.id, l.timestamp + FROM logs_fts + JOIN logs l ON l.id = logs_fts.rowid + WHERE logs_fts MATCH ?1{filters} + ORDER BY logs_fts.rowid DESC LIMIT {SEARCH_FTS_CANDIDATE_CAP} + ) + SELECT {FTS_SELECT_COLS} + FROM fts_candidates c + JOIN logs l ON l.id = c.id + ORDER BY c.ts DESC, l.id DESC LIMIT {}", + limit + 1 + ) + } else { + format!( + "SELECT {FTS_SELECT_COLS} + FROM logs l + WHERE 1=1{filters} + ORDER BY l.timestamp DESC, l.id DESC LIMIT {}", + limit + 1 + ) + }; + let mut stmt = conn.prepare(&sql)?; + let rows = stmt.query_map(rusqlite::params_from_iter(bindings.iter()), map_row)?; + Ok(rows.collect::>>()?) +} + +pub fn search_ai_related_logs( + pool: &DbPool, + params: &AiRelatedLogsParams, +) -> Result> { + if params.windows.is_empty() { + return Ok(Vec::new()); + } + if let Some(query) = ¶ms.query { + validate_fts_query(query)?; + } + + let conn = pool.get()?; + let limit = params.limit_per_anchor.clamp(1, 200) as usize; + + // Reject duplicate anchors up front (preserved contract). + { + let mut seen = std::collections::HashSet::with_capacity(params.windows.len()); + for window in ¶ms.windows { + if !seen.insert(window.anchor_index) { + anyhow::bail!( + "duplicate anchor_index {} in AiRelatedLogsParams windows", + window.anchor_index + ); + } + } + } + + // One bounded, index-served query PER ANCHOR instead of the previous + // single windowed CTE that ROW_NUMBER()-ranked EVERY log row inside every + // window before applying the per-anchor limit — SQLite cannot push the + // rank limit into the window scan, so a 10-minute window during a log + // storm sorted 100K+ rows per anchor (full-review PM1). Each per-anchor + // query remains bounded to one anchor window and returns only n+1 rows. + // Ranking by distance to the anchor makes the result evidentially useful: + // a busy window no longer discards the rows nearest the AI action merely + // because unrelated traffic arrived near the end of the window. Anchor + // counts are small (bounded by the anchor search limit) and the statement + // is compiled once via prepare_cached. + // + // Placeholders: FTS path ?1=query ?2=from ?3=to ?4=anchor, filters from ?5; + // plain path ?1=from ?2=to ?3=anchor, filters from ?4. + let first_filter_idx = if params.query.is_some() { 5 } else { 4 }; + let mut filter_sql = String::new(); + let mut sql_params = SqlParams::new(first_filter_idx); + let search_params = SearchParams { + query: None, + host: params.host.clone(), + source: params.source.clone(), + source_ip_prefix: None, + severity: None, + severity_in: Some(params.severity_in.clone()), + app: params.app.clone(), + facility: None, + exclude_facility: None, + process_id: None, + since: None, + until: None, + received_since: None, + received_until: None, + limit: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + event_action: None, + exclude_ai: true, + }; + append_filters( + &mut filter_sql, + &mut sql_params.bindings, + &mut sql_params.next_idx, + &search_params, + ); + + let sql = if params.query.is_some() { + format!( + "SELECT {FTS_SELECT_COLS} + FROM logs_fts + JOIN logs l ON l.id = logs_fts.rowid + WHERE logs_fts MATCH ?1 + AND l.timestamp >= ?2 AND l.timestamp <= ?3{filter_sql} + ORDER BY ABS(unixepoch(l.timestamp) - unixepoch(?4)), l.timestamp DESC, l.id DESC LIMIT {}", + limit + 1 + ) + } else { + format!( + "SELECT {FTS_SELECT_COLS} + FROM logs l + WHERE l.timestamp >= ?1 AND l.timestamp <= ?2{filter_sql} + ORDER BY ABS(unixepoch(l.timestamp) - unixepoch(?3)), l.timestamp DESC, l.id DESC LIMIT {}", + limit + 1 + ) + }; + + let mut grouped = Vec::with_capacity(params.windows.len()); + let mut stmt = conn.prepare_cached(&sql)?; + for window in ¶ms.windows { + let mut bindings: Vec = + Vec::with_capacity(3 + sql_params.bindings.len()); + if let Some(query) = ¶ms.query { + bindings.push(rusqlite::types::Value::Text(query.clone())); + } + bindings.push(rusqlite::types::Value::Text(window.window_from.clone())); + bindings.push(rusqlite::types::Value::Text(window.window_to.clone())); + bindings.push(rusqlite::types::Value::Text(window.anchor_time.clone())); + bindings.extend(sql_params.bindings.iter().cloned()); + + let mut logs = Vec::new(); + let mut truncated = false; + let mut rows = stmt.query(rusqlite::params_from_iter(bindings.iter()))?; + let mut row_count = 0usize; + while let Some(row) = rows.next()? { + row_count += 1; + if row_count > limit { + truncated = true; + } else { + logs.push(map_row(row)?); + } + } + grouped.push(AiRelatedLogsForAnchor { + anchor_index: window.anchor_index, + logs, + truncated, + }); + } + + Ok(grouped) +} + +/// Push each value as a bound `Text` parameter and return a `?, ?, …` +/// placeholder list of matching arity for an `IN (...)` clause. +pub(super) fn bind_in_list( + bindings: &mut Vec, + values: &[String], +) -> String { + let start = bindings.len(); + for v in values { + bindings.push(rusqlite::types::Value::Text(v.clone())); + } + vec!["?"; bindings.len() - start].join(", ") +} + +/// Controls which walked entities may contribute host-wide +/// (`l.hostname IN (…)`) predicates to the graph log fan-out. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum HostFanoutScope { + /// Any reached `host`/`container` entity adds its hostname. Session + /// correlation uses this deliberately: following `ai_session → host` + /// edges to the host's logs is the point of the query. + WalkReached, + /// Only `host` entities that were seeds themselves (exact topic match on + /// the host) add hostnames. Hosts reached transitively from app/container + /// seeds never drive host-wide log inclusion — topic correlation's + /// no-silent-fan-out guarantee. + SeedHostsOnly, +} + +/// Graph-anchored log fan-out: traverse the investigation graph outward from a +/// set of seed entities, then return the logs emitted by every related entity +/// within `[since, until]`. +/// +/// This is the **graph-first** query order (Performance Oracle research): +/// resolve ~tens of related entity keys via the indexed recursive CTE, map them +/// to `hostname` / `ai_project` / `ai_session_id` filters, and let the +/// `(hostname, app_name, timestamp)` covering index (migration 32) drive the +/// log scan — 10-100× fewer rows than an FTS-first scan over a common term. +/// +/// Entity → log-column mapping: +/// - `host` → `hostname` (subject to `host_fanout_scope`) +/// - `container` (`docker_host:…` keys) → leading `hostname` (only under +/// the `WalkReached` host-fanout mode) +/// - `ai_project` → `ai_project` +/// - `ai_session` (`project:tool:session` keys) → trailing `ai_session_id` +/// +/// `max_depth` is clamped to `[1, GRAPH_WALK_MAX_DEPTH]` and `limit` to +/// `[1, 1000]`. Returns raw `LogEntry` rows for the service layer to shape. +#[allow(clippy::too_many_arguments)] +pub fn search_logs_from_graph_related_entities( + pool: &DbPool, + entity_canonical_keys: &[String], + max_depth: u8, + since: Option<&str>, + until: Option<&str>, + source_kinds: Option<&[SourceKind]>, + limit: usize, + host_fanout_scope: HostFanoutScope, +) -> Result> { + if entity_canonical_keys.is_empty() { + return Ok(Vec::new()); + } + let limit = limit.clamp(1, 1000); + let conn = pool.get()?; + + // 1. Graph-first: traverse to the related entity set (seeds + N hops). + let entities = super::graph::graph_walk_n_hops(&conn, entity_canonical_keys, max_depth)?; + if entities.is_empty() { + return Ok(Vec::new()); + } + let seed_set: std::collections::HashSet<&str> = + entity_canonical_keys.iter().map(String::as_str).collect(); + + // 2. Map related entities to indexed log-column filters. + // + // Hard break (entity_resolution_v2): `service_instance` entities do NOT + // map to host-wide log filters here — service-scoped logs come from + // `search_logs_for_service_instances` predicates instead, so a service + // topic never silently expands to every log on its host. + let mut hostnames: Vec = Vec::new(); + let mut ai_projects: Vec = Vec::new(); + let mut ai_sessions: Vec = Vec::new(); + for entity in &entities { + match entity.entity_type.as_str() { + super::graph::ENTITY_TYPE_HOST => { + if host_fanout_scope == HostFanoutScope::WalkReached + || seed_set.contains(entity.canonical_key.as_str()) + { + hostnames.push(entity.canonical_key.clone()); + } + } + super::graph::ENTITY_TYPE_CONTAINER + if host_fanout_scope == HostFanoutScope::WalkReached => + { + // `docker_host:container_id` — the leading segment is the + // host the workload runs on. + if let Some(host) = + super::entity_resolution::container_key_host(&entity.canonical_key) + { + hostnames.push(host.to_string()); + } + } + super::graph::ENTITY_TYPE_AI_PROJECT => ai_projects.push(entity.canonical_key.clone()), + super::graph::ENTITY_TYPE_AI_SESSION => { + // `project:tool:session` — the session id is the 3rd segment. + if let Some(session) = entity.canonical_key.splitn(3, ':').nth(2) + && !session.is_empty() + { + ai_sessions.push(session.to_string()); + } + } + _ => {} + } + } + hostnames.sort(); + hostnames.dedup(); + ai_projects.sort(); + ai_projects.dedup(); + ai_sessions.sort(); + ai_sessions.dedup(); + + if hostnames.is_empty() && ai_projects.is_empty() && ai_sessions.is_empty() { + return Ok(Vec::new()); + } + + // 3. Build the graph-first log fan-out. hostname IN (...) leads on the + // covering index; ai_project / ai_session_id are OR-ed in via their own + // partial indexes. + let mut bindings: Vec = Vec::new(); + let mut entity_clauses: Vec = Vec::new(); + if !hostnames.is_empty() { + let ph = bind_in_list(&mut bindings, &hostnames); + entity_clauses.push(format!("l.hostname IN ({ph})")); + } + if !ai_projects.is_empty() { + let ph = bind_in_list(&mut bindings, &ai_projects); + entity_clauses.push(format!("l.ai_project IN ({ph})")); + } + if !ai_sessions.is_empty() { + let ph = bind_in_list(&mut bindings, &ai_sessions); + entity_clauses.push(format!("l.ai_session_id IN ({ph})")); + } + + let mut where_sql = format!("({})", entity_clauses.join(" OR ")); + if let Some(since) = since { + where_sql.push_str(" AND l.timestamp >= ?"); + bindings.push(rusqlite::types::Value::Text(since.to_string())); + } + if let Some(until) = until { + where_sql.push_str(" AND l.timestamp <= ?"); + bindings.push(rusqlite::types::Value::Text(until.to_string())); + } + if let Some(kinds) = source_kinds + && !kinds.is_empty() + { + let kind_strs: Vec = kinds.iter().map(|k| k.as_str().to_string()).collect(); + let ph = bind_in_list(&mut bindings, &kind_strs); + where_sql.push_str(&format!( + " AND json_extract(l.metadata_json, '$.source_kind') IN ({ph})" + )); + } + bindings.push(rusqlite::types::Value::Integer(limit as i64)); + + let sql = format!( + "SELECT {FTS_SELECT_COLS} + FROM logs l + WHERE {where_sql} + ORDER BY l.timestamp DESC, l.id DESC + LIMIT ?" + ); + let mut stmt = conn.prepare(&sql)?; + let logs = stmt + .query_map(rusqlite::params_from_iter(bindings.iter()), map_row)? + .collect::>>()?; + Ok(logs) +} + +/// Graph-anchored, session-scoped correlation inputs for `ai_correlate`. +/// +/// Resolves the session's time bounds from its log rows, finds the `ai_session` +/// graph entity (key ends in `:{session_id}`), traverses the graph (depth 2) to +/// discover related hosts/containers/services, then fans logs out across all +/// source kinds within the session window via +/// `search_logs_from_graph_related_entities`. The fan-out's `ai_session_id` +/// filter pulls in the agent-command lane (Claude's bash calls) and the +/// hostname filter pulls in the shell-history / syslog lanes on the discovered +/// hosts. +/// +/// Falls back to a plain `ai_session_id`-filtered query (`used_graph = false`) +/// when the graph has no entity for the session yet. Returns empty bounds when +/// the session has no rows at all. +/// +/// Deliberate scoping: session correlation uses host/container mapping only +/// and intentionally does not fan out via `service_instance` predicates. +pub fn correlate_session_graph( + pool: &DbPool, + session_id: &str, + limit: usize, +) -> Result { + let limit = limit.clamp(1, 1000); + let conn = pool.get()?; + + // Session window = [MIN, MAX] timestamp over the session's rows. + let bounds: Option<(String, String)> = conn + .query_row( + "SELECT MIN(timestamp), MAX(timestamp) FROM logs WHERE ai_session_id = ?1", + [session_id], + |row| { + Ok(( + row.get::<_, Option>(0)?, + row.get::<_, Option>(1)?, + )) + }, + ) + .optional()? + .and_then(|(min, max)| match (min, max) { + (Some(start), Some(end)) => Some((start, end)), + _ => None, + }); + + let Some((start, end)) = bounds else { + return Ok(SessionGraphInputs::default()); + }; + + // Find the ai_session graph entity (canonical_key `project:tool:session`). + let session_keys: Vec = { + let mut stmt = conn.prepare( + "SELECT canonical_key FROM graph_entities + WHERE entity_type = ?1 AND canonical_key LIKE '%:' || ?2", + )?; + stmt.query_map( + rusqlite::params![super::graph::ENTITY_TYPE_AI_SESSION, session_id], + |row| row.get::<_, String>(0), + )? + .collect::>>()? + }; + let used_graph = !session_keys.is_empty(); + + // Discover related entities/hosts by traversing from the session entity. + let mut discovered_entities: Vec = Vec::new(); + let mut discovered_hosts: Vec = Vec::new(); + if used_graph { + for entity in super::graph::graph_walk_n_hops(&conn, &session_keys, 2)? { + match entity.entity_type.as_str() { + super::graph::ENTITY_TYPE_HOST => { + discovered_hosts.push(entity.canonical_key.clone()) + } + super::graph::ENTITY_TYPE_CONTAINER => { + if let Some(host) = + super::entity_resolution::container_key_host(&entity.canonical_key) + { + discovered_hosts.push(host.to_string()); + } + } + super::graph::ENTITY_TYPE_SERVICE_INSTANCE => { + if let Some((host, _)) = + super::entity_resolution::split_service_instance_key(&entity.canonical_key) + { + discovered_hosts.push(host.to_string()); + } + } + _ => {} + } + discovered_entities.push(entity.canonical_key); + } + discovered_hosts.sort(); + discovered_hosts.dedup(); + discovered_entities.sort(); + discovered_entities.dedup(); + } + + // Fan logs out across all source kinds within the session window. + let logs = if used_graph { + drop(conn); + search_logs_from_graph_related_entities( + pool, + &session_keys, + 2, + Some(&start), + Some(&end), + None, + limit, + HostFanoutScope::WalkReached, + )? + } else { + // Fallback: the graph hasn't projected this session yet — return its own + // rows (transcript + agent-command lanes) by exact session id. + let mut stmt = conn.prepare(&format!( + "SELECT {FTS_SELECT_COLS} + FROM logs l + WHERE l.ai_session_id = ?1 + ORDER BY l.timestamp DESC, l.id DESC + LIMIT ?2" + ))?; + stmt.query_map(rusqlite::params![session_id, limit as i64], map_row)? + .collect::>>()? + }; + + Ok(SessionGraphInputs { + bounds: Some((start, end)), + discovered_hosts, + discovered_entities, + used_graph, + logs, + }) +} + +/// Topic-anchored universal correlation: resolve a set of (lowercased) topic +/// terms to graph entities, expand the graph `max_depth` hops, and fan logs out +/// across all source kinds within `[since, until]` via +/// `search_logs_from_graph_related_entities` (graph-first order). +/// +/// Returns the resolved seed entities, the graph expansion (reached entities +/// that were not seeds), the discovered hosts, and the correlated logs. Empty +/// when no term resolves to an entity. +#[allow(clippy::too_many_arguments)] +pub fn topic_correlate_inputs( + pool: &DbPool, + terms: &[String], + max_depth: u8, + since: Option<&str>, + until: Option<&str>, + source_kinds: Option<&[SourceKind]>, + limit: usize, +) -> Result { + if terms.is_empty() { + return Ok(TopicGraphInputs::default()); + } + let conn = pool.get()?; + let mut resolved = queries_service_instances::resolve_topic_entities(&conn, terms)?; + if resolved.is_empty() { + drop(conn); + return topic_correlate_ai_project_fallback(pool, terms, since, until, source_kinds, limit); + } + + // Partition seeds. Only `resolved` (exact/alias) identities drive log + // fan-out; weak prefix/label candidates stay visible as `ambiguous`. + // Service identity seeds resolve through service-instance predicates: + // an exact `logical_service` expands to its `instance_of` instances. + let mut instance_keys: Vec = Vec::new(); + let mut logical_keys: Vec = Vec::new(); + let mut generic_seeds: Vec = Vec::new(); + for entity in &resolved { + if entity.resolver_status != ResolverStatus::Resolved { + continue; + } + match entity.entity_type.as_str() { + super::graph::ENTITY_TYPE_LOGICAL_SERVICE => { + logical_keys.push(entity.canonical_key.clone()) + } + super::graph::ENTITY_TYPE_SERVICE_INSTANCE => { + instance_keys.push(entity.canonical_key.clone()) + } + _ => generic_seeds.push(entity.canonical_key.clone()), + } + } + if !logical_keys.is_empty() { + let linked = + queries_service_instances::service_instances_of_logical_services(&conn, &logical_keys)?; + // A resolved logical service with ZERO instance_of instances means + // the projection is stale or unbuilt (e.g. right after migration 41 + // before `cortex graph rebuild`). Mark it degraded so the empty + // service timeline is explained rather than silent. Canonical + // instance keys are `host/` by the resolver grammar. + let covered: std::collections::HashSet = linked + .iter() + .filter_map(|key| { + super::entity_resolution::split_service_instance_key(key) + .map(|(_, service)| service.to_string()) + }) + .collect(); + for entity in resolved.iter_mut() { + if entity.entity_type == super::graph::ENTITY_TYPE_LOGICAL_SERVICE + && entity.resolver_status == ResolverStatus::Resolved + && !covered.contains(entity.canonical_key.as_str()) + { + entity.resolver_status = ResolverStatus::Degraded; + } + } + instance_keys.extend(linked); + } + instance_keys.sort(); + instance_keys.dedup(); + generic_seeds.sort(); + generic_seeds.dedup(); + + // Graph expansion + host discovery. Service seeds use the bounded + // service-topic walk (proof relationships only); generic seeds keep the + // general walk. + let mut walk_entities = Vec::new(); + let mut service_seeds: Vec = logical_keys.clone(); + service_seeds.extend(instance_keys.iter().cloned()); + service_seeds.sort(); + service_seeds.dedup(); + let mut graph_walk_truncated = false; + if !service_seeds.is_empty() { + let (entities, truncated) = super::graph_resolver_projection::graph_walk_service_topic( + &conn, + &service_seeds, + max_depth, + )?; + walk_entities.extend(entities); + graph_walk_truncated |= truncated; + } + if !generic_seeds.is_empty() { + walk_entities.extend(super::graph::graph_walk_n_hops( + &conn, + &generic_seeds, + max_depth, + )?); + } + let seed_set: std::collections::HashSet<&str> = service_seeds + .iter() + .chain(generic_seeds.iter()) + .map(String::as_str) + .collect(); + let mut expansion: Vec<(String, String)> = Vec::new(); + let mut discovered_hosts: Vec = Vec::new(); + for entity in walk_entities { + match entity.entity_type.as_str() { + super::graph::ENTITY_TYPE_HOST => discovered_hosts.push(entity.canonical_key.clone()), + super::graph::ENTITY_TYPE_CONTAINER => { + if let Some(host) = + super::entity_resolution::container_key_host(&entity.canonical_key) + { + discovered_hosts.push(host.to_string()); + } + } + super::graph::ENTITY_TYPE_SERVICE_INSTANCE => { + if let Some((host, _)) = + super::entity_resolution::split_service_instance_key(&entity.canonical_key) + { + discovered_hosts.push(host.to_string()); + } + } + _ => {} + } + if !seed_set.contains(entity.canonical_key.as_str()) { + expansion.push((entity.entity_type, entity.canonical_key)); + } + } + discovered_hosts.sort(); + discovered_hosts.dedup(); + expansion.sort(); + expansion.dedup(); + drop(conn); + + // Log fan-out: service instances use service-scoped predicates; generic + // seeds use the graph-related fan-out. Never both for the same rows — + // results merge newest-first under the shared limit. + let mut logs: Vec = Vec::new(); + if !instance_keys.is_empty() { + logs.extend( + queries_service_instances::search_logs_for_service_instances( + pool, + &instance_keys, + since, + until, + source_kinds, + limit, + )?, + ); + } + if !generic_seeds.is_empty() { + // SeedHostsOnly: a host reached transitively from an app/container + // seed (e.g. app:plex —emitted_by→ host:nashost) must never drive + // host-wide `l.hostname IN (…)` inclusion labelled `resolved`. Only + // hosts that were exact topic matches themselves fan out. + logs.extend( + search_logs_from_graph_related_entities( + pool, + &generic_seeds, + max_depth, + since, + until, + source_kinds, + limit, + HostFanoutScope::SeedHostsOnly, + )? + .into_iter() + .map(|entry| GraphRelatedLogEntry { + entry, + inclusion_reason: INCLUSION_GRAPH_RELATED.to_string(), + resolver_status: ResolverStatus::Resolved, + fallback_kind: None, + }), + ); + } + + // Explicit degraded host-context fallback: a service topic whose + // instance predicates matched no rows falls back to the instances' host + // context, annotated (`explicit_degraded_host_context`) — never silent. + if logs.is_empty() && !instance_keys.is_empty() && generic_seeds.is_empty() { + let hosts: Vec = instance_keys + .iter() + .filter_map(|key| { + let split = super::entity_resolution::split_service_instance_key(key); + if split.is_none() { + tracing::debug!( + key = %key, + "discarding non-canonical service_instance key in host-context fallback" + ); + } + split.map(|(host, _)| host.to_string()) + }) + .collect(); + if !hosts.is_empty() { + logs.extend( + queries_service_instances::search_logs_by_hostnames( + pool, + &hosts, + since, + until, + source_kinds, + limit, + )? + .into_iter() + .map(|entry| GraphRelatedLogEntry { + entry, + inclusion_reason: INCLUSION_HOST_CONTEXT.to_string(), + resolver_status: ResolverStatus::Degraded, + fallback_kind: Some(FALLBACK_EXPLICIT_DEGRADED_HOST_CONTEXT.to_string()), + }), + ); + } + } + + logs.sort_by(|a, b| { + b.entry + .timestamp + .cmp(&a.entry.timestamp) + .then_with(|| b.entry.id.cmp(&a.entry.id)) + }); + logs.dedup_by_key(|row| row.entry.id); + logs.truncate(limit); + + Ok(TopicGraphInputs { + resolved, + expansion, + discovered_hosts, + logs, + graph_walk_truncated, + }) +} + +fn topic_correlate_ai_project_fallback( + pool: &DbPool, + terms: &[String], + since: Option<&str>, + until: Option<&str>, + source_kinds: Option<&[SourceKind]>, + limit: usize, +) -> Result { + let conn = pool.get()?; + let predicates = terms + .iter() + .map(|_| "(lower(ai_project) = ? OR lower(ai_project) LIKE '%/' || ?)") + .collect::>() + .join(" OR "); + let mut sql = format!( + "SELECT DISTINCT ai_project FROM logs WHERE ai_project IS NOT NULL AND ({predicates})" + ); + let mut bindings = Vec::with_capacity(terms.len() * 2 + 2); + for term in terms { + bindings.push(rusqlite::types::Value::Text(term.clone())); + bindings.push(rusqlite::types::Value::Text(term.clone())); + } + if let Some(value) = since { + sql.push_str(" AND timestamp >= ?"); + bindings.push(rusqlite::types::Value::Text(value.to_string())); + } + if let Some(value) = until { + sql.push_str(" AND timestamp <= ?"); + bindings.push(rusqlite::types::Value::Text(value.to_string())); + } + sql.push_str(" ORDER BY ai_project LIMIT 32"); + + let projects = conn + .prepare(&sql)? + .query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + row.get::<_, String>(0) + })? + .collect::>>()?; + drop(conn); + + let allowed_source_kinds = source_kinds.map(|kinds| { + kinds + .iter() + .map(|kind| kind.as_str()) + .collect::>() + }); + let mut logs = Vec::new(); + let mut discovered_hosts = Vec::new(); + let mut resolved = Vec::new(); + for project in projects { + let key = project + .rsplit('/') + .next() + .unwrap_or(&project) + .to_ascii_lowercase(); + resolved.push(ResolvedTopicEntity { + entity_type: super::graph::ENTITY_TYPE_AI_PROJECT.to_string(), + canonical_key: key, + match_kind: "exact", + resolver_status: ResolverStatus::Degraded, + }); + let rows = search_logs( + pool, + &SearchParams { + ai_project: Some(project), + since: since.map(str::to_string), + until: until.map(str::to_string), + limit: Some(limit.min(1000) as u32), + ..Default::default() + }, + )?; + for entry in rows { + if let Some(allowed) = &allowed_source_kinds { + let source_kind = entry + .metadata_json + .as_deref() + .and_then(|json| serde_json::from_str::(json).ok()) + .and_then(|value| value.get("source_kind")?.as_str().map(str::to_string)); + if !source_kind + .as_deref() + .is_some_and(|kind| allowed.contains(kind)) + { + continue; + } + } + discovered_hosts.push(entry.hostname.clone()); + logs.push(GraphRelatedLogEntry { + entry, + inclusion_reason: "direct_source_identity".to_string(), + resolver_status: ResolverStatus::Degraded, + fallback_kind: Some("direct_source_identity".to_string()), + }); + } + } + resolved.sort_by(|a, b| a.canonical_key.cmp(&b.canonical_key)); + resolved.dedup_by(|a, b| a.canonical_key == b.canonical_key); + discovered_hosts.sort(); + discovered_hosts.dedup(); + logs.sort_by(|a, b| b.entry.timestamp.cmp(&a.entry.timestamp)); + logs.dedup_by_key(|row| row.entry.id); + logs.truncate(limit); + + Ok(TopicGraphInputs { + resolved, + discovered_hosts, + logs, + ..Default::default() + }) +} + +const DEFAULT_AI_ABUSE_TERMS: &[&str] = &[ + "asshole", "bastard", "bitch", "biznitch", "bullshit", "crap", "damn", "dick", "fuck", + "fucked", "fucker", "fucking", "hell", "piss", "shit", "shitty", +]; + +pub fn search_ai_abuse(pool: &DbPool, params: &AiAbuseParams) -> Result { + let limit = params.limit.unwrap_or(20).clamp(1, 100) as usize; + let before = params.before.unwrap_or(2).min(20); + let after = params.after.unwrap_or(2).min(20); + let terms = normalized_abuse_terms(¶ms.terms); + if terms.len() > 16 { + anyhow::bail!("Too many abuse terms ({}); maximum is 16", terms.len()); + } + let conn = pool.get()?; + const CANDIDATE_CAP: usize = 10_000; + + let mut sql = String::from( + "WITH candidates(id) AS MATERIALIZED ( + SELECT l.id + FROM logs_fts + JOIN logs l ON l.id = logs_fts.rowid + WHERE logs_fts MATCH ?1 + AND l.ai_project IS NOT NULL AND l.ai_project != '' + AND l.ai_tool IS NOT NULL AND l.ai_tool != '' + AND l.ai_session_id IS NOT NULL AND l.ai_session_id != ''", + ); + let mut bindings = vec![rusqlite::types::Value::Text(abuse_fts_query(&terms))]; + let mut idx = 2usize; + + if let Some(project) = ¶ms.ai_project { + sql.push_str(&format!(" AND l.ai_project = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(project.clone())); + idx += 1; + } + if let Some(tool) = ¶ms.ai_tool { + sql.push_str(&format!(" AND l.ai_tool = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(tool.clone())); + idx += 1; + } + if let Some(from) = ¶ms.since { + sql.push_str(&format!(" AND l.timestamp >= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(from.clone())); + idx += 1; + } + if let Some(to) = ¶ms.until { + sql.push_str(&format!(" AND l.timestamp <= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(to.clone())); + } + sql.push_str(&format!( + " ORDER BY logs_fts.rowid DESC LIMIT {} + ) + SELECT l.id, l.timestamp, l.hostname, l.facility, l.severity, + l.app_name, l.process_id, l.message, l.received_at, l.source_ip, + l.ai_tool, l.ai_project, l.ai_session_id, l.ai_transcript_path, l.metadata_json + FROM candidates c + JOIN logs l ON l.id = c.id + ORDER BY l.timestamp DESC, l.id DESC", + CANDIDATE_CAP + 1 + )); + + let mut stmt = conn.prepare(&sql)?; + let candidate_rows = stmt + .query_map(rusqlite::params_from_iter(bindings.iter()), map_row)? + .collect::>>()?; + + let candidate_window_truncated = candidate_rows.len() > CANDIDATE_CAP; + let mut matches = Vec::new(); + let mut result_limit_truncated = false; + for entry in candidate_rows.iter().take(CANDIDATE_CAP) { + if let Some(term) = first_abuse_term(&entry.message, &terms) { + if matches.len() == limit { + result_limit_truncated = true; + break; + } + let (before_rows, after_rows) = ai_session_context(&conn, entry, before, after)?; + matches.push(AiAbuseMatch { + term, + entry: entry.clone(), + before: before_rows, + after: after_rows, + }); + } + } + + Ok(AiAbuseResult { + terms, + candidate_rows: candidate_rows.len().min(CANDIDATE_CAP), + candidate_cap: CANDIDATE_CAP, + candidate_window_truncated, + truncated: candidate_window_truncated || result_limit_truncated, + matches, + }) +} + +pub fn list_ai_tools(pool: &DbPool, params: &ListAiToolsParams) -> Result { + let conn = pool.get()?; + const LIMIT: usize = 100; + let mut sql = String::from( + "SELECT ai_tool, + COUNT(*) AS event_count, + COUNT(DISTINCT ai_session_id) AS session_count, + MIN(timestamp) AS first_seen, + MAX(timestamp) AS last_seen + FROM logs + WHERE ai_tool IS NOT NULL + AND ai_tool != ''", + ); + let mut bindings: Vec = vec![]; + let mut idx = 1usize; + + if let Some(project) = ¶ms.ai_project { + sql.push_str(&format!(" AND ai_project = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(project.clone())); + idx += 1; + } + if let Some(from) = ¶ms.since { + sql.push_str(&format!(" AND timestamp >= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(from.clone())); + idx += 1; + } + if let Some(to) = ¶ms.until { + sql.push_str(&format!(" AND timestamp <= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(to.clone())); + } + sql.push_str(&format!( + " GROUP BY ai_tool ORDER BY event_count DESC, ai_tool ASC LIMIT {}", + LIMIT + 1 + )); + + let mut stmt = conn.prepare(&sql)?; + let mut tools = stmt + .query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + Ok(AiToolInventoryEntry { + tool: row.get(0)?, + event_count: row.get(1)?, + session_count: row.get(2)?, + first_seen: row.get(3)?, + last_seen: row.get(4)?, + }) + })? + .collect::>>()?; + let truncated = truncate_to_limit(&mut tools, LIMIT); + Ok(ListAiToolsResult { + total_tools: tools.len(), + truncated, + tools, + }) +} + +pub fn list_ai_projects( + pool: &DbPool, + params: &ListAiProjectsParams, +) -> Result { + let conn = pool.get()?; + const LIMIT: usize = 200; + let mut sql = String::from( + "SELECT ai_project, + GROUP_CONCAT(DISTINCT ai_tool) AS tools, + COUNT(*) AS event_count, + COUNT(DISTINCT ai_session_id) AS session_count, + MIN(timestamp) AS first_seen, + MAX(timestamp) AS last_seen + FROM logs + WHERE ai_project IS NOT NULL + AND ai_project != ''", + ); + let mut bindings: Vec = vec![]; + let mut idx = 1usize; + + if let Some(tool) = ¶ms.ai_tool { + sql.push_str(&format!(" AND ai_tool = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(tool.clone())); + idx += 1; + } + if let Some(from) = ¶ms.since { + sql.push_str(&format!(" AND timestamp >= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(from.clone())); + idx += 1; + } + if let Some(to) = ¶ms.until { + sql.push_str(&format!(" AND timestamp <= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(to.clone())); + } + sql.push_str(&format!( + " GROUP BY ai_project ORDER BY event_count DESC, ai_project ASC LIMIT {}", + LIMIT + 1 + )); + + let mut stmt = conn.prepare(&sql)?; + let mut projects = stmt + .query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + let tools = row + .get::<_, Option>(1)? + .unwrap_or_default() + .split(',') + .filter(|value| !value.is_empty()) + .map(ToString::to_string) + .collect(); + Ok(AiProjectInventoryEntry { + project: row.get(0)?, + tools, + event_count: row.get(2)?, + session_count: row.get(3)?, + first_seen: row.get(4)?, + last_seen: row.get(5)?, + }) + })? + .collect::>>()?; + let truncated = truncate_to_limit(&mut projects, LIMIT); + Ok(ListAiProjectsResult { + total_projects: projects.len(), + truncated, + projects, + }) +} + +fn truncate_to_limit(values: &mut Vec, limit: usize) -> bool { + let truncated = values.len() > limit; + values.truncate(limit); + truncated +} + +pub fn search_ai_incidents(pool: &DbPool, params: &AiIncidentParams) -> Result { + use std::collections::HashMap; + + let limit = params.limit.unwrap_or(20).clamp(1, 100) as usize; + let window_secs = i64::from(params.window_minutes.unwrap_or(10).clamp(1, 120)) * 60; + let terms = normalized_abuse_terms(¶ms.terms); + const CANDIDATE_CAP: usize = 10_000; + + let conn = pool.get()?; + let (sql, bindings) = ai_incident_anchor_sql(params, &terms, CANDIDATE_CAP); + + // Fetch candidate abuse anchor rows (same FTS path as search_ai_abuse, + // no per-hit context needed here). + struct AnchorRow { + id: i64, + timestamp: String, + hostname: String, + tool: String, + project: String, + session_id: String, + message: String, + } + + let mut stmt = conn.prepare(&sql)?; + let candidate_rows: Vec = stmt + .query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + Ok(AnchorRow { + id: row.get(0)?, + timestamp: row.get(1)?, + hostname: row.get(2)?, + tool: row.get(3)?, + project: row.get(4)?, + session_id: row.get(5)?, + message: row.get(6)?, + }) + })? + .collect::>>()?; + + let candidate_window_truncated = candidate_rows.len() > CANDIDATE_CAP; + let raw_candidate_count = candidate_rows.len(); + + // Group by (project, tool, session_id, hostname) + window-minute buckets. + // Key: (project, tool, session_id, hostname, window_bucket) + // window_bucket = unix_secs / window_secs * window_secs (floor to window boundary) + type GroupKey = (String, String, String, String, i64); + let mut groups: HashMap> = HashMap::new(); + + for row in candidate_rows.iter().take(CANDIDATE_CAP) { + // Parse timestamp to unix seconds for bucketing. + let bucket = chrono::DateTime::parse_from_rfc3339(&row.timestamp) + .map(|dt| { + let secs = dt.timestamp(); + (secs / window_secs) * window_secs + }) + .unwrap_or(0); + let key = ( + row.project.clone(), + row.tool.clone(), + row.session_id.clone(), + row.hostname.clone(), + bucket, + ); + groups.entry(key).or_default().push(row); + } + + // Build incidents from groups. + let mut incidents: Vec = groups + .into_iter() + .map( + |((project, tool, session_id, hostname, _bucket), anchors)| { + let abuse_count = anchors.len(); + let first_seen = anchors + .first() + .map(|r| r.timestamp.clone()) + .unwrap_or_default(); + let last_seen = anchors + .last() + .map(|r| r.timestamp.clone()) + .unwrap_or_default(); + + // duration in seconds + let duration_secs = { + let t0 = chrono::DateTime::parse_from_rfc3339(&first_seen) + .map(|dt| dt.timestamp()) + .unwrap_or(0); + let t1 = chrono::DateTime::parse_from_rfc3339(&last_seen) + .map(|dt| dt.timestamp()) + .unwrap_or(0); + (t1 - t0).max(0) + }; + + // Collect unique terms found in this group's messages. + let mut found_terms: Vec = terms + .iter() + .filter(|term| { + anchors.iter().any(|r| { + first_abuse_term(&r.message, std::slice::from_ref(term)).is_some() + }) + }) + .cloned() + .collect(); + found_terms.sort(); + found_terms.dedup(); + + let mut anchor_ids: Vec = anchors.iter().map(|r| r.id).collect(); + anchor_ids.sort(); + + // Score: abuse_count dominates; density and term variety boost. + let density = if duration_secs > 0 { + abuse_count as f64 / (duration_secs as f64 / 60.0) + } else { + abuse_count as f64 + }; + let term_variety = found_terms.len() as f64; + let priority_score = abuse_count as f64 * 10.0 + density * 2.0 + term_variety; + + // Compare the f64 directly: `as u64` truncates and maps NaN + // to 0, which would mislabel a pathological score as "low" + // (full-review QL2). + let priority_label = if priority_score < 15.0 { + "low" + } else if priority_score < 30.0 { + "medium" + } else if priority_score < 50.0 { + "high" + } else { + "critical" + } + .to_string(); + + // Stable incident ID using a deterministic hash of session identity + anchor IDs. + let incident_id = { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + let mut h = DefaultHasher::new(); + project.hash(&mut h); + tool.hash(&mut h); + session_id.hash(&mut h); + hostname.hash(&mut h); + for id in &anchor_ids { + id.hash(&mut h); + } + format!("inc-{:016x}", h.finish()) + }; + + AbuseIncident { + incident_id, + project, + tool, + session_id, + hostname, + first_seen, + last_seen, + duration_secs, + abuse_count, + terms: found_terms, + anchor_ids, + priority_score, + priority_label, + window_minutes: (window_secs / 60) as u32, + } + }, + ) + .collect(); + + // Sort by priority_score descending, then last_seen descending. + // total_cmp is a total order (NaN sorts deterministically) — the + // partial_cmp/unwrap_or(Equal) idiom can produce a non-total order if a + // NaN ever sneaks into a score (full-review QL3). + incidents.sort_by(|a, b| { + b.priority_score + .total_cmp(&a.priority_score) + .then_with(|| b.last_seen.cmp(&a.last_seen)) + }); + + let total_incidents = incidents.len(); + let truncated = total_incidents > limit || candidate_window_truncated; + incidents.truncate(limit); + + Ok(AiIncidentResult { + incidents, + total_incidents, + candidate_rows: raw_candidate_count.min(CANDIDATE_CAP), + candidate_cap: CANDIDATE_CAP, + candidate_window_truncated, + truncated, + }) +} + +fn ai_incident_anchor_sql( + params: &AiIncidentParams, + terms: &[String], + candidate_cap: usize, +) -> (String, Vec) { + let mut sql = String::from( + "WITH candidates(id) AS MATERIALIZED ( + SELECT l.id + FROM logs_fts + JOIN logs l ON l.id = logs_fts.rowid + WHERE logs_fts MATCH ?1 + AND l.ai_project IS NOT NULL AND l.ai_project != '' + AND l.ai_tool IS NOT NULL AND l.ai_tool != '' + AND l.ai_session_id IS NOT NULL AND l.ai_session_id != ''", + ); + let mut bindings = vec![rusqlite::types::Value::Text(abuse_fts_query(terms))]; + let mut idx = 2usize; + + if let Some(project) = ¶ms.ai_project { + sql.push_str(&format!(" AND l.ai_project = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(project.clone())); + idx += 1; + } + if let Some(tool) = ¶ms.ai_tool { + sql.push_str(&format!(" AND l.ai_tool = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(tool.clone())); + idx += 1; + } + if let Some(from) = ¶ms.since { + sql.push_str(&format!(" AND l.timestamp >= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(from.clone())); + idx += 1; + } + if let Some(to) = ¶ms.until { + sql.push_str(&format!(" AND l.timestamp <= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(to.clone())); + } + let _ = idx; + sql.push_str(&format!( + " ORDER BY logs_fts.rowid ASC LIMIT {} + ) + SELECT l.id, l.timestamp, l.hostname, + l.ai_tool, l.ai_project, l.ai_session_id, l.message + FROM candidates c + JOIN logs l ON l.id = c.id", + candidate_cap + 1 + )); + (sql, bindings) +} + +pub fn investigate_ai_incidents( + pool: &DbPool, + params: &AiInvestigateParams, +) -> Result { + let limit = params.limit.unwrap_or(3).clamp(1, 10) as usize; + let incident_lookup_limit = if params.incident_id.is_some() { + 100 + } else { + limit as u32 + }; + let corr_mins = i64::from(params.correlation_window_minutes.unwrap_or(5).clamp(1, 120)); + + // Reuse incident grouping to find the top incidents. Exact incident + // assessment may target an ID outside the top investigation page, so it + // searches up to the incident-list cap and then builds one evidence bundle. + let incident_result = search_ai_incidents( + pool, + &AiIncidentParams { + ai_project: params.ai_project.clone(), + ai_tool: params.ai_tool.clone(), + since: params.since.clone(), + until: params.until.clone(), + limit: Some(incident_lookup_limit), + window_minutes: params.window_minutes, + terms: params.terms.clone(), + }, + )?; + let total_incidents = incident_result.total_incidents; + let truncated = incident_result.truncated; + let incidents = if let Some(incident_id) = ¶ms.incident_id { + incident_result + .incidents + .into_iter() + .filter(|incident| incident.incident_id == *incident_id) + .collect() + } else { + incident_result.incidents + }; + + let conn = pool.get()?; + let mut evidence = Vec::with_capacity(incidents.len()); + + for incident in incidents { + const TRANSCRIPT_CAP: usize = 20; + const NEARBY_CAP: usize = 50; + + // Fetch anchor log entries. + let anchors = if incident.anchor_ids.is_empty() { + Vec::new() + } else { + let placeholders: Vec = (1..=incident.anchor_ids.len()) + .map(|i| format!("?{i}")) + .collect(); + let sql = format!( + "SELECT id, timestamp, hostname, facility, severity, app_name, + process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs WHERE id IN ({}) ORDER BY timestamp ASC", + placeholders.join(",") + ); + let mut stmt = conn.prepare(&sql)?; + + stmt.query_map( + rusqlite::params_from_iter( + incident + .anchor_ids + .iter() + .map(|id| rusqlite::types::Value::Integer(*id)), + ), + map_row, + )? + .collect::>>()? + }; + + // Transcript context: entries in the same session before first anchor and after last anchor. + let (transcript_before, transcript_before_truncated) = if let Some(first) = anchors.first() + { + let rows = { + let mut stmt = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, app_name, + process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, + metadata_json + FROM logs + WHERE ai_session_id = ?1 AND ai_project = ?2 AND ai_tool = ?3 + AND timestamp < ?4 + ORDER BY timestamp DESC + LIMIT 21", + )?; + + stmt.query_map( + rusqlite::params![ + &incident.session_id, + &incident.project, + &incident.tool, + &first.timestamp, + ], + map_row, + )? + .collect::>>()? + }; + let truncated = rows.len() > TRANSCRIPT_CAP; + let mut out = rows; + out.truncate(TRANSCRIPT_CAP); + out.reverse(); // chronological order + (out, truncated) + } else { + (Vec::new(), false) + }; + + let (transcript_after, transcript_after_truncated) = if let Some(last) = anchors.last() { + let rows = { + let mut stmt = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, app_name, + process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs + WHERE ai_session_id = ?1 AND ai_project = ?2 AND ai_tool = ?3 + AND timestamp > ?4 + ORDER BY timestamp ASC + LIMIT 21", + )?; + + stmt.query_map( + rusqlite::params![ + &incident.session_id, + &incident.project, + &incident.tool, + &last.timestamp, + ], + map_row, + )? + .collect::>>()? + }; + let truncated = rows.len() > TRANSCRIPT_CAP; + let mut out = rows; + out.truncate(TRANSCRIPT_CAP); + (out, truncated) + } else { + (Vec::new(), false) + }; + + // Nearby non-AI logs in the correlation window. + let (nearby_logs, nearby_logs_truncated) = { + // Window: corr_mins before first_seen through corr_mins after last_seen. + let win_from = chrono::DateTime::parse_from_rfc3339(&incident.first_seen) + .map(|dt| { + use chrono::Duration; + (dt.with_timezone(&chrono::Utc) - Duration::minutes(corr_mins)) + .format("%Y-%m-%dT%H:%M:%S%.3fZ") + .to_string() + }) + .unwrap_or_else(|_| incident.first_seen.clone()); + let win_to = chrono::DateTime::parse_from_rfc3339(&incident.last_seen) + .map(|dt| { + use chrono::Duration; + (dt.with_timezone(&chrono::Utc) + Duration::minutes(corr_mins)) + .format("%Y-%m-%dT%H:%M:%S%.3fZ") + .to_string() + }) + .unwrap_or_else(|_| incident.last_seen.clone()); + + let mut stmt = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, app_name, + process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs + WHERE timestamp >= ?1 AND timestamp <= ?2 + AND (ai_project IS NULL OR ai_project = '') + ORDER BY timestamp ASC + LIMIT 51", + )?; + let rows = stmt + .query_map(rusqlite::params![win_from, win_to], map_row)? + .collect::>>()?; + let truncated = rows.len() > NEARBY_CAP; + let mut out = rows; + out.truncate(NEARBY_CAP); + (out, truncated) + }; + + // Nearby errors: subset of nearby_logs with severity warning+. + let error_sevs = ["emergency", "alert", "critical", "error", "warning"]; + let nearby_errors: Vec = nearby_logs + .iter() + .filter(|e| error_sevs.contains(&e.severity.as_str())) + .cloned() + .collect(); + + evidence.push(IncidentEvidence { + incident, + transcript_before, + transcript_before_truncated, + transcript_after, + transcript_after_truncated, + anchors, + nearby_logs, + nearby_logs_truncated, + nearby_errors, + }); + } + + Ok(AiInvestigateResult { + evidence, + total_incidents, + truncated, + }) +} + +fn normalized_abuse_terms(custom_terms: &[String]) -> Vec { + let source: Vec = if custom_terms.is_empty() { + DEFAULT_AI_ABUSE_TERMS + .iter() + .map(|term| (*term).to_string()) + .collect() + } else { + custom_terms.to_vec() + }; + + let mut terms = source + .into_iter() + .map(|term| term.trim().to_ascii_lowercase()) + .filter(|term| { + !term.is_empty() + && term.len() <= 64 + && term + .chars() + .all(|ch| ch.is_ascii_alphanumeric() || ch == '-' || ch == '_') + }) + .collect::>(); + terms.sort(); + terms.dedup(); + if terms.is_empty() { + DEFAULT_AI_ABUSE_TERMS + .iter() + .map(|term| (*term).to_string()) + .collect() + } else { + terms + } +} + +fn abuse_fts_query(terms: &[String]) -> String { + // FTS5 escapes a literal " inside a phrase by doubling it: "" → match one " + terms + .iter() + .map(|term| format!("\"{}\"", term.replace('"', "\"\""))) + .collect::>() + .join(" OR ") +} + +fn first_abuse_term(message: &str, terms: &[String]) -> Option { + let lower = message.to_ascii_lowercase(); + terms + .iter() + .filter_map(|term| first_term_index(&lower, term).map(|idx| (idx, term))) + .min_by_key(|(idx, _)| *idx) + .map(|(_, term)| term.clone()) +} + +fn first_term_index(message: &str, term: &str) -> Option { + let mut offset = 0usize; + while let Some(relative) = message[offset..].find(term) { + let start = offset + relative; + let end = start + term.len(); + if is_abuse_boundary(message[..start].chars().next_back()) + && is_abuse_boundary(message[end..].chars().next()) + { + return Some(start); + } + offset = end; + } + None +} + +fn is_abuse_boundary(ch: Option) -> bool { + ch.is_none_or(|ch| !ch.is_ascii_alphanumeric() && ch != '_') +} + +fn ai_session_context( + conn: &rusqlite::Connection, + entry: &LogEntry, + before: u32, + after: u32, +) -> Result<(Vec, Vec)> { + let Some(tool) = entry.ai_tool.as_deref() else { + return Ok((Vec::new(), Vec::new())); + }; + let Some(project) = entry.ai_project.as_deref() else { + return Ok((Vec::new(), Vec::new())); + }; + let Some(session_id) = entry.ai_session_id.as_deref() else { + return Ok((Vec::new(), Vec::new())); + }; + + let mut before_stmt = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, + app_name, process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs + WHERE hostname = ?1 + AND ai_tool = ?2 + AND ai_project = ?3 + AND ai_session_id = ?4 + AND (timestamp < ?5 OR (timestamp = ?5 AND id < ?6)) + ORDER BY timestamp DESC, id DESC + LIMIT ?7", + )?; + let mut before_rows = before_stmt + .query_map( + params![ + &entry.hostname, + tool, + project, + session_id, + &entry.timestamp, + entry.id, + before + ], + map_row, + )? + .collect::>>()?; + before_rows.reverse(); + + let mut after_stmt = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, + app_name, process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs + WHERE hostname = ?1 + AND ai_tool = ?2 + AND ai_project = ?3 + AND ai_session_id = ?4 + AND (timestamp > ?5 OR (timestamp = ?5 AND id > ?6)) + ORDER BY timestamp ASC, id ASC + LIMIT ?7", + )?; + let after_rows = after_stmt + .query_map( + params![ + &entry.hostname, + tool, + project, + session_id, + &entry.timestamp, + entry.id, + after + ], + map_row, + )? + .collect::>>()?; + + Ok((before_rows, after_rows)) +} + +/// Get database stats +pub fn get_stats(pool: &DbPool, config: &StorageConfig) -> Result { + get_stats_with_options(pool, config, false) +} + +/// `get_stats`, but `include_fts_diagnostics` controls whether the +/// `phantom_fts_rows` field is computed. That value requires +/// `COUNT(*) FROM logs_fts` — an external-content FTS5 index scan that is +/// cheap on small DBs but expensive on very large ones (the index has no +/// O(1) row counter). The default `stats` path passes `false` so the common +/// query stays fast; callers that specifically need the FTS merge-health +/// diagnostic pass `true`. +pub fn get_stats_with_options( + pool: &DbPool, + config: &StorageConfig, + include_fts_diagnostics: bool, +) -> Result { + let metrics = get_storage_metrics(pool, config)?; + let write_blocked = exceeds_trigger(&metrics, config); + let mut conn = pool.get()?; + + // Deferred read transaction ensures the log stats form a consistent snapshot + let tx = conn.transaction_with_behavior(rusqlite::TransactionBehavior::Deferred)?; + // total_logs reads the timeline_hourly rollup (O(#buckets)) plus the live + // delta of rows ingested since the rollup watermark, instead of the O(#rows) + // `COUNT(*) FROM logs` (~7s on multi-million-row DBs). The rollup covers + // `logs.id <= source_max_id`; the delta covers `id > source_max_id`, so the + // sum is exact at the current snapshot for ADDs. It is NOT perfectly exact + // under concurrent retention DELETEs of rows the rollup already counted: the + // retention prune (spawn_retention_task) trims whole stale buckets, leaving + // at most a transient single-boundary-hour overcount — accepted as a + // negligible drift for a stats counter. (bead syslog-mcp-kcvq) + let rollup_max_id: i64 = tx.query_row( + "SELECT source_max_id FROM timeline_hourly_meta WHERE id = 1", + [], + |r| r.get(0), + )?; + let rollup_total: i64 = tx.query_row( + "SELECT COALESCE(SUM(event_count), 0) FROM timeline_hourly", + [], + |r| r.get(0), + )?; + let live_delta: i64 = tx.query_row( + "SELECT COUNT(*) FROM logs WHERE id > ?1", + [rollup_max_id], + |r| r.get(0), + )?; + let total_logs: i64 = rollup_total + live_delta; + let total_hosts: i64 = tx.query_row("SELECT COUNT(*) FROM hosts", [], |r| r.get(0))?; + let phantom_fts_rows = if include_fts_diagnostics { + let fts_rows: i64 = tx + .query_row("SELECT COUNT(*) FROM logs_fts", [], |r| r.get(0)) + .unwrap_or(0); + Some((fts_rows - total_logs).max(0)) + } else { + None + }; + // MIN/MAX return a single nullable row; use get::<_, Option<_>> so NULL becomes + // None while real query errors (e.g. missing table) still propagate via `?`. + // Both use the covering index idx_logs_timestamp (SEARCH, O(log n)). + let oldest: Option = tx.query_row("SELECT MIN(timestamp) FROM logs", [], |r| { + r.get::<_, Option>(0) + })?; + let newest: Option = tx.query_row("SELECT MAX(timestamp) FROM logs", [], |r| { + r.get::<_, Option>(0) + })?; + tx.finish()?; + + Ok(DbStats { + total_logs, + total_hosts, + oldest_log: oldest, + newest_log: newest, + logical_db_size_mb: format!("{:.2}", metrics.logical_db_size_bytes as f64 / 1_048_576.0), + physical_db_size_mb: format!("{:.2}", metrics.physical_db_size_bytes as f64 / 1_048_576.0), + free_disk_mb: metrics + .free_disk_bytes + .map(|bytes| format!("{:.2}", bytes as f64 / 1_048_576.0)), + max_db_size_mb: config.max_db_size_mb, + min_free_disk_mb: config.min_free_disk_mb, + write_blocked, + phantom_fts_rows, + }) +} + +/// Syslog severity level names ordered by numeric value (0=emerg, 7=debug). +/// Used by both the MCP layer (for threshold filtering) and the syslog parser (for decoding). +pub const SEVERITY_LEVELS: &[&str] = &[ + "emerg", "alert", "crit", "err", "warning", "notice", "info", "debug", +]; + +/// Convert a severity name to its numeric syslog level (0=emerg, 7=debug). +/// Accepts the canonical RFC 5424 keywords (case-insensitive) plus common +/// aliases: `error`/`fatal`/`panic` for `err`, `warn` for `warning`, +/// `critical` for `crit`, `emergency` for `emerg`. +/// Returns `None` for unrecognised names. +pub fn severity_to_num(s: &str) -> Option { + let canonical = match s.to_ascii_lowercase().as_str() { + "emergency" => "emerg", + "critical" => "crit", + "error" | "fatal" | "panic" => "err", + "warn" => "warning", + other => { + return SEVERITY_LEVELS + .iter() + .position(|&l| l == other) + .map(|i| i as u8); + } + }; + SEVERITY_LEVELS + .iter() + .position(|&l| l == canonical) + .map(|i| i as u8) +} + +fn append_filters( + sql: &mut String, + bindings: &mut Vec, + idx: &mut usize, + params: &SearchParams, +) { + if let Some(ref h) = params.host { + append_host_selector(sql, bindings, idx, "l.hostname", h); + } + if let Some(ref source_ip) = params.source { + sql.push_str(&format!(" AND l.source_ip = ?{}", *idx)); + bindings.push(rusqlite::types::Value::Text(source_ip.clone())); + *idx += 1; + } + if let Some(ref prefix) = params.source_ip_prefix { + sql.push_str(&format!(" AND l.source_ip >= ?{}", *idx)); + bindings.push(rusqlite::types::Value::Text(prefix.clone())); + *idx += 1; + if let Some(upper) = prefix_upper_bound(prefix) { + sql.push_str(&format!(" AND l.source_ip < ?{}", *idx)); + bindings.push(rusqlite::types::Value::Text(upper)); + *idx += 1; + } + } + if let Some(ref s) = params.severity { + sql.push_str(&format!(" AND l.severity = ?{}", *idx)); + bindings.push(rusqlite::types::Value::Text(s.clone())); + *idx += 1; + } + if let Some(ref levels) = params.severity_in + && !levels.is_empty() + { + let placeholders: Vec = levels + .iter() + .enumerate() + .map(|(i, _)| format!("?{}", *idx + i)) + .collect(); + sql.push_str(&format!(" AND l.severity IN ({})", placeholders.join(", "))); + for level in levels { + bindings.push(rusqlite::types::Value::Text(level.clone())); + *idx += 1; + } + } + if let Some(ref a) = params.app { + sql.push_str(&format!(" AND l.app_name = ?{}", *idx)); + bindings.push(rusqlite::types::Value::Text(a.clone())); + *idx += 1; + } + if let Some(ref f) = params.facility { + sql.push_str(&format!(" AND l.facility = ?{}", *idx)); + bindings.push(rusqlite::types::Value::Text(f.clone())); + *idx += 1; + } + if let Some(ref f) = params.exclude_facility { + sql.push_str(&format!( + " AND (l.facility IS NULL OR l.facility != ?{})", + *idx + )); + bindings.push(rusqlite::types::Value::Text(f.clone())); + *idx += 1; + } + if let Some(ref pid) = params.process_id { + sql.push_str(&format!(" AND l.process_id = ?{}", *idx)); + bindings.push(rusqlite::types::Value::Text(pid.clone())); + *idx += 1; + } + if let Some(ref from) = params.since { + sql.push_str(&format!(" AND l.timestamp >= ?{}", *idx)); + bindings.push(rusqlite::types::Value::Text(from.clone())); + *idx += 1; + } + if let Some(ref to) = params.until { + sql.push_str(&format!(" AND l.timestamp <= ?{}", *idx)); + bindings.push(rusqlite::types::Value::Text(to.clone())); + *idx += 1; + } + if let Some(ref from) = params.received_since { + sql.push_str(&format!(" AND l.received_at >= ?{}", *idx)); + bindings.push(rusqlite::types::Value::Text(from.clone())); + *idx += 1; + } + if let Some(ref to) = params.received_until { + sql.push_str(&format!(" AND l.received_at <= ?{}", *idx)); + bindings.push(rusqlite::types::Value::Text(to.clone())); + *idx += 1; + } + if let Some(ref tool) = params.ai_tool { + sql.push_str(&format!(" AND l.ai_tool = ?{}", *idx)); + bindings.push(rusqlite::types::Value::Text(tool.clone())); + *idx += 1; + } + if let Some(ref project) = params.ai_project { + sql.push_str(&format!(" AND l.ai_project = ?{}", *idx)); + bindings.push(rusqlite::types::Value::Text(project.clone())); + *idx += 1; + } + if let Some(ref session_id) = params.ai_session_id { + sql.push_str(&format!(" AND l.ai_session_id = ?{}", *idx)); + bindings.push(rusqlite::types::Value::Text(session_id.clone())); + *idx += 1; + } + if let Some(ref event_action) = params.event_action { + sql.push_str(&format!(" AND l.event_action = ?{}", *idx)); + bindings.push(rusqlite::types::Value::Text(event_action.clone())); + *idx += 1; + } + if params.exclude_ai { + sql.push_str( + " AND (l.ai_project IS NULL OR l.ai_project = '') + AND (l.ai_tool IS NULL OR l.ai_tool = '') + AND (l.ai_session_id IS NULL OR l.ai_session_id = '') + AND (l.ai_transcript_path IS NULL OR l.ai_transcript_path = '') + AND ( + l.app_name IS NULL + OR l.app_name NOT IN ( + 'ai-transcript', + 'claude-transcript', + 'codex-transcript', + 'gemini-transcript' + ) + )", + ); + } +} + +/// Append an indexed hostname selector that accepts the canonical names +/// returned by [`list_hosts`]. Normalization is confined to the tiny `hosts` +/// aggregate table; the outer log lookup still uses exact stored values and +/// can probe the existing hostname indexes. +fn append_host_selector( + sql: &mut String, + bindings: &mut Vec, + idx: &mut usize, + column: &str, + hostname: &str, +) { + let param = format!("?{}", *idx); + let normalized_param = format!("lower(rtrim(trim({param}), '.'))"); + let normalized_host = "lower(rtrim(trim(h.hostname), '.'))"; + let normalized_bare = "lower(rtrim(trim(bare.hostname), '.'))"; + sql.push_str(&format!( + " AND {column} IN ( + SELECT h.hostname + FROM hosts h + WHERE {normalized_host} = {normalized_param} + OR ( + {normalized_host} LIKE {normalized_param} || '.%' + AND EXISTS ( + SELECT 1 + FROM hosts bare + WHERE {normalized_bare} = {normalized_param} + AND instr({normalized_bare}, '.') = 0 + ) + ) + )" + )); + bindings.push(rusqlite::types::Value::Text(hostname.to_string())); + *idx += 1; +} + +fn prefix_upper_bound(prefix: &str) -> Option { + let mut bytes = prefix.as_bytes().to_vec(); + for idx in (0..bytes.len()).rev() { + if bytes[idx] != u8::MAX { + bytes[idx] += 1; + bytes.truncate(idx + 1); + return String::from_utf8(bytes).ok(); + } + } + None +} + +pub(super) fn map_row(row: &rusqlite::Row) -> rusqlite::Result { + map_row_offset(row, 0) +} + +fn map_row_offset(row: &rusqlite::Row, offset: usize) -> rusqlite::Result { + Ok(LogEntry { + id: row.get(offset)?, + timestamp: row.get(offset + 1)?, + hostname: row.get(offset + 2)?, + facility: row.get(offset + 3)?, + severity: row.get(offset + 4)?, + app_name: row.get(offset + 5)?, + process_id: row.get(offset + 6)?, + message: row.get(offset + 7)?, + received_at: row.get(offset + 8)?, + source_ip: row.get(offset + 9)?, + ai_tool: row.get(offset + 10)?, + ai_project: row.get(offset + 11)?, + ai_session_id: row.get(offset + 12)?, + ai_transcript_path: row.get(offset + 13)?, + metadata_json: row.get(offset + 14)?, + }) +} + +/// Map a row that includes the unparsed `raw` syslog frame (column index 8). +pub(super) fn map_row_with_raw( + row: &rusqlite::Row, +) -> rusqlite::Result { + Ok(super::analytics::LogEntryWithRaw { + id: row.get(0)?, + timestamp: row.get(1)?, + hostname: row.get(2)?, + facility: row.get(3)?, + severity: row.get(4)?, + app_name: row.get(5)?, + process_id: row.get(6)?, + message: row.get(7)?, + raw: row.get(8)?, + received_at: row.get(9)?, + source_ip: row.get(10)?, + ai_tool: row.get(11)?, + ai_project: row.get(12)?, + ai_session_id: row.get(13)?, + ai_transcript_path: row.get(14)?, + metadata_json: row.get(15)?, + }) +} + +// --------------------------------------------------------------------------- +// RAG v1: similar_incidents, incident_context +// --------------------------------------------------------------------------- + +use super::models::{ + AppLogCount, CorrelatedSession, IncidentCluster, IncidentContextParams, IncidentContextResult, + SeverityCount, SimilarIncidentsParams, SimilarIncidentsResult, +}; + +/// Return incident clusters from FTS5 hits, grouped by hostname + app_name in +/// non-overlapping windows of `window_minutes` minutes (default 30). +/// +/// Algorithm: +/// 1. FTS5 MATCH over non-AI log rows (optionally filtered by host/app/time). +/// 2. Group hits by (hostname, app_name, floor(unix_epoch / window_secs)). +/// 3. For each cluster: derive severity_peak (min numeric rank = highest sev), +/// collect up to 3 representative message snippets, and look up correlated +/// AI sessions whose transcript timestamps overlap the cluster window. +pub fn similar_incidents_clusters( + pool: &DbPool, + params: &SimilarIncidentsParams, +) -> Result { + validate_fts_query(¶ms.query)?; + + let conn = pool.get()?; + let window_minutes = params.window_minutes.unwrap_or(30).clamp(5, 120); + let limit = params.limit.unwrap_or(10).clamp(1, 50) as usize; + let window_secs = i64::from(window_minutes) * 60; + + // Build the FTS5 + optional filter query. + // Exclude AI transcript rows so clusters contain only system logs. + let mut sql = String::from( + "WITH hits AS MATERIALIZED ( + SELECT l.id, l.timestamp, l.hostname, l.app_name, l.severity, l.message + FROM logs_fts + JOIN logs l ON l.id = logs_fts.rowid + WHERE logs_fts MATCH ?1 + AND (l.ai_project IS NULL OR l.ai_project = '')", + ); + + let mut query_params = SqlParams::new(2); + query_params + .bindings + .push(rusqlite::types::Value::Text(params.query.clone())); + + if let Some(hostname) = ¶ms.host { + let idx = query_params.push_text(hostname.clone()); + sql.push_str(&format!(" AND l.hostname = ?{idx}")); + } + if let Some(app_name) = ¶ms.app { + let idx = query_params.push_text(app_name.clone()); + sql.push_str(&format!(" AND l.app_name = ?{idx}")); + } + if let Some(from) = ¶ms.since { + let idx = query_params.push_text(from.clone()); + sql.push_str(&format!(" AND l.timestamp >= ?{idx}")); + } + if let Some(to) = ¶ms.until { + let idx = query_params.push_text(to.clone()); + sql.push_str(&format!(" AND l.timestamp <= ?{idx}")); + } + // Apply severity_min filter: include only logs at or above the threshold. + if let Some(severity_min) = ¶ms.severity_min { + let threshold = severity_to_num(severity_min).ok_or_else(|| { + anyhow::anyhow!( + "invalid severity_min '{}': must be one of {}", + severity_min, + SEVERITY_LEVELS.join(", ") + ) + })?; + let levels_in: Vec = SEVERITY_LEVELS[..=threshold as usize] + .iter() + .map(|s| s.to_string()) + .collect(); + let placeholders: Vec = levels_in + .iter() + .map(|s| { + let idx = query_params.push_text(s.clone()); + format!("?{idx}") + }) + .collect(); + sql.push_str(&format!(" AND l.severity IN ({})", placeholders.join(", "))); + } + + sql.push_str(&format!( + " ORDER BY l.id DESC LIMIT {SIMILAR_INCIDENT_FTS_CANDIDATE_CAP} + ), + bucketed AS ( + SELECT + hostname, + app_name, + CAST(strftime('%s', timestamp) AS INTEGER) / {window_secs} AS bucket, + MIN(timestamp) AS window_start, + MAX(timestamp) AS window_end, + COUNT(*) AS log_count, + GROUP_CONCAT(severity, ',') AS severities, + GROUP_CONCAT(SUBSTR(message, 1, 256), '|||') AS messages + FROM hits + GROUP BY hostname, app_name, bucket + ) + SELECT hostname, app_name, window_start, window_end, log_count, severities, messages + FROM bucketed + ORDER BY log_count DESC, window_start DESC + LIMIT {}", + limit + 1 + )); + + let mut stmt = conn.prepare(&sql).map_err(|e| { + tracing::error!(error = %e, "similar_incidents_clusters prepare failed"); + anyhow::anyhow!("similar_incidents query failed") + })?; + let rows = stmt + .query_map( + rusqlite::params_from_iter(query_params.bindings.iter()), + |row| { + Ok(( + row.get::<_, String>(0)?, // hostname + row.get::<_, Option>(1)?, // app_name + row.get::<_, String>(2)?, // window_start + row.get::<_, String>(3)?, // window_end + row.get::<_, i64>(4)?, // log_count + row.get::<_, String>(5)?, // severities (comma-joined) + row.get::<_, String>(6)?, // messages (|||joined) + )) + }, + ) + .map_err(|e| { + tracing::error!(error = %e, "similar_incidents_clusters query failed"); + anyhow::anyhow!("similar_incidents query failed") + })?; + + // Collect raw cluster rows first; keep one extra to detect truncation. + struct RawCluster { + hostname: String, + app_name: Option, + window_start: String, + window_end: String, + log_count: i64, + severity_peak: String, + representative_messages: Vec, + } + let mut raw: Vec = Vec::new(); + for row in rows { + let (hostname, app_name, window_start, window_end, log_count, severities, messages) = + row.map_err(|e| { + tracing::error!(error = %e, "similar_incidents_clusters row mapping failed"); + anyhow::anyhow!("similar_incidents row mapping failed") + })?; + + // Find peak severity (lowest numeric value = highest severity). + let severity_peak = severities + .split(',') + .filter_map(|s| severity_to_num(s).map(|n| (n, s.to_string()))) + .min_by_key(|(n, _)| *n) + .map(|(_, s)| s) + .unwrap_or_else(|| "info".to_string()); + + // Collect up to 3 representative messages. + let representative_messages: Vec = messages + .split("|||") + .take(3) + .map(|m| m.to_string()) + .collect(); + + raw.push(RawCluster { + hostname, + app_name, + window_start, + window_end, + log_count, + severity_peak, + representative_messages, + }); + } + + // Detect truncation (we queried limit+1 rows) and trim to the true limit. + let truncated = raw.len() > limit; + raw.truncate(limit); + + // Build one UNION ALL query across all cluster windows so each window gets + // its own per-session match_count. This is O(1) roundtrips while keeping + // counts accurate (no global-span inflation when a session spans clusters). + let per_cluster_sessions = find_correlated_sessions_per_cluster( + &conn, + &raw.iter() + .map(|c| (c.window_start.as_str(), c.window_end.as_str())) + .collect::>(), + )?; + + let clusters: Vec = raw + .into_iter() + .map(|rc| { + let key = (rc.window_start.clone(), rc.window_end.clone()); + let correlated_sessions = per_cluster_sessions.get(&key).cloned().unwrap_or_default(); + IncidentCluster { + hostname: rc.hostname, + app_name: rc.app_name, + window_start: rc.window_start, + window_end: rc.window_end, + log_count: rc.log_count, + severity_peak: rc.severity_peak, + representative_messages: rc.representative_messages, + correlated_sessions, + } + }) + .collect(); + + let total_clusters = clusters.len(); + Ok(SimilarIncidentsResult { + query: params.query.clone(), + total_clusters, + truncated, + clusters, + }) +} + +/// Per-cluster session lookup using a single UNION ALL query so each cluster +/// window gets its own accurate match_count rather than an inflated global count. +/// Returns a map keyed by (window_start, window_end) → top-5 sessions. +fn find_correlated_sessions_per_cluster( + conn: &rusqlite::Connection, + windows: &[(&str, &str)], +) -> Result>> { + use std::collections::HashMap; + + if windows.is_empty() { + return Ok(HashMap::new()); + } + + // Build UNION ALL: one SELECT per cluster window, tagging each row with ws/we. + // Parameters use stride-2 (?{p} = ws, ?{p+1} = we for window i). SQLite ?N + // numbered bindings reuse the same value within one arm's subquery without + // requiring duplicate params in the binding list. + let mut arms: Vec = Vec::with_capacity(windows.len()); + for (i, _) in windows.iter().enumerate() { + let p = 1 + i * 2; + arms.push(format!( + "SELECT ?{p} AS ws, ?{p1} AS we, + l.ai_project, l.ai_tool, l.ai_session_id, + COUNT(*) AS match_count, + (SELECT l2.message FROM logs l2 + WHERE l2.ai_project = l.ai_project + AND l2.ai_tool = l.ai_tool + AND l2.ai_session_id = l.ai_session_id + AND l2.timestamp BETWEEN ?{p} AND ?{p1} + ORDER BY l2.timestamp DESC LIMIT 1) AS best_snippet + FROM logs l + WHERE l.ai_project IS NOT NULL AND l.ai_project != '' + AND l.ai_tool IS NOT NULL AND l.ai_tool != '' + AND l.ai_session_id IS NOT NULL AND l.ai_session_id != '' + AND l.timestamp BETWEEN ?{p} AND ?{p1} + GROUP BY l.ai_project, l.ai_tool, l.ai_session_id", + p = p, + p1 = p + 1, + )); + } + let sql = arms.join("\nUNION ALL\n"); + + let mut stmt = conn + .prepare(&sql) + .map_err(|e| anyhow::anyhow!("find_correlated_sessions_per_cluster prepare: {e}"))?; + + // Two params per window (ws, we); ?N reuse within each arm handles the rest. + let params: Vec<&dyn rusqlite::ToSql> = windows + .iter() + .flat_map(|(ws, we)| { + let v: [&dyn rusqlite::ToSql; 2] = [ws, we]; + v + }) + .collect(); + + let rows = stmt + .query_map(params.as_slice(), |row| { + let ws: String = row.get(0)?; + let we: String = row.get(1)?; + let project: String = row.get(2)?; + let tool: String = row.get(3)?; + let session_id: String = row.get(4)?; + let match_count: i64 = row.get(5)?; + let best_snippet: Option = row.get(6)?; + Ok((ws, we, project, tool, session_id, match_count, best_snippet)) + }) + .map_err(|e| anyhow::anyhow!("find_correlated_sessions_per_cluster query: {e}"))?; + + // Collect all sessions per cluster before sorting — UNION ALL rows arrive + // unordered, so the top-5 cap must come after sorting, not during insertion. + let mut map: HashMap<(String, String), Vec> = HashMap::new(); + for row in rows { + let (ws, we, project, tool, session_id, match_count, best_snippet) = + row.map_err(|e| anyhow::anyhow!("find_correlated_sessions_per_cluster row: {e}"))?; + map.entry((ws, we)).or_default().push(CorrelatedSession { + project, + tool, + session_id, + match_count, + best_snippet, + }); + } + // Sort by match_count descending, then cap at 5 per cluster. + for sessions in map.values_mut() { + sessions.sort_by_key(|b| std::cmp::Reverse(b.match_count)); + sessions.truncate(5); + } + Ok(map) +} + +/// Return aggregate log statistics + error logs + correlated AI sessions for a +/// given time window. +pub fn incident_context_summary( + pool: &DbPool, + params: &IncidentContextParams, +) -> Result { + let conn = pool.get()?; + let limit = params.limit.unwrap_or(50).clamp(1, 200) as usize; + if let Some(query) = params.query.as_deref() { + validate_fts_query(query)?; + } + + // Resolve severity threshold. Default to "warning" (numeric 4). + let severity_threshold = params + .severity_min + .as_deref() + .map(|s| { + severity_to_num(s).ok_or_else(|| { + anyhow::anyhow!( + "invalid severity_min '{}': must be one of emerg, alert, crit, err, warning, notice, info, debug", + s + ) + }) + }) + .transpose()? + .unwrap_or_else(|| severity_to_num("warning").unwrap()); + + // Build reusable aggregate params with host/app/AI-exclusion filters. + // Params: ?1=from, ?2=to, then optional host/app starting at ?3. + // All aggregate queries exclude AI transcript rows (ai_project IS NULL or ''). + let mut agg_params = SqlParams::new(3); + agg_params + .bindings + .push(rusqlite::types::Value::Text(params.since.clone())); + agg_params + .bindings + .push(rusqlite::types::Value::Text(params.until.clone())); + let mut agg_host_clause = String::new(); + let mut agg_app_clause = String::new(); + if let Some(hostname) = ¶ms.host { + let idx = agg_params.push_text(hostname.clone()); + agg_host_clause = format!(" AND hostname = ?{idx}"); + } + if let Some(app_name) = ¶ms.app { + let idx = agg_params.push_text(app_name.clone()); + agg_app_clause = format!(" AND app_name = ?{idx}"); + } + let agg_base_filter = format!( + "WHERE (ai_project IS NULL OR ai_project = '') + AND timestamp BETWEEN ?1 AND ?2{agg_host_clause}{agg_app_clause}" + ); + + // Total log count in window (system logs only, scoped by host/app). + let total_logs: i64 = conn + .query_row( + &format!("SELECT COUNT(*) FROM logs INDEXED BY idx_logs_timestamp {agg_base_filter}"), + rusqlite::params_from_iter(agg_params.bindings.iter()), + |r| r.get(0), + ) + .map_err(|e| anyhow::anyhow!("incident_context total_logs: {e}"))?; + + // Counts by severity (system logs only, scoped by host/app). + let mut by_sev_stmt = conn + .prepare(&format!( + "SELECT severity, COUNT(*) FROM logs INDEXED BY idx_logs_timestamp + {agg_base_filter} + GROUP BY severity + ORDER BY COUNT(*) DESC" + )) + .map_err(|e| anyhow::anyhow!("incident_context by_severity prepare: {e}"))?; + let by_severity: Vec = by_sev_stmt + .query_map( + rusqlite::params_from_iter(agg_params.bindings.iter()), + |row| { + Ok(SeverityCount { + severity: row.get(0)?, + count: row.get(1)?, + }) + }, + ) + .map_err(|e| anyhow::anyhow!("incident_context by_severity query: {e}"))? + .collect::>>()?; + + // Counts by app_name (top 20, system logs only, scoped by host/app). + let mut by_app_stmt = conn + .prepare(&format!( + "SELECT app_name, COUNT(*) FROM logs INDEXED BY idx_logs_timestamp + {agg_base_filter} + GROUP BY app_name + ORDER BY COUNT(*) DESC + LIMIT 20" + )) + .map_err(|e| anyhow::anyhow!("incident_context by_app prepare: {e}"))?; + let by_app: Vec = by_app_stmt + .query_map( + rusqlite::params_from_iter(agg_params.bindings.iter()), + |row| { + Ok(AppLogCount { + app_name: row.get(0)?, + count: row.get(1)?, + }) + }, + ) + .map_err(|e| anyhow::anyhow!("incident_context by_app query: {e}"))? + .collect::>>()?; + + // Error logs: system logs at or above severity threshold in the window. + let error_severities: Vec = SEVERITY_LEVELS[..=severity_threshold as usize] + .iter() + .map(|s| s.to_string()) + .collect(); + + // Build parameterized query for error logs. + // Params: ?1=from, ?2=to, ?3..=?N=severities, then optional host/app. + // SqlParams::new(3) sets next_idx=3 so push_text calls start at ?3, after + // the two manually-pushed bindings for from (?1) and to (?2). + let mut err_params = SqlParams::new(3); + err_params + .bindings + .push(rusqlite::types::Value::Text(params.since.clone())); + err_params + .bindings + .push(rusqlite::types::Value::Text(params.until.clone())); + + let query_idx = params + .query + .as_ref() + .map(|query| err_params.push_text(query.clone())); + + let sev_placeholders: Vec = error_severities + .iter() + .map(|s| { + let idx = err_params.push_text(s.clone()); + format!("?{idx}") + }) + .collect(); + + let mut err_sql = format!("SELECT {FTS_SELECT_COLS} "); + match query_idx { + Some(idx) => err_sql.push_str(&format!( + "FROM logs_fts + JOIN logs l ON l.id = logs_fts.rowid + WHERE logs_fts MATCH ?{idx} + AND l.timestamp BETWEEN ?1 AND ?2" + )), + None => err_sql.push_str( + "FROM logs l INDEXED BY idx_logs_timestamp + WHERE l.timestamp BETWEEN ?1 AND ?2", + ), + } + err_sql.push_str(&format!( + " AND l.severity IN ({}) + AND (l.ai_project IS NULL OR l.ai_project = '')", + sev_placeholders.join(", ") + )); + + if let Some(hostname) = ¶ms.host { + let idx = err_params.push_text(hostname.clone()); + err_sql.push_str(&format!(" AND l.hostname = ?{idx}")); + } + if let Some(app_name) = ¶ms.app { + let idx = err_params.push_text(app_name.clone()); + err_sql.push_str(&format!(" AND l.app_name = ?{idx}")); + } + // Query limit+1 rows so we can detect true truncation. + err_sql.push_str(&format!(" ORDER BY l.timestamp DESC LIMIT {}", limit + 1)); + + let mut err_stmt = conn.prepare(&err_sql).map_err(|e| { + tracing::error!(error = %e, "incident_context error_logs prepare failed"); + anyhow::anyhow!("incident_context error_logs query failed") + })?; + let error_rows = err_stmt + .query_map( + rusqlite::params_from_iter(err_params.bindings.iter()), + map_row, + ) + .map_err(|e| { + tracing::error!(error = %e, "incident_context error_logs query failed"); + anyhow::anyhow!("incident_context error_logs query failed") + })?; + let mut error_logs: Vec = + error_rows.collect::>>()?; + let error_logs_truncated = error_logs.len() > limit; + error_logs.truncate(limit); + + // AI sessions active in the window — query on the already-held conn to + // avoid a second pool.get() call (which deadlocks on single-connection test pools). + let ai_sessions = { + let mut ai_sql = String::from( + "SELECT ai_project, ai_tool, ai_session_id, + MIN(ai_transcript_path) AS ai_transcript_path, + hostname, + MIN(timestamp) AS first_seen, + MAX(timestamp) AS last_seen, + COUNT(*) AS event_count + FROM logs + WHERE ai_project IS NOT NULL AND ai_project != '' + AND ai_tool IS NOT NULL AND ai_tool != '' + AND ai_session_id IS NOT NULL AND ai_session_id != '' + AND timestamp BETWEEN ?1 AND ?2", + ); + let mut ai_bindings: Vec = vec![ + rusqlite::types::Value::Text(params.since.clone()), + rusqlite::types::Value::Text(params.until.clone()), + ]; + if let Some(hostname) = ¶ms.host { + ai_bindings.push(rusqlite::types::Value::Text(hostname.clone())); + ai_sql.push_str(&format!(" AND hostname = ?{}", ai_bindings.len())); + } + ai_sql.push_str( + " GROUP BY ai_project, ai_tool, ai_session_id, hostname + ORDER BY last_seen DESC LIMIT 20", + ); + let mut ai_stmt = conn + .prepare(&ai_sql) + .map_err(|e| anyhow::anyhow!("incident_context ai_sessions prepare: {e}"))?; + let rows = ai_stmt + .query_map(rusqlite::params_from_iter(ai_bindings.iter()), |row| { + Ok(super::models::AiSessionEntry { + ai_project: row.get(0)?, + ai_tool: row.get(1)?, + ai_session_id: row.get(2)?, + ai_transcript_path: row.get(3)?, + hostname: row.get(4)?, + first_seen: row.get(5)?, + last_seen: row.get(6)?, + event_count: row.get(7)?, + }) + }) + .map_err(|e| anyhow::anyhow!("incident_context ai_sessions query: {e}"))?; + rows.collect::>>()? + }; + + Ok(IncidentContextResult { + window_from: params.since.clone(), + window_to: params.until.clone(), + total_logs, + by_severity, + by_app, + error_logs, + error_logs_truncated, + ai_sessions, + }) +} + +#[cfg(test)] +#[path = "queries_tests.rs"] +mod tests; + +#[cfg(test)] +#[path = "queries_graph_tests.rs"] +mod graph_tests; diff --git a/crates/shared/cortex/storage-sqlite/src/queries_graph_tests.rs b/crates/shared/cortex/storage-sqlite/src/queries_graph_tests.rs new file mode 100644 index 00000000..b0c06d7c --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/queries_graph_tests.rs @@ -0,0 +1,401 @@ +//! Tests for graph-anchored traversal and log fan-out +//! (`graph_walk_n_hops`, `search_logs_from_graph_related_entities`). + +use super::*; +use crate::graph::{ + self, GRAPH_WALK_MAX_DEPTH, REL_RUNS_ON, graph_walk_n_hops, refresh_graph_projection, +}; +use crate::{LogBatchEntry, init_pool, insert_logs_batch}; + +fn test_pool(name: &str) -> (tempfile::TempDir, DbPool) { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test(dir.path().join(name))).unwrap(); + (dir, pool) +} + +fn insert_entity(conn: &rusqlite::Connection, entity_type: &str, key: &str) -> i64 { + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, trust_level) + VALUES (?1, ?2, ?2, 'verified')", + rusqlite::params![entity_type, key], + ) + .unwrap(); + conn.last_insert_rowid() +} + +fn insert_rel(conn: &rusqlite::Connection, src: i64, dst: i64, rel: &str) { + conn.execute( + "INSERT INTO graph_relationships + (relationship_key, src_entity_id, dst_entity_id, relationship_type, + reason_code, trust_level, confidence, last_seen_at) + VALUES (?1, ?2, ?3, ?4, 'log_app_name', 'inferred', 0.5, + '2026-01-01T00:00:00Z')", + rusqlite::params![format!("{src}:{rel}:{dst}"), src, dst, rel], + ) + .unwrap(); +} + +fn keys(entities: &[graph::GraphWalkEntity]) -> Vec { + let mut k: Vec = entities.iter().map(|e| e.canonical_key.clone()).collect(); + k.sort(); + k +} + +#[test] +fn graph_walk_single_hop_returns_seed_and_neighbour() { + let (_d, pool) = test_pool("walk-1hop.db"); + let conn = pool.get().unwrap(); + let a = insert_entity(&conn, graph::ENTITY_TYPE_HOST, "host-a"); + let b = insert_entity(&conn, graph::ENTITY_TYPE_APP, "app-b"); + insert_rel(&conn, b, a, REL_RUNS_ON); + + let reached = graph_walk_n_hops(&conn, &["host-a".to_string()], 1).unwrap(); + assert_eq!(keys(&reached), vec!["app-b", "host-a"]); +} + +#[test] +fn graph_walk_two_hops_respects_depth() { + let (_d, pool) = test_pool("walk-2hop.db"); + let conn = pool.get().unwrap(); + let a = insert_entity(&conn, graph::ENTITY_TYPE_HOST, "a"); + let b = insert_entity(&conn, graph::ENTITY_TYPE_APP, "b"); + let c = insert_entity(&conn, graph::ENTITY_TYPE_APP, "c"); + insert_rel(&conn, a, b, REL_RUNS_ON); + insert_rel(&conn, b, c, REL_RUNS_ON); + + // depth 1 reaches only the direct neighbour. + let depth1 = graph_walk_n_hops(&conn, &["a".to_string()], 1).unwrap(); + assert_eq!(keys(&depth1), vec!["a", "b"]); + + // depth 2 reaches the far node too. + let depth2 = graph_walk_n_hops(&conn, &["a".to_string()], 2).unwrap(); + assert_eq!(keys(&depth2), vec!["a", "b", "c"]); +} + +#[test] +fn graph_walk_terminates_on_cycle() { + let (_d, pool) = test_pool("walk-cycle.db"); + let conn = pool.get().unwrap(); + let a = insert_entity(&conn, graph::ENTITY_TYPE_HOST, "a"); + let b = insert_entity(&conn, graph::ENTITY_TYPE_APP, "b"); + let c = insert_entity(&conn, graph::ENTITY_TYPE_APP, "c"); + // 3-cycle: a → b → c → a + insert_rel(&conn, a, b, REL_RUNS_ON); + insert_rel(&conn, b, c, REL_RUNS_ON); + insert_rel(&conn, c, a, REL_RUNS_ON); + + // UNION (not UNION ALL) dedups visited rows, so the walk converges. + let reached = graph_walk_n_hops(&conn, &["a".to_string()], GRAPH_WALK_MAX_DEPTH).unwrap(); + assert_eq!(keys(&reached), vec!["a", "b", "c"]); +} + +#[test] +fn graph_walk_clamps_depth_and_handles_empty_seed() { + let (_d, pool) = test_pool("walk-clamp.db"); + let conn = pool.get().unwrap(); + let a = insert_entity(&conn, graph::ENTITY_TYPE_HOST, "a"); + let b = insert_entity(&conn, graph::ENTITY_TYPE_APP, "b"); + insert_rel(&conn, a, b, REL_RUNS_ON); + + // depth 0 is clamped up to 1 (still reaches the direct neighbour). + let clamped_low = graph_walk_n_hops(&conn, &["a".to_string()], 0).unwrap(); + assert_eq!(keys(&clamped_low), vec!["a", "b"]); + + // Oversized depth is clamped to the ceiling without error. + let clamped_high = graph_walk_n_hops(&conn, &["a".to_string()], 250).unwrap(); + assert_eq!(keys(&clamped_high), vec!["a", "b"]); + + // Empty seed set returns empty. + assert!(graph_walk_n_hops(&conn, &[], 3).unwrap().is_empty()); +} + +#[test] +fn graph_walk_uses_relationship_indexes() { + let (_d, pool) = test_pool("walk-plan.db"); + let conn = pool.get().unwrap(); + let plan: Vec = conn + .prepare( + "EXPLAIN QUERY PLAN + WITH RECURSIVE graph_walk(entity_id, depth) AS ( + SELECT id, 0 FROM graph_entities WHERE canonical_key IN ('a') + UNION + SELECT CASE WHEN r.src_entity_id = gw.entity_id + THEN r.dst_entity_id ELSE r.src_entity_id END, + gw.depth + 1 + FROM graph_relationships r + JOIN graph_walk gw + ON r.src_entity_id = gw.entity_id OR r.dst_entity_id = gw.entity_id + WHERE gw.depth < 6 + ) + SELECT DISTINCT e.entity_type, e.canonical_key + FROM graph_entities e JOIN graph_walk gw ON e.id = gw.entity_id", + ) + .unwrap() + .query_map([], |row| row.get::<_, String>(3)) + .unwrap() + .collect::>>() + .unwrap(); + // The recursive relationship join must be index-served on src/dst entity id, + // never a full table scan of graph_relationships. + assert!( + !plan.iter().any(|p| p == "SCAN graph_relationships"), + "recursive hop must not full-scan graph_relationships: {plan:?}" + ); +} + +fn syslog_row(ts: &str, host: &str, app: &str) -> LogBatchEntry { + LogBatchEntry { + timestamp: ts.to_string(), + hostname: host.to_string(), + facility: None, + severity: "info".to_string(), + app_name: Some(app.to_string()), + process_id: None, + message: format!("{app} message"), + raw: format!("{app} message"), + source_ip: "10.0.0.5:514".to_string(), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: Some(r#"{"source_kind":"syslog-udp"}"#.to_string()), + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + } +} + +fn agent_command_row(ts: &str, host: &str, session: &str, cwd: &str) -> LogBatchEntry { + let mut row = syslog_row(ts, host, "claude"); + row.source_ip = format!("agent-command://{host}/claude/{session}"); + row.ai_tool = Some("claude".to_string()); + row.ai_project = Some(cwd.to_string()); + row.ai_session_id = Some(session.to_string()); + row.metadata_json = Some(format!( + r#"{{"source_kind":"agent-command","agent_command":{{"cwd":"{cwd}"}}}}"# + )); + row +} + +#[test] +fn search_logs_from_graph_fans_out_from_session_to_host_logs() { + let _guard = graph::GRAPH_TEST_LOCK.lock(); + let (_d, pool) = test_pool("fanout.db"); + + insert_logs_batch( + &pool, + &[ + // Session sess-7 ran commands on devhost (links ai_session → host). + agent_command_row( + "2026-01-01T00:00:00Z", + "devhost", + "sess-7", + "/home/jmagar/workspace/cortex", + ), + // Plain syslog on the same host — should be reached via the host edge. + syslog_row("2026-01-01T00:01:00Z", "devhost", "swag"), + // Unrelated host — must NOT be returned. + syslog_row("2026-01-01T00:02:00Z", "edgehost", "authelia"), + ], + ) + .unwrap(); + refresh_graph_projection(&pool).unwrap(); + + // Seed from the AI session entity; traversal reaches host:devhost. + let session_key = "cortex:claude:sess-7".to_string(); + let logs = search_logs_from_graph_related_entities( + &pool, + &[session_key], + 2, + None, + None, + None, + 100, + HostFanoutScope::WalkReached, + ) + .unwrap(); + + let hosts: std::collections::HashSet<&str> = logs.iter().map(|l| l.hostname.as_str()).collect(); + assert!( + hosts.contains("devhost"), + "must fan out to devhost logs: {hosts:?}" + ); + assert!( + !hosts.contains("edgehost"), + "unrelated host must be excluded" + ); + assert!( + logs.iter().any(|l| l.app_name.as_deref() == Some("swag")), + "the swag syslog row on the related host must be returned" + ); +} + +#[test] +fn search_logs_from_graph_respects_source_kind_filter() { + let _guard = graph::GRAPH_TEST_LOCK.lock(); + let (_d, pool) = test_pool("fanout-source.db"); + + insert_logs_batch( + &pool, + &[ + agent_command_row( + "2026-01-01T00:00:00Z", + "devhost", + "sess-7", + "/home/jmagar/workspace/cortex", + ), + syslog_row("2026-01-01T00:01:00Z", "devhost", "swag"), + ], + ) + .unwrap(); + refresh_graph_projection(&pool).unwrap(); + + // Restrict to syslog-udp only → the agent-command row is filtered out. + let logs = search_logs_from_graph_related_entities( + &pool, + &["cortex:claude:sess-7".to_string()], + 2, + None, + None, + Some(&[SourceKind::SyslogUdp]), + 100, + HostFanoutScope::WalkReached, + ) + .unwrap(); + assert!(!logs.is_empty(), "syslog row should survive the filter"); + assert!( + logs.iter() + .all(|l| !l.source_ip.starts_with("agent-command://")), + "agent-command rows must be excluded by the source_kind filter" + ); +} + +#[test] +fn search_logs_from_graph_empty_for_unknown_seed() { + let (_d, pool) = test_pool("fanout-empty.db"); + let logs = search_logs_from_graph_related_entities( + &pool, + &["does-not-exist".to_string()], + 2, + None, + None, + None, + 100, + HostFanoutScope::WalkReached, + ) + .unwrap(); + assert!(logs.is_empty()); +} + +#[test] +fn topic_resolving_logical_service_with_no_instances_reports_degraded() { + let (_dir, pool) = test_pool("topic-zero-instance-degraded.db"); + // Stale/unbuilt projection: the logical service entity exists but has no + // `instance_of` service instances (e.g. right after migration 41 before + // `cortex graph rebuild` runs). The resolved entity must surface as + // degraded so the empty service timeline is explained, never silent. + { + let conn = pool.get().unwrap(); + insert_entity(&conn, graph::ENTITY_TYPE_LOGICAL_SERVICE, "plex"); + } + let inputs = + topic_correlate_inputs(&pool, &["plex".to_string()], 2, None, None, None, 100).unwrap(); + let entity = inputs + .resolved + .iter() + .find(|e| e.entity_type == graph::ENTITY_TYPE_LOGICAL_SERVICE && e.canonical_key == "plex") + .expect("logical service must resolve"); + assert_eq!(entity.resolver_status, ResolverStatus::Degraded); + assert!(inputs.logs.is_empty(), "no instances → no service fan-out"); +} + +#[test] +fn topic_correlate_app_seed_does_not_fan_out_to_whole_host() { + let (_dir, pool) = test_pool("topic-app-seed-no-host-fanout.db"); + // Bare `plex` app label (no agent-docker metadata) plus an unrelated + // kernel row on the same host. + insert_logs_batch( + &pool, + &[ + syslog_row("2026-01-01T00:00:00Z", "nashost", "plex"), + syslog_row("2026-01-01T00:01:00Z", "nashost", "kernel"), + ], + ) + .unwrap(); + // Graph: app:plex —emitted_by→ host:nashost (log-identity edge). + { + let conn = pool.get().unwrap(); + let app = insert_entity(&conn, graph::ENTITY_TYPE_APP, "plex"); + let host = insert_entity(&conn, graph::ENTITY_TYPE_HOST, "nashost"); + insert_rel(&conn, app, host, graph::REL_EMITTED_BY); + } + + let inputs = + topic_correlate_inputs(&pool, &["plex".to_string()], 2, None, None, None, 100).unwrap(); + // The topic resolves to the raw app entity and the walk reaches + // host:nashost, but the transitively reached host must never drive + // host-wide log inclusion labelled `resolved`. + assert!( + inputs + .resolved + .iter() + .any(|entity| entity.entity_type == graph::ENTITY_TYPE_APP + && entity.canonical_key == "plex"), + "topic must resolve the raw app entity: {:?}", + inputs.resolved + ); + assert!( + !inputs.logs.iter().any(|row| { + row.entry.app_name.as_deref() == Some("kernel") + && row.resolver_status == ResolverStatus::Resolved + }), + "unrelated kernel row on the host must not be included as resolved: {:?}", + inputs + .logs + .iter() + .map(|row| ( + row.entry.app_name.clone(), + row.resolver_status, + row.fallback_kind.clone() + )) + .collect::>() + ); +} + +#[test] +fn ambiguous_prefix_candidates_surface_without_log_fanout() { + let (_dir, pool) = test_pool("topic-ambiguous-prefix.db"); + insert_logs_batch( + &pool, + &[syslog_row( + "2026-01-01T00:00:00Z", + "nashost", + "plexmediaserver", + )], + ) + .unwrap(); + { + let conn = pool.get().unwrap(); + insert_entity(&conn, graph::ENTITY_TYPE_LOGICAL_SERVICE, "plexmediaserver"); + } + // "plexmedia" prefix-matches logical_service:plexmediaserver: the + // candidate must surface as ambiguous but contribute ZERO log fan-out. + let inputs = + topic_correlate_inputs(&pool, &["plexmedia".to_string()], 2, None, None, None, 100) + .unwrap(); + let entity = inputs + .resolved + .iter() + .find(|e| e.canonical_key == "plexmediaserver") + .expect("prefix candidate must surface"); + assert_eq!(entity.match_kind, "prefix"); + assert_eq!(entity.resolver_status, ResolverStatus::Ambiguous); + assert!( + inputs.logs.is_empty(), + "ambiguous candidates must contribute zero log fan-out" + ); +} diff --git a/crates/shared/cortex/storage-sqlite/src/queries_hosts.rs b/crates/shared/cortex/storage-sqlite/src/queries_hosts.rs new file mode 100644 index 00000000..bec4ba96 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/queries_hosts.rs @@ -0,0 +1,127 @@ +use anyhow::Result; + +use super::models::HostEntry; +use super::pool::DbPool; + +/// Lowercase, trim, and strip trailing dots from a hostname so case and a +/// trailing FQDN dot don't split one machine into several host rows. Does not +/// fold FQDNs to short names — that's [`canonical_host_keys`]'s data-driven step. +fn case_fold_host(raw: &str) -> String { + raw.trim().trim_end_matches('.').to_ascii_lowercase() +} + +/// Map each input hostname to its canonical identity, applying two folds: +/// 1. **Case / trailing-dot** via [`case_fold_host`] (`BACKUPHOST` → `backuphost`). +/// 2. **FQDN → short name, only when the short name independently exists** among +/// the inputs (`nashost.` → `nashost` when a bare `nashost` is present, +/// but `host.docker.internal` is left alone). This never invents a merge that +/// could mask a distinct machine. +/// +/// Shared by [`dedupe_hosts`] (the `hosts` action) and `clock_skew` so every +/// host-keyed view collapses the same case/FQDN variants. +pub(crate) fn canonical_host_keys( + hostnames: &[String], +) -> std::collections::HashMap { + let cased: Vec<(String, String)> = hostnames + .iter() + .map(|h| (h.clone(), case_fold_host(h))) + .collect(); + let shorts: std::collections::HashSet<&str> = cased + .iter() + .filter(|(_, c)| !c.is_empty() && !c.contains('.')) + .map(|(_, c)| c.as_str()) + .collect(); + cased + .iter() + .map(|(raw, c)| { + let canonical = match c.split_once('.') { + Some((head, _)) if shorts.contains(head) => head.to_string(), + _ => c.clone(), + }; + (raw.clone(), canonical) + }) + .collect() +} + +/// Merge host rows that refer to the same machine. Two folds are applied: +/// 1. **Case / trailing-dot** — `BACKUPHOST` and `backuphost` collapse, `WINHOST`→`winhost`. +/// 2. **FQDN → short name, only when the short name independently exists** as +/// its own host. So `nashost.example.ts.net` folds into `nashost` +/// (a real host), but `host.docker.internal` is left alone because no bare +/// `host` row exists — we never invent a merge that could mask a distinct +/// machine. +/// +/// Blank hostnames are excluded because they cannot be selected or correlated. +/// Other ambiguous self-identifiers (`localhost`, `host:user` forms with no +/// dot) are left untouched: resolving those to a real machine needs the +/// network-verified `source_ip`, which is a deferred follow-up. +/// Merged rows sum `log_count`, take the earliest `first_seen` and latest +/// `last_seen`, and display the canonical (lowercased) name. +pub(super) fn dedupe_hosts(rows: Vec) -> Vec { + let names: Vec = rows.iter().map(|h| h.hostname.clone()).collect(); + let canon = canonical_host_keys(&names); + let mut merged: std::collections::HashMap = std::collections::HashMap::new(); + let mut order: Vec = Vec::new(); + for entry in rows { + let canonical = canon + .get(&entry.hostname) + .cloned() + .unwrap_or_else(|| case_fold_host(&entry.hostname)); + if canonical.is_empty() { + continue; + } + match merged.get_mut(&canonical) { + Some(acc) => { + acc.log_count += entry.log_count; + if entry.first_seen < acc.first_seen { + acc.first_seen = entry.first_seen.clone(); + } + if entry.last_seen > acc.last_seen { + acc.last_seen = entry.last_seen.clone(); + } + } + None => { + order.push(canonical.clone()); + merged.insert( + canonical.clone(), + HostEntry { + hostname: canonical.clone(), + first_seen: entry.first_seen.clone(), + last_seen: entry.last_seen.clone(), + log_count: entry.log_count, + }, + ); + } + } + } + let mut out: Vec = order + .into_iter() + .map(|k| merged.remove(&k).expect("key inserted above")) + .collect(); + out.sort_by(|a, b| b.last_seen.cmp(&a.last_seen)); + out +} + +/// List all known hosts with stats, deduplicated across case and FQDN variants. +pub fn list_hosts(pool: &DbPool) -> Result> { + let conn = pool.get()?; + let mut stmt = conn.prepare( + "SELECT hostname, first_seen, last_seen, log_count FROM hosts ORDER BY last_seen DESC", + )?; + + let rows = stmt.query_map([], |row| { + Ok(HostEntry { + hostname: row.get(0)?, + first_seen: row.get(1)?, + last_seen: row.get(2)?, + log_count: row.get(3)?, + }) + })?; + + let rows = rows.collect::>>()?; + Ok(dedupe_hosts(rows)) +} + +#[cfg(test)] +#[path = "queries_hosts_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/queries_hosts_tests.rs b/crates/shared/cortex/storage-sqlite/src/queries_hosts_tests.rs new file mode 100644 index 00000000..ea5f7bee --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/queries_hosts_tests.rs @@ -0,0 +1,114 @@ +use super::*; + +fn host_entry(name: &str, first: &str, last: &str, count: i64) -> HostEntry { + HostEntry { + hostname: name.to_string(), + first_seen: first.to_string(), + last_seen: last.to_string(), + log_count: count, + } +} + +#[test] +fn dedupe_hosts_folds_case_variants() { + let out = dedupe_hosts(vec![ + host_entry( + "BACKUPHOST", + "2026-06-01T00:00:00Z", + "2026-06-10T00:00:00Z", + 10, + ), + host_entry( + "backuphost", + "2026-06-02T00:00:00Z", + "2026-06-12T00:00:00Z", + 5, + ), + ]); + assert_eq!(out.len(), 1); + assert_eq!(out[0].hostname, "backuphost"); + assert_eq!(out[0].log_count, 15); + assert_eq!(out[0].first_seen, "2026-06-01T00:00:00Z"); // earliest + assert_eq!(out[0].last_seen, "2026-06-12T00:00:00Z"); // latest +} + +#[test] +fn dedupe_hosts_folds_fqdn_into_existing_short_name() { + let out = dedupe_hosts(vec![ + host_entry( + "nashost", + "2026-06-01T00:00:00Z", + "2026-06-10T00:00:00Z", + 100, + ), + host_entry( + "nashost.example.ts.net", + "2026-06-03T00:00:00Z", + "2026-06-09T00:00:00Z", + 7, + ), + ]); + assert_eq!(out.len(), 1); + assert_eq!(out[0].hostname, "nashost"); + assert_eq!(out[0].log_count, 107); +} + +#[test] +fn dedupe_hosts_keeps_fqdn_when_no_matching_short_name() { + // No bare `host` row exists, so `host.docker.internal` must NOT be folded to + // `host` — folding there would invent a merge and could mask a real machine. + let out = dedupe_hosts(vec![host_entry( + "host.docker.internal", + "2026-06-01T00:00:00Z", + "2026-06-10T00:00:00Z", + 42, + )]); + assert_eq!(out.len(), 1); + assert_eq!(out[0].hostname, "host.docker.internal"); +} + +#[test] +fn dedupe_hosts_leaves_ambiguous_self_identifiers_untouched() { + // localhost and dotless host:user forms are deferred (need source_ip). + let out = dedupe_hosts(vec![ + host_entry( + "localhost", + "2026-06-01T00:00:00Z", + "2026-06-10T00:00:00Z", + 3, + ), + host_entry("devhost", "2026-06-01T00:00:00Z", "2026-06-11T00:00:00Z", 9), + host_entry( + "devhost:jmagar", + "2026-06-01T00:00:00Z", + "2026-06-05T00:00:00Z", + 2, + ), + ]); + let names: std::collections::HashSet<&str> = out.iter().map(|h| h.hostname.as_str()).collect(); + assert!(names.contains("localhost")); + assert!(names.contains("devhost")); + assert!(names.contains("devhost:jmagar")); // colon, no dot → not folded into devhost + assert_eq!(out.len(), 3); +} + +#[test] +fn dedupe_hosts_excludes_blank_hostnames() { + let out = dedupe_hosts(vec![ + host_entry("", "2026-06-01T00:00:00Z", "2026-06-10T00:00:00Z", 3), + host_entry(" ", "2026-06-01T00:00:00Z", "2026-06-10T00:00:00Z", 4), + host_entry("devhost", "2026-06-01T00:00:00Z", "2026-06-11T00:00:00Z", 9), + ]); + assert_eq!(out.len(), 1); + assert_eq!(out[0].hostname, "devhost"); +} + +#[test] +fn dedupe_hosts_orders_by_last_seen_desc() { + let out = dedupe_hosts(vec![ + host_entry("alpha", "2026-06-01T00:00:00Z", "2026-06-05T00:00:00Z", 1), + host_entry("bravo", "2026-06-01T00:00:00Z", "2026-06-20T00:00:00Z", 1), + ]); + assert_eq!(out[0].hostname, "bravo"); // most recent first + assert_eq!(out[1].hostname, "alpha"); +} diff --git a/crates/shared/cortex/storage-sqlite/src/queries_service_instances.rs b/crates/shared/cortex/storage-sqlite/src/queries_service_instances.rs new file mode 100644 index 00000000..8ce56587 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/queries_service_instances.rs @@ -0,0 +1,401 @@ +//! Service-instance and topic-resolution query glue: the resolver-decision +//! consumers that turn canonical `logical_service` / `service_instance` +//! graph identity, and free-text topic terms, into log query predicates. +//! +//! Extracted from `queries.rs` (syslog-mcp-6ipjl). `topic_correlate_inputs` +//! (the single entry point that ties these helpers together with the +//! general-purpose graph-walk and graph-related-entities log fan-out) stays +//! in `queries.rs` because it is the natural integration point, not +//! resolver-specific glue in its own right. + +use anyhow::Result; + +use cortex_ingest_core::SourceKind; + +use super::entity_resolution::{INCLUSION_SERVICE_INSTANCE, ResolverStatus}; +use super::graph; +use super::models::{GraphRelatedLogEntry, LogEntry, ResolvedTopicEntity}; +use super::pool::DbPool; +use super::queries::{FTS_SELECT_COLS, bind_in_list, map_row}; + +/// Escape SQL `LIKE` wildcards (`%`, `_`) and the escape character itself +/// (`\`) so a literal value can be embedded in a pattern used with +/// `LIKE ? ESCAPE '\'`. +fn escape_like(value: &str) -> String { + let mut out = String::with_capacity(value.len()); + for ch in value.chars() { + if matches!(ch, '%' | '_' | '\\') { + out.push('\\'); + } + out.push(ch); + } + out +} + +/// Build the shared per-arm `since`/`until`/`source_kind` filter tail used by +/// the UNION ALL fan-out queries. Returns the SQL fragment (leading ` AND …`) +/// and the bindings it consumes, in order. +fn log_window_filter_tail( + since: Option<&str>, + until: Option<&str>, + source_kinds: Option<&[SourceKind]>, +) -> (String, Vec) { + let mut sql = String::new(); + let mut bindings: Vec = Vec::new(); + if let Some(since) = since { + sql.push_str(" AND l.timestamp >= ?"); + bindings.push(rusqlite::types::Value::Text(since.to_string())); + } + if let Some(until) = until { + sql.push_str(" AND l.timestamp <= ?"); + bindings.push(rusqlite::types::Value::Text(until.to_string())); + } + if let Some(kinds) = source_kinds + && !kinds.is_empty() + { + let kind_strs: Vec = kinds.iter().map(|k| k.as_str().to_string()).collect(); + let ph = bind_in_list(&mut bindings, &kind_strs); + sql.push_str(&format!( + " AND json_extract(l.metadata_json, '$.source_kind') IN ({ph})" + )); + } + (sql, bindings) +} + +/// Run per-arm `UNION ALL` log queries, then merge the arms newest-first, +/// drop duplicate ids, and bound to `limit`. Each arm carries its own +/// `LIMIT` pushdown, so this reconciles them into one ordered, bounded +/// result. Empty `arms` yields no rows. +fn run_union_all_log_arms( + conn: &rusqlite::Connection, + arms: &[String], + bindings: &[rusqlite::types::Value], + limit: usize, +) -> Result> { + if arms.is_empty() { + return Ok(Vec::new()); + } + let sql = arms.join(" UNION ALL "); + let mut stmt = conn.prepare(&sql)?; + let mut logs = stmt + .query_map(rusqlite::params_from_iter(bindings.iter()), map_row)? + .collect::>>()?; + logs.sort_by(|a, b| b.timestamp.cmp(&a.timestamp).then_with(|| b.id.cmp(&a.id))); + logs.dedup_by_key(|entry| entry.id); + logs.truncate(limit); + Ok(logs) +} + +/// Fetch logs that belong to specific service instances (`host/service` +/// keys) using indexed service-scoped predicates: exact hostname AND +/// (app label equal to the service name OR starting with `{service}/`, or +/// the structured agent-docker compose service matches). This is the +/// canonical service log fan-out — it never expands to all logs on the +/// host. +pub fn search_logs_for_service_instances( + pool: &DbPool, + service_instance_keys: &[String], + since: Option<&str>, + until: Option<&str>, + source_kinds: Option<&[SourceKind]>, + limit: usize, +) -> Result> { + if service_instance_keys.is_empty() { + return Ok(Vec::new()); + } + let limit = limit.clamp(1, 1000); + let conn = pool.get()?; + let (tail_sql, tail_bindings) = log_window_filter_tail(since, until, source_kinds); + // Per-key UNION ALL arms, each with its own LIMIT pushdown so every arm + // is an index search on `idx_logs_host_time` (hostname = ?, timestamp + // descending) with no full-set temp b-tree. Rows merge in Rust below. + let mut arms: Vec = Vec::new(); + let mut bindings: Vec = Vec::new(); + for key in service_instance_keys { + let Some((host, service)) = super::entity_resolution::split_service_instance_key(key) + else { + tracing::debug!( + key = %key, + "discarding non-canonical service_instance key in service log fan-out" + ); + continue; + }; + arms.push(format!( + "SELECT * FROM (SELECT {FTS_SELECT_COLS} + FROM logs l + WHERE l.hostname = ? AND (l.app_name = ? OR l.app_name LIKE ? ESCAPE '\\' \ + OR json_extract(l.metadata_json, '$.agent_docker.compose_service') = ?){tail_sql} + ORDER BY l.timestamp DESC, l.id DESC + LIMIT ?)" + )); + bindings.push(host.to_string().into()); + bindings.push(service.to_string().into()); + bindings.push(format!("{}/%", escape_like(service)).into()); + bindings.push(service.to_string().into()); + bindings.extend(tail_bindings.iter().cloned()); + bindings.push((limit as i64).into()); + } + let entries = run_union_all_log_arms(&conn, &arms, &bindings, limit)?; + Ok(entries + .into_iter() + .map(|entry| GraphRelatedLogEntry { + entry, + inclusion_reason: INCLUSION_SERVICE_INSTANCE.to_string(), + resolver_status: ResolverStatus::Resolved, + fallback_kind: None, + }) + .collect()) +} + +/// Resolve the `service_instance` keys linked to the given logical services +/// via non-refuted `instance_of` edges. +pub(super) fn service_instances_of_logical_services( + conn: &rusqlite::Connection, + logical_keys: &[String], +) -> Result> { + if logical_keys.is_empty() { + return Ok(Vec::new()); + } + let placeholders = vec!["?"; logical_keys.len()].join(", "); + let sql = format!( + "SELECT inst.canonical_key + FROM graph_relationships r + JOIN graph_entities inst ON inst.id = r.src_entity_id + JOIN graph_entities logical ON logical.id = r.dst_entity_id + WHERE r.relationship_type = ? + AND r.trust_level != 'refuted' + AND inst.entity_type = ? + AND logical.entity_type = ? + AND logical.canonical_key IN ({placeholders})" + ); + let mut bindings: Vec = vec![ + graph::REL_INSTANCE_OF.to_string().into(), + graph::ENTITY_TYPE_SERVICE_INSTANCE.to_string().into(), + graph::ENTITY_TYPE_LOGICAL_SERVICE.to_string().into(), + ]; + bindings.extend( + logical_keys + .iter() + .map(|key| rusqlite::types::Value::Text(key.clone())), + ); + let mut stmt = conn.prepare(&sql)?; + let keys = stmt + .query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + row.get::<_, String>(0) + })? + .collect::>>()?; + Ok(keys) +} + +/// Fetch logs by exact hostname list (bounded), used only by the explicit +/// degraded host-context fallback. +pub(super) fn search_logs_by_hostnames( + pool: &DbPool, + hostnames: &[String], + since: Option<&str>, + until: Option<&str>, + source_kinds: Option<&[SourceKind]>, + limit: usize, +) -> Result> { + if hostnames.is_empty() { + return Ok(Vec::new()); + } + let limit = limit.clamp(1, 1000); + let conn = pool.get()?; + let (tail_sql, tail_bindings) = log_window_filter_tail(since, until, source_kinds); + // Per-hostname UNION ALL arms with LIMIT pushdown (same shape as + // `search_logs_for_service_instances`): each arm is an index search on + // `idx_logs_host_time`, merged and re-truncated in Rust. + let mut arms: Vec = Vec::new(); + let mut bindings: Vec = Vec::new(); + for hostname in hostnames { + arms.push(format!( + "SELECT * FROM (SELECT {FTS_SELECT_COLS} + FROM logs l + WHERE l.hostname = ?{tail_sql} + ORDER BY l.timestamp DESC, l.id DESC + LIMIT ?)" + )); + bindings.push(hostname.clone().into()); + bindings.extend(tail_bindings.iter().cloned()); + bindings.push((limit as i64).into()); + } + run_union_all_log_arms(&conn, &arms, &bindings, limit) +} + +/// Escape a term for safe embedding in a SQLite `GLOB` prefix pattern, then +/// append the trailing `*` wildcard. GLOB's own metacharacters (`*`, `?`, +/// `[`) are wrapped in a single-character bracket class (e.g. `*` -> `[*]`) +/// so they match literally instead of acting as wildcards. +fn glob_prefix_pattern(term: &str) -> String { + let mut pattern = String::with_capacity(term.len() + 1); + for ch in term.chars() { + match ch { + '*' | '?' | '[' => { + pattern.push('['); + pattern.push(ch); + pattern.push(']'); + } + _ => pattern.push(ch), + } + } + pattern.push('*'); + pattern +} + +/// Resolve topic terms to graph entities by exact / prefix / label / alias +/// match. `terms` must already be lowercased. Strongest match wins per entity +/// (exact > prefix > label > alias). Capped per term and overall. +/// +/// Query-plan / complexity notes (syslog-mcp-csukc): +/// - The exact (`canonical_key = term`) and prefix (`canonical_key` starts +/// with `term`) tiers run as a single statement using `GLOB` rather than +/// `LIKE ?1 || '%'` for the prefix condition. SQLite's LIKE-to-range-scan +/// optimization only activates under `PRAGMA case_sensitive_like = ON` +/// (a connection-wide setting we don't want to flip for one query), while +/// `GLOB` gets the equivalent range-scan optimization unconditionally. +/// `canonical_key` is always ASCII-lowercased at write time +/// (`normalize_key` in `graph.rs`) and callers already lowercase `terms`, +/// so GLOB's case sensitivity is a non-issue here. Both disjuncts hit +/// `idx_graph_entities_canonical_key` via SQLite's `MULTI-INDEX OR` plan — +/// confirmed via `EXPLAIN QUERY PLAN` — so this tier is an indexed lookup, +/// not a table scan. +/// - The label tier (`lower(display_label) LIKE '%term%'`) is a genuine +/// substring match. SQLite cannot use *any* B-tree index for a +/// leading-wildcard LIKE — that's a fundamental limitation of B-tree +/// indexes, not a missing-index problem, and no index we could add here +/// changes that. This tier is therefore an O(n) scan of `graph_entities` +/// per term where it runs. +/// - To bound the damage, the label tier's query only executes when the +/// indexed tier didn't already fill `PER_TERM_CAP` matches for that term +/// (its own `LIMIT` is `PER_TERM_CAP` minus however many indexed hits were +/// found). A term that resolves cleanly via exact/prefix match on +/// `canonical_key` skips the full scan entirely; only terms that are +/// genuinely fuzzy, or typos with few/no key matches, still pay the O(n) +/// cost — same worst case as before, no longer paid unconditionally by +/// every term regardless of match quality. +/// - This still degrades linearly with `graph_entities` row count in the +/// fuzzy case. Removing that residual cost would require a trigram/FTS5 +/// index over `display_label`, which is a larger structural change than +/// this hardening pass covers. +pub(super) fn resolve_topic_entities( + conn: &rusqlite::Connection, + terms: &[String], +) -> Result> { + const PER_TERM_CAP: usize = 25; + const TOTAL_CAP: usize = 100; + // (entity_type, canonical_key) -> match priority (lower = stronger). + let mut best: std::collections::HashMap<(String, String), u8> = + std::collections::HashMap::new(); + + // Tier 0/1 (exact / prefix): index-backed, see doc comment above. + let mut key_stmt = conn.prepare( + "SELECT entity_type, canonical_key, + CASE WHEN canonical_key = ?1 THEN 0 ELSE 1 END AS pri + FROM graph_entities + WHERE canonical_key = ?1 OR canonical_key GLOB ?2 + ORDER BY pri + LIMIT ?3", + )?; + // Tier 2 (label substring, fallback-only): unavoidable full scan, see + // doc comment above. Only invoked when the indexed tier above didn't + // already fill PER_TERM_CAP for the current term. + let mut label_stmt = conn.prepare( + "SELECT entity_type, canonical_key + FROM graph_entities + WHERE lower(display_label) LIKE '%' || ?1 || '%' ESCAPE '\\' + LIMIT ?2", + )?; + let mut alias = conn.prepare( + "SELECT e.entity_type, e.canonical_key + FROM graph_entity_aliases a + JOIN graph_entities e ON e.id = a.entity_id + WHERE a.alias_key = ?1 + LIMIT ?2", + )?; + + for term in terms { + let glob_pattern = glob_prefix_pattern(term); + let mut key_hits = 0usize; + let rows = key_stmt.query_map( + rusqlite::params![term, glob_pattern, PER_TERM_CAP as i64], + |row| { + Ok(( + row.get::<_, String>(0)?, + row.get::<_, String>(1)?, + row.get::<_, i64>(2)? as u8, + )) + }, + )?; + for row in rows { + let (entity_type, key, pri) = row?; + key_hits += 1; + let slot = best.entry((entity_type, key)).or_insert(u8::MAX); + *slot = (*slot).min(pri); + } + + // Only pay for the unindexable substring scan when the indexed tier + // left room under the per-term cap. + let label_limit = PER_TERM_CAP.saturating_sub(key_hits); + if label_limit > 0 { + let escaped_term = escape_like(term); + let label_rows = label_stmt + .query_map(rusqlite::params![escaped_term, label_limit as i64], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })?; + for row in label_rows { + let (entity_type, key) = row?; + let slot = best.entry((entity_type, key)).or_insert(u8::MAX); + *slot = (*slot).min(2); + } + } + + let alias_rows = alias.query_map(rusqlite::params![term, PER_TERM_CAP as i64], |row| { + Ok((row.get::<_, String>(0)?, row.get::<_, String>(1)?)) + })?; + for row in alias_rows { + let (entity_type, key) = row?; + // Alias match has priority 3 (weakest), only fills if nothing stronger. + let slot = best.entry((entity_type, key)).or_insert(u8::MAX); + *slot = (*slot).min(3); + } + } + + let mut resolved: Vec = best + .into_iter() + .map(|((entity_type, canonical_key), pri)| ResolvedTopicEntity { + entity_type, + canonical_key, + match_kind: match pri { + 0 => "exact", + 1 => "prefix", + 2 => "label", + _ => "alias", + }, + // Weak prefix/label candidates surface for the caller but never + // drive log fan-out (deterministic resolution only). + resolver_status: match pri { + 0 | 3 => ResolverStatus::Resolved, + _ => ResolverStatus::Ambiguous, + }, + }) + .collect(); + // Stable, deterministic ordering: strongest match first, then key. + resolved.sort_by(|a, b| { + let rank = |m: &str| match m { + "exact" => 0, + "prefix" => 1, + "label" => 2, + _ => 3, + }; + rank(a.match_kind) + .cmp(&rank(b.match_kind)) + .then_with(|| a.canonical_key.cmp(&b.canonical_key)) + }); + resolved.truncate(TOTAL_CAP); + Ok(resolved) +} + +#[cfg(test)] +#[path = "queries_service_instances_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/queries_service_instances_tests.rs b/crates/shared/cortex/storage-sqlite/src/queries_service_instances_tests.rs new file mode 100644 index 00000000..fe9e3002 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/queries_service_instances_tests.rs @@ -0,0 +1,413 @@ +use super::*; +use crate::config::StorageConfig; +use crate::graph; +use crate::{DbPool, LogBatchEntry, init_pool, insert_logs_batch}; + +fn test_pool(name: &str) -> (tempfile::TempDir, DbPool) { + let dir = tempfile::tempdir().unwrap(); + let pool = init_pool(&StorageConfig::for_test(dir.path().join(name))).unwrap(); + (dir, pool) +} + +fn insert_entity(conn: &rusqlite::Connection, entity_type: &str, key: &str) -> i64 { + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, trust_level) + VALUES (?1, ?2, ?2, 'verified')", + rusqlite::params![entity_type, key], + ) + .unwrap(); + conn.last_insert_rowid() +} + +fn syslog_row(ts: &str, host: &str, app: &str) -> LogBatchEntry { + LogBatchEntry { + timestamp: ts.to_string(), + hostname: host.to_string(), + facility: None, + severity: "info".to_string(), + app_name: Some(app.to_string()), + process_id: None, + message: format!("{app} message"), + raw: format!("{app} message"), + source_ip: "10.0.0.5:514".to_string(), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: Some(r#"{"source_kind":"syslog-udp"}"#.to_string()), + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + } +} + +#[test] +fn search_logs_for_service_instances_uses_service_predicates_not_host_fanout() { + let (_dir, pool) = test_pool("service-instance-predicates.db"); + let mut plex = syslog_row("2026-01-01T00:01:00Z", "nashost", "plex/plex/plex"); + plex.metadata_json = Some( + r#"{"source_kind":"agent-docker","agent_docker":{"host":"nashost","container_id":"abc","container_name":"plex","compose_service":"plex","stream":"stdout"}}"# + .to_string(), + ); + let mut exact = syslog_row("2026-01-01T00:02:00Z", "nashost", "plex"); + exact.metadata_json = None; + insert_logs_batch( + &pool, + &[ + syslog_row("2026-01-01T00:00:00Z", "nashost", "kernel"), + plex, + exact, + syslog_row("2026-01-01T00:03:00Z", "backuphost", "plex"), + ], + ) + .unwrap(); + + let rows = search_logs_for_service_instances( + &pool, + &["nashost/plex".to_string()], + None, + None, + None, + 50, + ) + .unwrap(); + // Matches: exact app label, prefixed nested label, structured compose + // service — all scoped to the instance's host. The kernel row on the + // same host and the other host's plex row stay out. + assert_eq!(rows.len(), 2); + assert!(rows.iter().all(|row| row.entry.hostname == "nashost")); + assert!( + rows.iter() + .all(|row| row.entry.app_name.as_deref() != Some("kernel")) + ); + assert!( + rows.iter() + .all(|row| row.inclusion_reason == "service_instance") + ); + assert!( + rows.iter() + .all(|row| row.resolver_status == ResolverStatus::Resolved) + ); + assert!(rows.iter().all(|row| row.fallback_kind.is_none())); +} + +#[test] +fn search_logs_for_service_instances_escapes_like_wildcards() { + let (_dir, pool) = test_pool("service-instance-like-escape.db"); + insert_logs_batch( + &pool, + &[ + // `_` is a LIKE single-char wildcard; an unescaped pattern + // `my_app/%` would match this row. + syslog_row("2026-01-01T00:00:00Z", "nashost", "myxapp/x"), + syslog_row("2026-01-01T00:01:00Z", "nashost", "my_app/x"), + ], + ) + .unwrap(); + let rows = search_logs_for_service_instances( + &pool, + &["nashost/my_app".to_string()], + None, + None, + None, + 50, + ) + .unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].entry.app_name.as_deref(), Some("my_app/x")); +} + +#[test] +fn service_instance_fanout_arms_use_index_search_without_temp_btree() { + let (_dir, pool) = test_pool("service-instance-plan.db"); + let conn = pool.get().unwrap(); + // Two-arm UNION ALL replica of the search_logs_for_service_instances + // shape: every arm must be an index search (no `SCAN logs`) and no arm + // may sort through a temp b-tree — LIMIT pushdown streams each arm off + // idx_logs_host_time in timestamp order. + let arm = "SELECT * FROM (SELECT l.id + FROM logs l + WHERE l.hostname = ? AND (l.app_name = ? OR l.app_name LIKE ? ESCAPE '\\' \ + OR json_extract(l.metadata_json, '$.agent_docker.compose_service') = ?) + ORDER BY l.timestamp DESC, l.id DESC + LIMIT ?)"; + let sql = format!("EXPLAIN QUERY PLAN {arm} UNION ALL {arm}"); + let plan: Vec = conn + .prepare(&sql) + .unwrap() + .query_map( + rusqlite::params![ + "nashost", + "plex", + "plex/%", + "plex", + 100, + "backuphost", + "plex", + "plex/%", + "plex", + 100 + ], + |row| row.get::<_, String>(3), + ) + .unwrap() + .collect::>>() + .unwrap(); + assert!( + plan.iter() + .any(|p| p.contains("USING INDEX idx_logs_host_time")), + "arms must search idx_logs_host_time: {plan:?}" + ); + assert!( + !plan.iter().any(|p| p.starts_with("SCAN logs")), + "no arm may full-scan logs: {plan:?}" + ); + assert!( + !plan.iter().any(|p| p.contains("TEMP B-TREE")), + "no temp b-tree sort allowed: {plan:?}" + ); +} + +#[test] +fn search_logs_for_service_instances_rejects_legacy_keys() { + let (_dir, pool) = test_pool("service-instance-legacy-keys.db"); + insert_logs_batch( + &pool, + &[syslog_row("2026-01-01T00:00:00Z", "nashost", "plex")], + ) + .unwrap(); + // Legacy shapes never split into (host, service) and yield no predicate. + let rows = search_logs_for_service_instances( + &pool, + &["nashost:plex".to_string(), "plex/plex/plex".to_string()], + None, + None, + None, + 50, + ) + .unwrap(); + assert!(rows.is_empty()); +} + +#[test] +fn service_instance_fanout_truncates_globally_newest_first_across_arms() { + let (_dir, pool) = test_pool("service-instance-union-truncation.db"); + // Two instances on different hosts, limit + 2 rows each, sharing the + // same timestamp set so the merge exercises the tie-break. + let mut rows = Vec::new(); + for i in 0..8 { + let ts = format!("2026-01-01T00:00:{i:02}Z"); + rows.push(syslog_row(&ts, "nashost", "plex")); + rows.push(syslog_row(&ts, "backuphost", "plex")); + } + insert_logs_batch(&pool, &rows).unwrap(); + + let limit = 6; + let out = search_logs_for_service_instances( + &pool, + &["nashost/plex".to_string(), "backuphost/plex".to_string()], + None, + None, + None, + limit, + ) + .unwrap(); + // UNION ALL arms each fetch up to `limit`; the Rust merge must truncate + // to exactly `limit` rows globally. + assert_eq!(out.len(), limit); + // Global newest-first with the (timestamp DESC, id DESC) tie-break. + for pair in out.windows(2) { + let (a, b) = (&pair[0].entry, &pair[1].entry); + assert!( + a.timestamp > b.timestamp || (a.timestamp == b.timestamp && a.id > b.id), + "rows must be (timestamp DESC, id DESC): {}#{} then {}#{}", + a.timestamp, + a.id, + b.timestamp, + b.id + ); + } + // The globally newest timestamp wins across both arms: both hosts' + // :07 rows lead the merged result. + assert!(out[0].entry.timestamp.ends_with(":07Z")); + assert!(out[1].entry.timestamp.ends_with(":07Z")); +} + +#[test] +fn mixed_case_hostname_does_not_match_canonical_instance_key() { + let (_dir, pool) = test_pool("service-instance-case-miss.db"); + insert_logs_batch( + &pool, + &[syslog_row("2026-01-01T00:00:00Z", "Nashost", "plex")], + ) + .unwrap(); + // Pins the case-sensitivity limitation: canonical keys are lowercase + // and the log predicates compare with SQLite's default BINARY + // collation, so a mixed-case syslog hostname ("Nashost") never matches + // the canonical instance key ("nashost/plex"). Documented in + // docs/contracts/investigation-graph.md; hostname case normalization + // at ingest is tracked separately. + let rows = search_logs_for_service_instances( + &pool, + &["nashost/plex".to_string()], + None, + None, + None, + 50, + ) + .unwrap(); + assert!(rows.is_empty(), "mixed-case hostname must miss (pinned)"); +} + +// -- syslog-mcp-csukc: resolve_topic_entities index-backed exact/prefix tier -- + +#[test] +fn glob_prefix_pattern_escapes_glob_metacharacters() { + // '*', '?' and '[' are GLOB metacharacters; they must be wrapped in a + // single-character bracket class so they match literally, not as + // wildcards, before the trailing '*' prefix wildcard is appended. + assert_eq!(super::glob_prefix_pattern("plex"), "plex*"); + assert_eq!(super::glob_prefix_pattern("a*b"), "a[*]b*"); + assert_eq!(super::glob_prefix_pattern("a?b"), "a[?]b*"); + assert_eq!(super::glob_prefix_pattern("a[b"), "a[[]b*"); + assert_eq!(super::glob_prefix_pattern("a*b?c[d"), "a[*]b[?]c[[]d*"); +} + +#[test] +fn resolve_topic_entities_exact_and_prefix_use_indexed_tier() { + let (_dir, pool) = test_pool("resolve-topic-exact-prefix.db"); + let conn = pool.get().unwrap(); + insert_entity(&conn, graph::ENTITY_TYPE_HOST, "nashost"); + insert_entity(&conn, graph::ENTITY_TYPE_APP, "plex"); + insert_entity(&conn, graph::ENTITY_TYPE_APP, "plexmediaserver"); + + let resolved = super::resolve_topic_entities(&conn, &["plex".to_string()]).unwrap(); + let exact = resolved + .iter() + .find(|e| e.canonical_key == "plex") + .expect("exact canonical_key match must resolve"); + assert_eq!(exact.match_kind, "exact"); + assert_eq!(exact.resolver_status, ResolverStatus::Resolved); + + let prefix = resolved + .iter() + .find(|e| e.canonical_key == "plexmediaserver") + .expect("prefix canonical_key match must resolve"); + assert_eq!(prefix.match_kind, "prefix"); + assert_eq!(prefix.resolver_status, ResolverStatus::Ambiguous); +} + +#[test] +fn resolve_topic_entities_falls_back_to_label_substring_scan() { + let (_dir, pool) = test_pool("resolve-topic-label-fallback.db"); + let conn = pool.get().unwrap(); + // canonical_key has no relationship to the search term at all; only the + // human-readable display_label contains it as a mid-string substring. + // This must still be found by the (unavoidable) label scan tier. + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, trust_level) + VALUES (?1, ?2, ?3, 'verified')", + rusqlite::params![ + graph::ENTITY_TYPE_APP, + "backup-job-42", + "nightly backup-job-42 (plex library)" + ], + ) + .unwrap(); + + let resolved = super::resolve_topic_entities(&conn, &["plex".to_string()]).unwrap(); + let label = resolved + .iter() + .find(|e| e.canonical_key == "backup-job-42") + .expect("label substring match must still resolve via fallback scan"); + assert_eq!(label.match_kind, "label"); + assert_eq!(label.resolver_status, ResolverStatus::Ambiguous); +} + +#[test] +fn resolve_topic_entities_label_scan_escapes_like_wildcards() { + let (_dir, pool) = test_pool("resolve-topic-label-escape.db"); + let conn = pool.get().unwrap(); + // A topic term containing a literal '%' must not act as a LIKE wildcard + // in the label-substring fallback tier — it should only match labels + // that contain the literal character, not everything. + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, trust_level) + VALUES (?1, ?2, ?3, 'verified')", + rusqlite::params![ + graph::ENTITY_TYPE_APP, + "cpu-100pct-alert", + "cpu at 100% alert" + ], + ) + .unwrap(); + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, trust_level) + VALUES (?1, ?2, ?3, 'verified')", + rusqlite::params![ + graph::ENTITY_TYPE_APP, + "unrelated-entity", + "totally unrelated" + ], + ) + .unwrap(); + + let resolved = super::resolve_topic_entities(&conn, &["100%".to_string()]).unwrap(); + assert!( + resolved + .iter() + .any(|e| e.canonical_key == "cpu-100pct-alert"), + "literal '100%' substring must still match a label containing it" + ); + assert!( + !resolved + .iter() + .any(|e| e.canonical_key == "unrelated-entity"), + "unescaped '%' would wildcard-match everything; escaping must prevent that" + ); +} + +#[test] +fn resolve_topic_entities_skips_label_scan_when_indexed_tier_fills_cap() { + let (_dir, pool) = test_pool("resolve-topic-cap-skips-label.db"); + let conn = pool.get().unwrap(); + // 25 canonical_key prefix matches (== PER_TERM_CAP) plus one entity that + // would only ever be found via the label substring scan. Once the + // indexed tier alone fills the per-term cap, the label tier must not + // contribute any additional candidates for this term. + for i in 0..25 { + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, trust_level) + VALUES (?1, ?2, ?2, 'verified')", + rusqlite::params![graph::ENTITY_TYPE_APP, format!("svc-{i:02}")], + ) + .unwrap(); + } + conn.execute( + "INSERT INTO graph_entities + (entity_type, canonical_key, display_label, trust_level) + VALUES (?1, ?2, ?3, 'verified')", + rusqlite::params![ + graph::ENTITY_TYPE_APP, + "unrelated-key", + "totally unrelated but mentions svc- in passing" + ], + ) + .unwrap(); + + let resolved = super::resolve_topic_entities(&conn, &["svc-".to_string()]).unwrap(); + assert_eq!(resolved.len(), 25, "per-term cap must still be honored"); + assert!( + !resolved.iter().any(|e| e.canonical_key == "unrelated-key"), + "label-only candidate must not appear once the indexed tier fills the cap: {:?}", + resolved + ); +} diff --git a/crates/shared/cortex/storage-sqlite/src/queries_tests.rs b/crates/shared/cortex/storage-sqlite/src/queries_tests.rs new file mode 100644 index 00000000..b2f357f3 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/queries_tests.rs @@ -0,0 +1,3193 @@ +use super::*; +use crate::config::StorageConfig; +use crate::{AiRelatedWindow, DbPool, LogBatchEntry, init_pool, insert_logs_batch}; + +fn test_storage_config(db_path: std::path::PathBuf) -> StorageConfig { + StorageConfig::for_test(db_path) +} + +/// Create an isolated test pool using a temp file (not :memory: — FTS5 needs file) +fn test_pool() -> (DbPool, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let config = test_storage_config(db_path); + let pool = init_pool(&config).unwrap(); + (pool, dir) // keep dir alive for test duration +} + +fn query_plan(pool: &DbPool, sql: &str, bindings: &[rusqlite::types::Value]) -> String { + let conn = pool.get().unwrap(); + let mut stmt = conn.prepare(&format!("EXPLAIN QUERY PLAN {sql}")).unwrap(); + stmt.query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + row.get::<_, String>(3) + }) + .unwrap() + .collect::>>() + .unwrap() + .join("\n") +} + +fn make_entry(ts: &str, host: &str, severity: &str, msg: &str) -> LogBatchEntry { + LogBatchEntry { + timestamp: ts.to_string(), + hostname: host.to_string(), + facility: None, + severity: severity.to_string(), + app_name: None, + process_id: None, + message: msg.to_string(), + raw: msg.to_string(), + source_ip: "127.0.0.1:514".to_string(), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + } +} + +#[test] +fn search_logs_fts_plan_uses_bounded_candidate_window() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[make_entry( + "2026-01-01T00:00:00Z", + "host-a", + "err", + "bounded search candidate", + )], + ) + .unwrap(); + + let params = SearchParams { + query: Some("bounded".to_string()), + ..Default::default() + }; + let (sql, bindings) = search_logs_fts_sql(params.query.as_deref().unwrap(), ¶ms, 100); + assert!(sql.contains("fts_candidates")); + assert!(sql.contains("LIMIT ?")); + assert!( + bindings + .iter() + .any(|value| matches!(value, rusqlite::types::Value::Integer(limit) if *limit == SEARCH_FTS_CANDIDATE_CAP as i64)), + "FTS candidate cap should be carried as a bound parameter" + ); + + let plan = query_plan(&pool, &sql, &bindings); + assert!( + plan.contains("MATERIALIZE fts_candidates"), + "FTS search should cap candidates before final sort; got:\n{plan}" + ); + assert!( + plan.contains("SCAN logs_fts VIRTUAL TABLE"), + "FTS search should remain FTS-driven; got:\n{plan}" + ); +} + +#[test] +fn tail_logs_severity_only_uses_per_severity_index_probes() { + // full-review PM6: severity-only tails previously walked + // idx_logs_timestamp newest-first and filtered — O(table) for rare + // severities. The fast path probes idx_logs_sev_time once per severity. + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[ + make_entry("2026-01-01T00:00:01Z", "host-a", "emerg", "kernel meltdown"), + make_entry("2026-01-01T00:00:02Z", "host-a", "info", "routine chatter"), + make_entry("2026-01-01T00:00:03Z", "host-b", "alert", "raid degraded"), + ], + ) + .unwrap(); + + let levels = vec!["emerg".to_string(), "alert".to_string()]; + let (sql, bindings) = tail_logs_sql(None, None, None, Some(&levels), 50); + assert!( + sql.contains("UNION ALL"), + "severity-only tail must use per-severity probes, got:\n{sql}" + ); + let plan = query_plan(&pool, &sql, &bindings); + assert!( + plan.contains("idx_logs_sev_time"), + "each arm must probe the (severity, timestamp) index; got:\n{plan}" + ); + assert!( + !plan.contains("SCAN logs USING INDEX idx_logs_timestamp"), + "severity-only tail must not walk the global timestamp index; got:\n{plan}" + ); + + // Behavior: newest-first across severities, info excluded. + let rows = tail_logs(&pool, None, None, None, Some(&levels), 50).unwrap(); + assert_eq!(rows.len(), 2); + assert_eq!(rows[0].severity, "alert"); + assert_eq!(rows[1].severity, "emerg"); + + // A second filter still takes the generic plan. + let (sql, _) = tail_logs_sql(Some("host-a"), None, None, Some(&levels), 50); + assert!(!sql.contains("UNION ALL")); +} + +#[test] +fn tail_logs_limit_is_bound_and_clamped() { + let levels = vec!["err".to_string(), "warning".to_string()]; + let (sql, bindings) = tail_logs_sql( + Some("host-a"), + Some("10.0.0.1:514"), + Some("sshd"), + Some(&levels), + 50_000, + ); + assert!( + sql.contains("LIMIT ?"), + "tail_logs must bind LIMIT instead of interpolating it: {sql}" + ); + assert!( + bindings + .iter() + .any(|value| matches!(value, rusqlite::types::Value::Integer(limit) if *limit == 500)), + "tail_logs should clamp and bind limit=500, got: {bindings:?}" + ); +} + +#[test] +fn host_filters_accept_the_canonical_name_returned_by_list_hosts() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[ + make_entry( + "2026-01-01T00:00:01Z", + "The-Gatewayhost", + "info", + "case variant", + ), + make_entry("2026-01-01T00:00:02Z", "nashost", "info", "short name"), + make_entry( + "2026-01-01T00:00:03Z", + "nashost.example.test", + "info", + "fqdn variant", + ), + ], + ) + .unwrap(); + + let hosts = list_hosts(&pool).unwrap(); + assert!(hosts.iter().any(|host| host.hostname == "the-gatewayhost")); + assert!(hosts.iter().any(|host| host.hostname == "nashost")); + + let case_rows = tail_logs(&pool, Some("the-gatewayhost"), None, None, None, 10).unwrap(); + assert_eq!(case_rows.len(), 1); + assert_eq!(case_rows[0].hostname, "The-Gatewayhost"); + + let alias_rows = tail_logs(&pool, Some("nashost"), None, None, None, 10).unwrap(); + assert_eq!(alias_rows.len(), 2); + assert!( + alias_rows + .iter() + .any(|row| row.hostname == "nashost.example.test") + ); + + let params = SearchParams { + host: Some("the-gatewayhost".to_string()), + limit: Some(10), + ..Default::default() + }; + let filtered = search_logs(&pool, ¶ms).unwrap(); + assert_eq!(filtered.len(), 1); + assert_eq!(filtered[0].hostname, "The-Gatewayhost"); +} + +#[test] +fn get_error_summary_limit_is_bound_and_min_clamped() { + let (sql, bindings) = get_error_summary_sql( + Some("2026-01-01T00:00:00Z"), + Some("2026-01-01T01:00:00Z"), + true, + Some(0), + ); + assert!( + sql.contains("LIMIT ?"), + "get_error_summary must bind LIMIT instead of interpolating it: {sql}" + ); + assert!( + bindings + .iter() + .any(|value| matches!(value, rusqlite::types::Value::Integer(limit) if *limit == 1)), + "get_error_summary should clamp and bind limit=1, got: {bindings:?}" + ); +} + +#[test] +fn app_filtered_search_order_uses_app_timestamp_index() { + let (pool, _dir) = test_pool(); + let plan = query_plan( + &pool, + "SELECT l.id + FROM logs l + WHERE l.app_name = ?1 + ORDER BY l.timestamp DESC + LIMIT 100", + &[rusqlite::types::Value::Text("nginx".into())], + ); + assert!( + plan.contains("idx_logs_app_name_timestamp"), + "app filtered timestamp-ordered search should use app/timestamp index; got:\n{plan}" + ); + assert!( + !plan.contains("USE TEMP B-TREE"), + "app filtered timestamp-ordered search should not temp-sort; got:\n{plan}" + ); +} + +#[test] +fn test_search_fts() { + let (pool, _dir) = test_pool(); + let entries = vec![ + make_entry( + "2026-01-01T00:00:01Z", + "host-a", + "err", + "disk full on /dev/sda", + ), + make_entry( + "2026-01-01T00:00:02Z", + "host-b", + "info", + "connection established", + ), + ]; + insert_logs_batch(&pool, &entries).unwrap(); + + let params = SearchParams { + query: Some("disk".to_string()), + ..Default::default() + }; + let results = search_logs(&pool, ¶ms).unwrap(); + assert_eq!(results.len(), 1); + assert!(results[0].message.contains("disk full")); +} + +#[test] +fn search_fts_hostname_filter_returns_only_that_host() { + // Both hosts emit a matching message; the hostname filter must scope to one. + // Exercises the index-led intersect plan (the fix for the ~200s host-scoped + // search) and verifies it stays correct/complete. + let (pool, _dir) = test_pool(); + let mut entries = Vec::new(); + for i in 0..50 { + entries.push(make_entry( + &format!("2026-01-01T00:{:02}:00Z", i % 60), + "host-a", + "err", + "kernel panic detected", + )); + entries.push(make_entry( + &format!("2026-01-01T01:{:02}:00Z", i % 60), + "host-b", + "err", + "kernel panic detected", + )); + } + insert_logs_batch(&pool, &entries).unwrap(); + + let params = SearchParams { + query: Some("panic".to_string()), + host: Some("host-a".to_string()), + limit: Some(1000), + ..Default::default() + }; + let results = search_logs(&pool, ¶ms).unwrap(); + assert_eq!(results.len(), 50, "should return all 50 host-a matches"); + assert!( + results.iter().all(|r| r.hostname == "host-a"), + "hostname filter must exclude host-b rows" + ); + // Newest-first ordering preserved. + assert!(results.windows(2).all(|w| w[0].timestamp >= w[1].timestamp)); +} + +#[test] +fn search_fts_plan_selection_branches_on_indexed_filter() { + // No indexed equality filter → capped materialized-candidate plan. + let plain = SearchParams { + query: Some("disk".to_string()), + ..Default::default() + }; + assert!(!plain.has_indexed_equality_filter()); + let (sql, _) = search_logs_fts_sql("disk", &plain, 50); + assert!(sql.contains("fts_candidates"), "unfiltered uses CTE plan"); + assert!(!sql.contains("l.id IN (SELECT rowid")); + + // Selective indexed equality filter → index-led intersect plan with a + // bounded match-set subquery (full-review PH1: the non-correlated IN + // subquery is materialized in full, so it must carry a cap). + let filtered = SearchParams { + host: Some("host-a".to_string()), + ..plain.clone() + }; + assert!(filtered.has_indexed_equality_filter()); + let (sql, _) = search_logs_fts_sql("disk", &filtered, 50); + assert!( + sql.contains("l.id IN (SELECT rowid FROM logs_fts WHERE logs_fts MATCH ?1"), + "filtered search must use the intersect plan, got:\n{sql}" + ); + assert!( + sql.contains(&format!( + "ORDER BY rowid DESC LIMIT {SEARCH_FTS_FAST_PATH_MATCH_CAP}" + )), + "intersect plan must bound the materialized match set, got:\n{sql}" + ); + assert!( + !sql.contains("fts_candidates"), + "intersect plan has no CTE cap" + ); + assert!( + sql.contains("l.hostname IN (") && sql.contains("FROM hosts h"), + "canonical host filter applied via append_filters" + ); +} + +#[test] +fn search_fts_severity_only_filter_uses_capped_candidate_plan() { + // Severity partitions are huge (a single severity can be >90% of the + // table); a rare term + severity-only filter previously walked the whole + // partition via the fast path (full-review PH1). Severity-only must take + // the capped-candidate plan. + let severity_only = SearchParams { + query: Some("disk".to_string()), + severity: Some("info".to_string()), + ..Default::default() + }; + assert!(!severity_only.has_indexed_equality_filter()); + let (sql, _) = search_logs_fts_sql("disk", &severity_only, 50); + assert!( + sql.contains("fts_candidates"), + "severity-only search must use the capped CTE plan, got:\n{sql}" + ); + + let severity_in_only = SearchParams { + query: Some("disk".to_string()), + severity_in: Some(vec!["emerg".to_string(), "alert".to_string()]), + ..Default::default() + }; + assert!(!severity_in_only.has_indexed_equality_filter()); + + // Severity combined with a selective filter still gets the fast path + // (the selective column's index leads). + let combined = SearchParams { + query: Some("disk".to_string()), + severity: Some("info".to_string()), + host: Some("host-a".to_string()), + ..Default::default() + }; + assert!(combined.has_indexed_equality_filter()); +} + +#[test] +fn test_search_invalid_fts_returns_error() { + let (pool, _dir) = test_pool(); + // FTS5 treats bare parentheses as a syntax error + let params = SearchParams { + query: Some("(invalid fts syntax".to_string()), + ..Default::default() + }; + let result = search_logs(&pool, ¶ms); + assert!(result.is_err(), "invalid FTS5 query should return Err"); + // Error message must be generic — no schema details leaked + let msg = result.unwrap_err().to_string(); + assert_eq!(msg, "Search query failed", "error must be generic"); +} + +// --- validate_fts_query unit tests --- + +#[test] +fn test_validate_fts_query_valid() { + assert!(validate_fts_query("disk error").is_ok()); + assert!(validate_fts_query("nginx AND 502").is_ok()); + // Exactly 16 terms should pass + let sixteen = (0..16) + .map(|i| format!("term{i}")) + .collect::>() + .join(" "); + assert!(validate_fts_query(&sixteen).is_ok()); + // Exactly 512 chars should pass + let at_limit = "a".repeat(512); + assert!(validate_fts_query(&at_limit).is_ok()); +} + +#[test] +fn test_validate_fts_query_too_long() { + let long_query = "a".repeat(513); + let result = validate_fts_query(&long_query); + assert!(result.is_err(), "query > 512 chars should be rejected"); + let msg = result.unwrap_err().to_string(); + assert!(msg.contains("513"), "error should mention actual length"); + assert!(msg.contains("512"), "error should mention the limit"); +} + +#[test] +fn test_validate_fts_query_too_many_terms() { + let many_terms = (0..17) + .map(|i| format!("term{i}")) + .collect::>() + .join(" "); + let result = validate_fts_query(&many_terms); + assert!(result.is_err(), "query with 17 terms should be rejected"); + let msg = result.unwrap_err().to_string(); + assert!(msg.contains("17"), "error should mention actual term count"); + assert!(msg.contains("16"), "error should mention the limit"); +} + +#[test] +fn test_get_stats_empty_db() { + let (pool, dir) = test_pool(); + let stats = get_stats(&pool, &test_storage_config(dir.path().join("test.db"))).unwrap(); + assert_eq!(stats.total_logs, 0); + assert_eq!(stats.total_hosts, 0); + // oldest_log and newest_log should be None on empty DB + assert!(stats.oldest_log.is_none()); + assert!(stats.newest_log.is_none()); + assert!(stats.free_disk_mb.is_some()); +} + +#[test] +fn test_get_stats_total_logs_matches_across_rollup_states() { + // stats.total_logs = SUM(timeline_hourly.event_count) + COUNT(logs WHERE + // id > watermark). It must equal the true row count whether the rollup is + // empty (all rows in the live delta), fully refreshed (all rows in rollup), + // or partially refreshed (some in each). (bead syslog-mcp-kcvq) + let (pool, dir) = test_pool(); + let cfg = test_storage_config(dir.path().join("test.db")); + + let entries = vec![ + make_entry("2026-01-01T00:00:01Z", "h1", "info", "a"), + make_entry("2026-01-01T00:30:00Z", "h1", "info", "b"), + make_entry("2026-01-01T01:00:00Z", "h2", "err", "c"), + ]; + insert_logs_batch(&pool, &entries).unwrap(); + + // Rollup empty: everything counted via the live delta. + assert_eq!(get_stats(&pool, &cfg).unwrap().total_logs, 3); + + // Fully refreshed: everything counted via the rollup, delta empty. + crate::refresh_timeline_rollup(&pool).unwrap(); + assert_eq!(get_stats(&pool, &cfg).unwrap().total_logs, 3); + + // Insert more after refresh: partial — rollup holds 3, delta holds 2. + insert_logs_batch( + &pool, + &[ + make_entry("2026-01-01T02:00:00Z", "h1", "info", "d"), + make_entry("2026-01-01T02:05:00Z", "h1", "info", "e"), + ], + ) + .unwrap(); + assert_eq!(get_stats(&pool, &cfg).unwrap().total_logs, 5); + + // Refresh again: all 5 now in the rollup. + crate::refresh_timeline_rollup(&pool).unwrap(); + assert_eq!(get_stats(&pool, &cfg).unwrap().total_logs, 5); +} + +#[test] +fn test_get_stats_skips_fts_diagnostic_by_default() { + // Issue 4: the default stats path must NOT run COUNT(*) FROM logs_fts + // (expensive on large DBs), reflected as phantom_fts_rows == None. The + // opt-in path computes it. + let (pool, dir) = test_pool(); + let cfg = test_storage_config(dir.path().join("test.db")); + + let fast = get_stats(&pool, &cfg).unwrap(); + assert_eq!( + fast.phantom_fts_rows, None, + "default stats must skip the FTS diagnostic" + ); + + let full = get_stats_with_options(&pool, &cfg, true).unwrap(); + assert_eq!( + full.phantom_fts_rows, + Some(0), + "opt-in stats must compute phantom_fts_rows (0 on a clean DB)" + ); +} + +#[test] +fn test_tail_filter_by_host() { + let (pool, _dir) = test_pool(); + let entries = vec![ + make_entry("2026-01-01T00:00:01Z", "host-a", "info", "from a"), + make_entry("2026-01-01T00:00:02Z", "host-b", "info", "from b"), + ]; + insert_logs_batch(&pool, &entries).unwrap(); + + let rows = tail_logs(&pool, Some("host-a"), None, None, None, 10).unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].hostname, "host-a"); +} + +#[test] +fn test_search_timestamp_range_filtering() { + let (pool, _dir) = test_pool(); + let entries = vec![ + make_entry("2026-01-01T00:00:00Z", "host-a", "info", "early message"), + make_entry("2026-06-15T12:00:00Z", "host-a", "info", "mid message"), + make_entry("2026-12-31T23:59:59Z", "host-a", "info", "late message"), + ]; + insert_logs_batch(&pool, &entries).unwrap(); + + // from only + let params = SearchParams { + since: Some("2026-06-01T00:00:00Z".into()), + ..Default::default() + }; + let results = search_logs(&pool, ¶ms).unwrap(); + assert_eq!(results.len(), 2, "from filter should return mid + late"); + + // to only + let params = SearchParams { + until: Some("2026-06-30T00:00:00Z".into()), + ..Default::default() + }; + let results = search_logs(&pool, ¶ms).unwrap(); + assert_eq!(results.len(), 2, "to filter should return early + mid"); + + // from + to (narrow window) + let params = SearchParams { + since: Some("2026-06-01T00:00:00Z".into()), + until: Some("2026-06-30T00:00:00Z".into()), + ..Default::default() + }; + let results = search_logs(&pool, ¶ms).unwrap(); + assert_eq!(results.len(), 1, "from+to filter should return only mid"); + assert_eq!(results[0].message, "mid message"); +} + +#[test] +fn test_search_received_at_range_filtering() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[ + make_entry("2026-01-01T00:00:00Z", "host-a", "info", "received early"), + make_entry("2026-01-01T00:00:00Z", "host-a", "info", "received mid"), + make_entry("2026-01-01T00:00:00Z", "host-a", "info", "received late"), + ], + ) + .unwrap(); + + let conn = pool.get().unwrap(); + conn.execute( + "UPDATE logs SET received_at = ?1 WHERE message = ?2", + rusqlite::params!["2026-01-01T00:00:00Z", "received early"], + ) + .unwrap(); + conn.execute( + "UPDATE logs SET received_at = ?1 WHERE message = ?2", + rusqlite::params!["2026-01-01T00:30:00Z", "received mid"], + ) + .unwrap(); + conn.execute( + "UPDATE logs SET received_at = ?1 WHERE message = ?2", + rusqlite::params!["2026-01-01T01:00:00Z", "received late"], + ) + .unwrap(); + drop(conn); + + let results = search_logs( + &pool, + &SearchParams { + received_since: Some("2026-01-01T00:15:00Z".into()), + received_until: Some("2026-01-01T00:45:00Z".into()), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(results.len(), 1); + assert_eq!(results[0].message, "received mid"); +} + +#[test] +fn test_search_exclude_facility_keeps_unknown_facility_rows() { + let (pool, _dir) = test_pool(); + let mut auth = make_entry("2026-01-01T00:00:00Z", "host-a", "info", "auth event"); + auth.facility = Some("auth".into()); + let mut daemon = make_entry("2026-01-01T00:00:01Z", "host-a", "info", "daemon event"); + daemon.facility = Some("daemon".into()); + let unknown = make_entry("2026-01-01T00:00:02Z", "host-a", "info", "unknown event"); + insert_logs_batch(&pool, &[auth, daemon, unknown]).unwrap(); + + let results = search_logs( + &pool, + &SearchParams { + exclude_facility: Some("auth".into()), + ..Default::default() + }, + ) + .unwrap(); + let messages: Vec<&str> = results.iter().map(|row| row.message.as_str()).collect(); + assert_eq!(messages, vec!["unknown event", "daemon event"]); +} + +#[test] +fn test_severity_to_num() { + assert_eq!(severity_to_num("emerg"), Some(0)); + assert_eq!(severity_to_num("alert"), Some(1)); + assert_eq!(severity_to_num("crit"), Some(2)); + assert_eq!(severity_to_num("err"), Some(3)); + assert_eq!(severity_to_num("warning"), Some(4)); + assert_eq!(severity_to_num("notice"), Some(5)); + assert_eq!(severity_to_num("info"), Some(6)); + assert_eq!(severity_to_num("debug"), Some(7)); + // Edge cases + assert_eq!(severity_to_num(""), None); + // Aliases (case-insensitive) + assert_eq!(severity_to_num("ERROR"), Some(3), "case-insensitive alias"); + assert_eq!(severity_to_num("Error"), Some(3), "mixed-case alias"); + assert_eq!(severity_to_num("critical"), Some(2), "alias for 'crit'"); + assert_eq!(severity_to_num("warn"), Some(4), "alias for 'warning'"); + assert_eq!(severity_to_num("emergency"), Some(0), "alias for 'emerg'"); + assert_eq!(severity_to_num("fatal"), Some(3), "alias for 'err'"); + assert_eq!(severity_to_num("panic"), Some(3), "alias for 'err'"); + // Truly invalid + assert_eq!(severity_to_num("bogus"), None); + assert_eq!( + severity_to_num("trace"), + None, + "syslog has no 'trace' level" + ); +} + +#[test] +fn test_error_summary_severity_filter() { + let (pool, _dir) = test_pool(); + let entries = vec![ + make_entry("2026-01-01T00:00:00Z", "host-a", "err", "error msg"), + make_entry("2026-01-01T00:00:01Z", "host-a", "warning", "warn msg"), + make_entry("2026-01-01T00:00:02Z", "host-a", "info", "info msg"), + make_entry("2026-01-01T00:00:03Z", "host-a", "debug", "debug msg"), + ]; + insert_logs_batch(&pool, &entries).unwrap(); + + let summary = get_error_summary(&pool, None, None, false, None).unwrap(); + // Only err and warning should appear (not info, debug) + assert_eq!(summary.len(), 2); + let severities: Vec<&str> = summary.iter().map(|e| e.severity.as_str()).collect(); + assert!(severities.contains(&"err")); + assert!(severities.contains(&"warning")); +} + +#[test] +fn test_search_severity_in_filter() { + let (pool, _dir) = test_pool(); + let entries = vec![ + make_entry("2026-01-01T00:00:00Z", "host-a", "emerg", "emerg msg"), + make_entry("2026-01-01T00:00:01Z", "host-a", "err", "err msg"), + make_entry("2026-01-01T00:00:02Z", "host-a", "warning", "warn msg"), + make_entry("2026-01-01T00:00:03Z", "host-a", "info", "info msg"), + make_entry("2026-01-01T00:00:04Z", "host-a", "debug", "debug msg"), + ]; + insert_logs_batch(&pool, &entries).unwrap(); + + let params = SearchParams { + severity_in: Some(vec!["emerg".into(), "err".into(), "warning".into()]), + ..Default::default() + }; + let results = search_logs(&pool, ¶ms).unwrap(); + assert_eq!(results.len(), 3, "severity_in should match exactly 3"); + for r in &results { + assert!( + ["emerg", "err", "warning"].contains(&r.severity.as_str()), + "unexpected severity: {}", + r.severity + ); + } +} + +#[test] +fn tail_logs_filters_multiple_severities() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[ + make_entry("2026-01-01T00:00:00Z", "host-a", "err", "err msg"), + make_entry("2026-01-01T00:00:01Z", "host-a", "warning", "warn msg"), + make_entry("2026-01-01T00:00:02Z", "host-a", "info", "info msg"), + ], + ) + .unwrap(); + + let severities = vec!["err".to_string(), "warning".to_string()]; + let rows = tail_logs(&pool, Some("host-a"), None, None, Some(&severities), 10).unwrap(); + + assert_eq!(rows.len(), 2); + assert!(rows.iter().all(|row| row.hostname == "host-a")); + assert!( + rows.iter() + .all(|row| ["err", "warning"].contains(&row.severity.as_str())) + ); +} + +#[test] +fn search_logs_ignores_deleted_fts_phantom_rows() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[make_entry( + "2026-01-01T00:00:00Z", + "host-a", + "info", + "live message", + )], + ) + .unwrap(); + + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO logs_fts(rowid, message) VALUES (?1, ?2)", + rusqlite::params![999_999_i64, "phantom-token orphan row"], + ) + .unwrap(); + drop(conn); + + let params = SearchParams { + query: Some("\"phantom-token\"".to_string()), + ..Default::default() + }; + let results = search_logs(&pool, ¶ms).unwrap(); + assert!(results.is_empty(), "FTS-only phantom rows must not leak"); +} + +fn make_ai_entry( + ts: &str, + host: &str, + tool: &str, + project: &str, + session_id: &str, + message: &str, +) -> LogBatchEntry { + LogBatchEntry { + timestamp: ts.to_string(), + hostname: host.to_string(), + facility: Some("local0".to_string()), + severity: "info".to_string(), + app_name: Some("ai-transcript".to_string()), + process_id: None, + message: message.to_string(), + raw: message.to_string(), + source_ip: "127.0.0.1:514".to_string(), + docker_checkpoint: None, + ai_tool: Some(tool.to_string()), + ai_project: Some(project.to_string()), + ai_session_id: Some(session_id.to_string()), + ai_transcript_path: Some(format!("{project}/{session_id}.jsonl")), + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + } +} + +#[test] +fn search_logs_exclude_ai_filters_structured_and_transcript_app_rows() { + let (pool, _dir) = test_pool(); + let mut legacy_transcript = make_entry( + "2026-01-01T00:00:00Z", + "localhost", + "info", + "legacy codex transcript event", + ); + legacy_transcript.app_name = Some("codex-transcript".into()); + + insert_logs_batch( + &pool, + &[ + legacy_transcript, + make_ai_entry( + "2026-01-01T00:00:01Z", + "localhost", + "codex", + "/tmp/project", + "sess-1", + "structured ai transcript event", + ), + make_entry( + "2026-01-01T00:00:02Z", + "host-a", + "warning", + "real host event", + ), + ], + ) + .unwrap(); + + let rows = search_logs( + &pool, + &SearchParams { + query: Some("event".into()), + exclude_ai: true, + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].message, "real host event"); +} + +#[test] +fn search_logs_filters_by_source_ip_prefix_without_fts() { + let (pool, _dir) = test_pool(); + let mut docker_stdout = make_entry( + "2026-01-01T00:00:00Z", + "devhost", + "info", + "container stdout line", + ); + docker_stdout.source_ip = "docker://devhost/cortex/stdout".into(); + + let mut docker_stderr = make_entry( + "2026-01-01T00:00:01Z", + "devhost", + "warning", + "container stderr line", + ); + docker_stderr.source_ip = "docker://devhost/cortex/stderr".into(); + + let mut other = make_entry( + "2026-01-01T00:00:02Z", + "devhost", + "info", + "different container line", + ); + other.source_ip = "docker://devhost/other/stdout".into(); + + insert_logs_batch(&pool, &[docker_stdout, docker_stderr, other]).unwrap(); + + let rows = search_logs( + &pool, + &SearchParams { + source_ip_prefix: Some("docker://devhost/cortex/".into()), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(rows.len(), 2); + assert!( + rows.iter() + .all(|row| row.source_ip.starts_with("docker://devhost/cortex/")) + ); +} + +#[test] +fn search_logs_filters_by_event_action_column() { + let (pool, _dir) = test_pool(); + let mut die = make_entry( + "2026-01-01T00:00:00Z", + "devhost", + "notice", + "container died", + ); + die.source_ip = "docker-event://devhost/cortex/die".into(); + die.event_action = Some("die".into()); + + let mut start = make_entry( + "2026-01-01T00:00:01Z", + "devhost", + "notice", + "container started", + ); + start.source_ip = "docker-event://devhost/cortex/start".into(); + start.event_action = Some("start".into()); + + insert_logs_batch(&pool, &[die, start]).unwrap(); + + let rows = search_logs( + &pool, + &SearchParams { + source_ip_prefix: Some("docker-event://".into()), + event_action: Some("die".into()), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].message, "container died"); +} + +#[test] +fn search_ai_sessions_groups_results() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[ + make_ai_entry( + "2026-01-01T00:00:00Z", + "host-a", + "claude", + "/tmp/project", + "sess-1", + "authentication bug fixed", + ), + make_ai_entry( + "2026-01-01T00:01:00Z", + "host-a", + "claude", + "/tmp/project", + "sess-1", + "authentication tests passing", + ), + make_ai_entry( + "2026-01-01T00:02:00Z", + "host-a", + "claude", + "/tmp/project", + "sess-1", + "unmatched context", + ), + ], + ) + .unwrap(); + + let result = search_ai_sessions( + &pool, + &SearchAiSessionsParams { + query: "authentication".into(), + limit: Some(10), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(result.sessions.len(), 1); + assert_eq!(result.sessions[0].match_count, 2); + assert_eq!(result.sessions[0].event_count, 3); +} + +#[test] +fn search_ai_sessions_query_plan_uses_session_host_time_index() { + let (pool, _dir) = test_pool(); + let mut entries = Vec::new(); + for i in 0..150 { + entries.push(make_ai_entry( + &format!("2026-01-01T00:{:02}:00Z", i % 60), + "host-a", + "claude", + "/tmp/project", + "sess-1", + if i % 10 == 0 { + "indexed authentication match" + } else { + "background transcript event" + }, + )); + } + for i in 0..50 { + entries.push(make_ai_entry( + &format!("2026-01-02T00:{:02}:00Z", i % 60), + "host-b", + "codex", + "/tmp/other", + "sess-2", + "indexed authentication match", + )); + } + insert_logs_batch(&pool, &entries).unwrap(); + refresh_ai_session_rollup(&pool).unwrap(); + + let params = SearchAiSessionsParams { + query: "authentication".into(), + ai_project: Some("/tmp/project".into()), + ai_tool: Some("claude".into()), + limit: Some(10), + ..Default::default() + }; + let result = search_ai_sessions(&pool, ¶ms).unwrap(); + assert_eq!(result.sessions.len(), 1); + assert_eq!(result.sessions[0].event_count, 150); + + let (sql, bindings) = search_ai_sessions_sql(¶ms, 10); + assert!(sql.contains("candidates AS MATERIALIZED")); + assert!( + sql.contains("FROM logs_fts") && sql.contains("WHERE logs_fts MATCH ?1"), + "session search candidates must be FTS-first" + ); + assert!( + sql.contains("LIMIT 5000"), + "session search must bound filtered matching rows" + ); + assert!( + sql.contains("LEFT JOIN ai_session_rollup rollup"), + "selected sessions should combine rollup statistics with post-refresh rows" + ); + + let conn = pool.get().unwrap(); + let mut stmt = conn.prepare(&format!("EXPLAIN QUERY PLAN {sql}")).unwrap(); + let plan = stmt + .query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + row.get::<_, String>(3) + }) + .unwrap() + .collect::>>() + .unwrap() + .join("\n"); + // Pin the EXACT index that the new (host, time) composite was added + // for so a planner regression that falls back to the broader + // `idx_logs_ai_session` is caught by this guard. If SQLite version + // drift starts flapping this in CI, prefer documenting the SQLite + // version pin over loosening this assertion. + assert!( + plan.contains("idx_logs_ai_session_host_time"), + "expected AI session event-count plan to use idx_logs_ai_session_host_time, got:\n{plan}" + ); +} + +#[test] +fn search_ai_sessions_finds_rows_inserted_after_rollup_refresh() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[make_ai_entry( + "2026-01-01T00:00:00Z", + "host-a", + "claude", + "/tmp/project", + "before-refresh", + "background transcript event", + )], + ) + .unwrap(); + refresh_ai_session_rollup(&pool).unwrap(); + insert_logs_batch( + &pool, + &[make_ai_entry( + "2026-01-01T00:01:00Z", + "host-a", + "claude", + "/tmp/project", + "after-refresh", + "freshneedle appears immediately", + )], + ) + .unwrap(); + + let result = search_ai_sessions( + &pool, + &SearchAiSessionsParams { + query: "freshneedle".into(), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.total_candidates, 1); + assert_eq!(result.sessions[0].ai_session_id, "after-refresh"); + assert!(!result.truncated); +} + +#[test] +fn search_ai_sessions_combines_rollup_with_post_refresh_session_rows() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[make_ai_entry( + "2026-01-01T00:00:00Z", + "host-a", + "claude", + "/tmp/project", + "shared-session", + "background transcript event", + )], + ) + .unwrap(); + refresh_ai_session_rollup(&pool).unwrap(); + insert_logs_batch( + &pool, + &[make_ai_entry( + "2026-01-01T00:01:00Z", + "host-a", + "claude", + "/tmp/project", + "shared-session", + "freshneedle appears immediately", + )], + ) + .unwrap(); + + let result = search_ai_sessions( + &pool, + &SearchAiSessionsParams { + query: "freshneedle".into(), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.sessions.len(), 1); + assert_eq!(result.sessions[0].event_count, 2); + assert_eq!(result.sessions[0].first_seen, "2026-01-01T00:00:00Z"); + assert_eq!(result.sessions[0].last_seen, "2026-01-01T00:01:00Z"); +} + +#[test] +fn search_ai_sessions_finds_selective_match_older_than_newest_thousand_sessions() { + let (pool, _dir) = test_pool(); + let mut entries = vec![make_ai_entry( + "2026-01-01T00:00:00Z", + "host-a", + "claude", + "/tmp/project", + "historical-match", + "historicalneedle remains discoverable", + )]; + for i in 0..1_001 { + entries.push(make_ai_entry( + "2026-01-02T00:00:00Z", + "host-a", + "claude", + "/tmp/project", + &format!("newer-{i}"), + "routine newer transcript event", + )); + } + insert_logs_batch(&pool, &entries).unwrap(); + refresh_ai_session_rollup(&pool).unwrap(); + + let result = search_ai_sessions( + &pool, + &SearchAiSessionsParams { + query: "historicalneedle".into(), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.total_candidates, 1); + assert_eq!(result.candidate_rows, 1); + assert_eq!(result.sessions[0].ai_session_id, "historical-match"); + assert!(!result.candidate_window_truncated); + assert!(!result.truncated); +} + +#[test] +fn search_ai_sessions_metadata_counts_only_filtered_matches() { + let (pool, _dir) = test_pool(); + let mut entries = Vec::new(); + for i in 0..1_001 { + entries.push(make_ai_entry( + "2026-01-02T00:00:00Z", + "host-a", + "claude", + "/tmp/unrelated", + &format!("unrelated-{i}"), + "scopedneedle", + )); + } + entries.push(make_ai_entry( + "2026-01-01T00:00:00Z", + "host-a", + "claude", + "/tmp/selected", + "selected-a", + "scopedneedle", + )); + entries.push(make_ai_entry( + "2026-01-01T00:01:00Z", + "host-a", + "claude", + "/tmp/selected", + "selected-b", + "scopedneedle", + )); + insert_logs_batch(&pool, &entries).unwrap(); + refresh_ai_session_rollup(&pool).unwrap(); + + let result = search_ai_sessions( + &pool, + &SearchAiSessionsParams { + query: "scopedneedle".into(), + ai_project: Some("/tmp/selected".into()), + limit: Some(10), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.total_candidates, 2); + assert_eq!(result.candidate_rows, 2); + assert_eq!(result.sessions.len(), 2); + assert!(!result.candidate_window_truncated); + assert!(!result.truncated); +} + +#[test] +fn search_ai_sessions_candidate_cap_prefers_newer_rows() { + let (pool, _dir) = test_pool(); + let mut entries = Vec::new(); + for i in 0..=5000 { + entries.push(make_ai_entry( + &format!("2026-01-01T00:{:02}:00Z", i % 60), + "host-a", + "claude", + "/tmp/old", + &format!("old-{i}"), + "commontoken", + )); + } + entries.push(make_ai_entry( + "2026-01-02T00:00:00Z", + "host-a", + "claude", + "/tmp/new", + "newest", + "commontoken", + )); + insert_logs_batch(&pool, &entries).unwrap(); + + let result = search_ai_sessions( + &pool, + &SearchAiSessionsParams { + query: "commontoken".into(), + limit: Some(10), + ..Default::default() + }, + ) + .unwrap(); + + assert!(result.truncated); + assert_eq!(result.candidate_cap, 5000); + assert_eq!(result.candidate_rows, 5000); + assert!(result.candidate_window_truncated); + assert_eq!(result.sessions[0].ai_session_id, "newest"); + assert_eq!(result.sessions[0].ai_project, "/tmp/new"); +} + +// Regression: results must be ranked by match recency (latest matching row), +// not full-session last_seen. `stale-match` matched earlier but kept logging +// unrelated activity afterward; it must still rank below `recent-match`, whose +// match is newer. A pre-fix `ORDER BY last_seen DESC` ranked `stale-match` +// first because its non-matching tail is the newest activity overall. +#[test] +fn search_ai_sessions_orders_by_match_recency_not_session_activity() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[ + // Older match, but newest overall activity (non-matching tail). + make_ai_entry( + "2026-01-01T00:00:00Z", + "host-a", + "claude", + "/tmp/project", + "stale-match", + "orderneedle historical hit", + ), + make_ai_entry( + "2026-01-03T00:00:00Z", + "host-a", + "claude", + "/tmp/project", + "stale-match", + "routine unrelated activity", + ), + // Newer match, no later activity. + make_ai_entry( + "2026-01-02T00:00:00Z", + "host-a", + "claude", + "/tmp/project", + "recent-match", + "orderneedle fresh hit", + ), + ], + ) + .unwrap(); + + let result = search_ai_sessions( + &pool, + &SearchAiSessionsParams { + query: "orderneedle".into(), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.sessions.len(), 2); + assert_eq!(result.sessions[0].ai_session_id, "recent-match"); + assert_eq!(result.sessions[1].ai_session_id, "stale-match"); + // The stale session's displayed last_seen still reflects its full-session + // tail; only the ordering key changed. + assert_eq!(result.sessions[1].last_seen, "2026-01-03T00:00:00Z"); +} + +#[test] +fn search_ai_sessions_zero_limit_clamps_to_one_with_metadata() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[make_ai_entry( + "2026-01-01T00:00:00Z", + "host-a", + "claude", + "/tmp/project", + "sess-1", + "zerolimit", + )], + ) + .unwrap(); + + let result = search_ai_sessions( + &pool, + &SearchAiSessionsParams { + query: "zerolimit".into(), + limit: Some(0), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.sessions.len(), 1); + assert_eq!(result.total_candidates, 1); + assert_eq!(result.candidate_rows, 1); + assert!(!result.truncated); +} + +#[test] +fn search_ai_related_logs_batches_windows_and_caps_per_anchor() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[ + make_ai_entry( + "2026-01-01T00:00:00Z", + "localhost", + "codex", + "/tmp/project", + "sess-1", + "anchor one deploy failure", + ), + make_entry( + "2026-01-01T00:00:30Z", + "host-a", + "err", + "deploy failed on host-a", + ), + make_entry( + "2026-01-01T00:00:40Z", + "host-a", + "warning", + "deploy warning on host-a", + ), + make_entry( + "2026-01-01T00:00:50Z", + "host-a", + "info", + "deploy info below severity", + ), + make_ai_entry( + "2026-01-01T00:10:00Z", + "localhost", + "codex", + "/tmp/project", + "sess-2", + "anchor two deploy failure", + ), + make_entry( + "2026-01-01T00:10:30Z", + "host-b", + "err", + "deploy failed on host-b", + ), + ], + ) + .unwrap(); + + let rows = search_ai_related_logs( + &pool, + &AiRelatedLogsParams { + windows: vec![ + AiRelatedWindow { + anchor_index: 0, + anchor_time: "2026-01-01T00:00:00Z".into(), + window_from: "2026-01-01T00:00:00.000Z".into(), + window_to: "2026-01-01T00:01:00.000Z".into(), + }, + AiRelatedWindow { + anchor_index: 1, + anchor_time: "2026-01-01T00:10:00Z".into(), + window_from: "2026-01-01T00:10:00.000Z".into(), + window_to: "2026-01-01T00:11:00.000Z".into(), + }, + ], + query: Some("deploy".into()), + severity_in: vec!["err".into(), "warning".into()], + limit_per_anchor: 1, + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(rows.len(), 2); + assert_eq!(rows[0].anchor_index, 0); + assert_eq!(rows[0].logs.len(), 1); + assert_eq!(rows[0].logs[0].message, "deploy failed on host-a"); + assert!(rows[0].truncated); + assert_eq!(rows[1].anchor_index, 1); + assert_eq!(rows[1].logs.len(), 1); + assert_eq!(rows[1].logs[0].message, "deploy failed on host-b"); + assert!(!rows[1].truncated); +} + +#[test] +fn search_ai_related_logs_prefers_rows_nearest_the_anchor() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[ + make_entry("2026-01-01T00:05:01Z", "host-a", "info", "near-after"), + make_entry("2026-01-01T00:09:59Z", "host-a", "info", "far-after"), + make_entry("2026-01-01T00:04:58Z", "host-a", "info", "near-before"), + ], + ) + .unwrap(); + + let rows = search_ai_related_logs( + &pool, + &AiRelatedLogsParams { + windows: vec![AiRelatedWindow { + anchor_index: 0, + anchor_time: "2026-01-01T00:05:00Z".into(), + window_from: "2026-01-01T00:00:00Z".into(), + window_to: "2026-01-01T00:10:00Z".into(), + }], + severity_in: vec!["info".into()], + limit_per_anchor: 2, + ..Default::default() + }, + ) + .unwrap(); + + let messages: Vec<_> = rows[0] + .logs + .iter() + .map(|row| row.message.as_str()) + .collect(); + assert_eq!(messages, vec!["near-after", "near-before"]); + assert!(rows[0].truncated); +} + +#[test] +fn search_ai_abuse_returns_same_session_context() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[ + make_ai_entry( + "2026-01-01T00:00:00Z", + "host-a", + "codex", + "/tmp/project", + "sess-1", + "before row", + ), + make_ai_entry( + "2026-01-01T00:01:00Z", + "host-a", + "codex", + "/tmp/project", + "sess-1", + "this shit needs context", + ), + make_ai_entry( + "2026-01-01T00:02:00Z", + "host-a", + "codex", + "/tmp/project", + "sess-1", + "after row", + ), + make_ai_entry( + "2026-01-01T00:03:00Z", + "host-a", + "codex", + "/tmp/project", + "sess-2", + "other session row", + ), + make_ai_entry( + "2026-01-01T00:04:00Z", + "host-a", + "codex", + "/tmp/project", + "sess-1", + "assistant is not a abuse false positive", + ), + ], + ) + .unwrap(); + + let result = search_ai_abuse( + &pool, + &AiAbuseParams { + ai_project: Some("/tmp/project".into()), + ai_tool: Some("codex".into()), + limit: Some(10), + before: Some(1), + after: Some(1), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.matches.len(), 1); + let hit = &result.matches[0]; + assert_eq!(hit.term, "shit"); + assert_eq!(hit.entry.message, "this shit needs context"); + assert_eq!(hit.before.len(), 1); + assert_eq!(hit.before[0].message, "before row"); + assert_eq!(hit.after.len(), 1); + assert_eq!(hit.after[0].message, "after row"); +} + +#[test] +fn search_ai_abuse_truncates_only_when_additional_match_exists() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[ + make_ai_entry( + "2026-01-01T00:00:00Z", + "host-a", + "codex", + "/tmp/project", + "sess-1", + "one shit", + ), + make_ai_entry( + "2026-01-01T00:01:00Z", + "host-a", + "codex", + "/tmp/project", + "sess-1", + "plain row", + ), + ], + ) + .unwrap(); + + let exact = search_ai_abuse( + &pool, + &AiAbuseParams { + ai_project: Some("/tmp/project".into()), + ai_tool: Some("codex".into()), + limit: Some(1), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(exact.matches.len(), 1); + assert!(!exact.truncated); + + insert_logs_batch( + &pool, + &[make_ai_entry( + "2026-01-01T00:02:00Z", + "host-a", + "codex", + "/tmp/project", + "sess-1", + "two shit", + )], + ) + .unwrap(); + let truncated = search_ai_abuse( + &pool, + &AiAbuseParams { + ai_project: Some("/tmp/project".into()), + ai_tool: Some("codex".into()), + limit: Some(1), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(truncated.matches.len(), 1); + assert!(truncated.truncated); +} + +#[test] +fn search_ai_incidents_anchor_plan_uses_selective_fts_first() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[make_ai_entry( + "2026-01-01T00:00:00Z", + "host-a", + "codex", + "/tmp/project", + "sess-1", + "tooling transcript row", + )], + ) + .unwrap(); + let params = AiIncidentParams { + terms: vec!["tooling".into()], + limit: Some(1), + ..Default::default() + }; + let terms = normalized_abuse_terms(¶ms.terms); + let (sql, bindings) = ai_incident_anchor_sql(¶ms, &terms, 10_000); + + let conn = pool.get().unwrap(); + let mut stmt = conn.prepare(&format!("EXPLAIN QUERY PLAN {sql}")).unwrap(); + let plan = stmt + .query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + row.get::<_, String>(3) + }) + .unwrap() + .collect::>>() + .unwrap() + .join("\n"); + + assert!( + plan.contains("SCAN logs_fts VIRTUAL TABLE"), + "incident anchor query must drive from selective FTS terms; got:\n{plan}" + ); + assert!( + plan.contains("SEARCH l USING INTEGER PRIMARY KEY"), + "incident anchor query must resolve FTS rowids by the logs primary key; got:\n{plan}" + ); +} + +#[test] +fn investigate_ai_incidents_exact_id_can_fetch_beyond_top_ten() { + let (pool, _dir) = test_pool(); + let entries = (0..12) + .map(|i| { + make_ai_entry( + &format!("2026-01-01T00:{i:02}:00Z"), + "host-a", + "codex", + "/tmp/project", + &format!("sess-{i:02}"), + "this shit needs assessment", + ) + }) + .collect::>(); + insert_logs_batch(&pool, &entries).unwrap(); + + let listed = search_ai_incidents( + &pool, + &AiIncidentParams { + ai_project: Some("/tmp/project".into()), + ai_tool: Some("codex".into()), + limit: Some(12), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(listed.incidents.len(), 12); + let target_id = listed.incidents.last().unwrap().incident_id.clone(); + + let top_ten = investigate_ai_incidents( + &pool, + &AiInvestigateParams { + ai_project: Some("/tmp/project".into()), + ai_tool: Some("codex".into()), + limit: Some(10), + ..Default::default() + }, + ) + .unwrap(); + assert!( + !top_ten + .evidence + .iter() + .any(|bundle| bundle.incident.incident_id == target_id) + ); + + let exact = investigate_ai_incidents( + &pool, + &AiInvestigateParams { + incident_id: Some(target_id.clone()), + ai_project: Some("/tmp/project".into()), + ai_tool: Some("codex".into()), + limit: Some(1), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(exact.evidence.len(), 1); + assert_eq!(exact.evidence[0].incident.incident_id, target_id); + assert_eq!(exact.evidence[0].anchors.len(), 1); +} + +#[test] +fn ai_session_queries_respect_filters() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[ + make_ai_entry( + "2026-01-01T00:00:00Z", + "host-a", + "claude", + "/tmp/a", + "s1", + "auth needle", + ), + make_ai_entry( + "2026-01-01T01:00:00Z", + "host-b", + "codex", + "/tmp/b", + "s2", + "auth needle", + ), + make_ai_entry( + "2026-01-02T00:00:00Z", + "host-a", + "claude", + "/tmp/a", + "s3", + "auth needle", + ), + ], + ) + .unwrap(); + + let listed = list_ai_sessions( + &pool, + &ListAiSessionsParams { + ai_project: Some("/tmp/a".into()), + ai_tool: Some("claude".into()), + host: Some("host-a".into()), + since: Some("2026-01-01T00:00:00Z".into()), + until: Some("2026-01-01T23:59:59Z".into()), + limit: Some(10), + }, + ) + .unwrap(); + assert_eq!(listed.len(), 1); + assert_eq!(listed[0].ai_session_id, "s1"); + + let searched = search_ai_sessions( + &pool, + &SearchAiSessionsParams { + query: "needle".into(), + ai_project: Some("/tmp/b".into()), + ai_tool: Some("codex".into()), + host: None, + app: None, + since: Some("2026-01-01T00:30:00Z".into()), + until: Some("2026-01-01T01:30:00Z".into()), + limit: Some(10), + }, + ) + .unwrap(); + assert_eq!(searched.sessions.len(), 1); + assert_eq!(searched.sessions[0].ai_session_id, "s2"); + assert_eq!(searched.sessions[0].hostname, "host-b"); +} + +#[test] +fn list_ai_tool_and_project_inventory() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[ + make_ai_entry( + "2026-01-01T00:00:00Z", + "host-a", + "claude", + "/tmp/a", + "s1", + "one", + ), + make_ai_entry( + "2026-01-01T00:01:00Z", + "host-a", + "codex", + "/tmp/b", + "s2", + "two", + ), + make_ai_entry( + "2026-01-01T00:02:00Z", + "host-a", + "claude", + "/tmp/a", + "s1", + "three", + ), + ], + ) + .unwrap(); + + let tools = list_ai_tools(&pool, &ListAiToolsParams::default()).unwrap(); + assert_eq!(tools.tools.len(), 2); + let projects = list_ai_projects( + &pool, + &ListAiProjectsParams { + ai_tool: Some("claude".into()), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(projects.projects.len(), 1); + assert_eq!(projects.projects[0].project, "/tmp/a"); +} + +#[test] +fn list_ai_inventory_reports_truncation() { + // When truncated, total_X == len (the limit); truncated flag is authoritative. + let (pool, _dir) = test_pool(); + let mut entries = Vec::new(); + for i in 0..201 { + entries.push(make_ai_entry( + "2026-01-01T00:00:00Z", + "host-a", + &format!("tool-{i:03}"), + &format!("/tmp/project-{i:03}"), + &format!("session-{i:03}"), + "inventory", + )); + } + insert_logs_batch(&pool, &entries).unwrap(); + + let tools = list_ai_tools(&pool, &ListAiToolsParams::default()).unwrap(); + assert_eq!(tools.tools.len(), 100); + assert_eq!(tools.total_tools, 100); + assert!(tools.truncated); + + let projects = list_ai_projects(&pool, &ListAiProjectsParams::default()).unwrap(); + assert_eq!(projects.projects.len(), 200); + assert_eq!(projects.total_projects, 200); + assert!(projects.truncated); +} + +#[test] +fn list_ai_sessions_groups_by_project_tool_session_and_hostname() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[ + LogBatchEntry { + timestamp: "2026-05-11T00:00:00Z".into(), + hostname: "devhost".into(), + facility: Some("local7".into()), + severity: "info".into(), + app_name: Some("codex-transcript".into()), + process_id: None, + message: "{}".into(), + raw: "{}".into(), + source_ip: "10.0.0.1:514".into(), + docker_checkpoint: None, + ai_tool: Some("codex".into()), + ai_project: Some("/home/jmagar/workspace/cortex".into()), + ai_session_id: Some("abc".into()), + ai_transcript_path: Some( + "/home/jmagar/.codex/sessions/2026/05/11/rollout-abc.jsonl".into(), + ), + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + }, + LogBatchEntry { + timestamp: "2026-05-11T00:01:00Z".into(), + hostname: "devhost".into(), + facility: Some("local7".into()), + severity: "info".into(), + app_name: Some("codex-transcript".into()), + process_id: None, + message: "{}".into(), + raw: "{}".into(), + source_ip: "10.0.0.1:514".into(), + docker_checkpoint: None, + ai_tool: Some("codex".into()), + ai_project: Some("/home/jmagar/workspace/cortex".into()), + ai_session_id: Some("abc".into()), + ai_transcript_path: Some( + "/home/jmagar/.codex/sessions/2026/05/11/rollout-abc.jsonl".into(), + ), + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + }, + ], + ) + .unwrap(); + + let rows = list_ai_sessions( + &pool, + &ListAiSessionsParams { + ai_project: Some("/home/jmagar/workspace/cortex".into()), + limit: Some(10), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].ai_tool, "codex"); + assert_eq!(rows[0].ai_session_id, "abc"); + assert_eq!(rows[0].event_count, 2); + assert_eq!(rows[0].first_seen, "2026-05-11T00:00:00Z"); + assert_eq!(rows[0].last_seen, "2026-05-11T00:01:00Z"); +} + +// --------------------------------------------------------------------------- +// AI session rollup (bead cortex-2vre) correctness tests +// --------------------------------------------------------------------------- + +/// Insert a spread of AI sessions across projects/tools/sessions/hosts so the +/// rollup vs live comparison exercises real grouping. +fn seed_ai_sessions(pool: &DbPool) { + let mut batch = Vec::new(); + for s in 0..12u32 { + let tool = if s % 2 == 0 { "codex" } else { "claude" }; + let project = format!("/proj/{}", s % 3); + let session = format!("sess-{s}"); + let host = format!("host{}", s % 2); + let event_count = 3 + s % 4; + // Globally DISTINCT, strictly increasing timestamps per session so each + // session's MAX(last_seen) is unique — no ordering ties between the + // live and rollup paths (both order by last_seen DESC only). + for e in 0..event_count { + // Encode (session, event) into a unique minute/second so no two + // rows across sessions share a timestamp. + let total = s * 10 + e; // < 120, fits in minutes + let ts = format!("2026-05-01T{:02}:{:02}:00Z", total / 60, total % 60); + batch.push(make_ai_entry( + &ts, + &host, + tool, + &project, + &session, + &format!("event {e} of {session}"), + )); + } + } + insert_logs_batch(pool, &batch).unwrap(); +} + +fn default_session_params() -> ListAiSessionsParams { + ListAiSessionsParams { + ai_project: None, + ai_tool: None, + host: None, + since: None, + until: None, + limit: Some(100), + } +} + +#[test] +fn rollup_result_equals_live_aggregation() { + let (pool, _dir) = test_pool(); + seed_ai_sessions(&pool); + + // Before refresh: rollup empty => list_ai_sessions falls back to live. + let pre = list_ai_sessions(&pool, &default_session_params()).unwrap(); + let live = list_ai_sessions_live(&pool, &default_session_params()).unwrap(); + assert_eq!( + pre.len(), + live.len(), + "fallback must match live before refresh" + ); + + // After refresh: list_ai_sessions serves from the rollup and must match. + let total = refresh_ai_session_rollup(&pool).unwrap(); + assert_eq!(total, live.len(), "rollup row count must equal #sessions"); + assert_eq!( + ai_session_rollup_status(&pool).unwrap().row_count, + total as i64 + ); + + let rolled = list_ai_sessions(&pool, &default_session_params()).unwrap(); + assert_eq!(rolled.len(), live.len()); + for (l, r) in live.iter().zip(rolled.iter()) { + assert_eq!(l.ai_project, r.ai_project); + assert_eq!(l.ai_tool, r.ai_tool); + assert_eq!(l.ai_session_id, r.ai_session_id); + assert_eq!(l.hostname, r.hostname); + assert_eq!(l.first_seen, r.first_seen, "first_seen drift"); + assert_eq!(l.last_seen, r.last_seen, "last_seen drift"); + assert_eq!(l.event_count, r.event_count, "event_count drift"); + assert_eq!(l.ai_transcript_path, r.ai_transcript_path); + } +} + +#[test] +fn rollup_status_reports_refresh_time() { + let (pool, _dir) = test_pool(); + seed_ai_sessions(&pool); + + // Never refreshed => no staleness timestamp. + assert!( + ai_session_rollup_status(&pool) + .unwrap() + .refreshed_at + .is_none() + ); + + refresh_ai_session_rollup(&pool).unwrap(); + let status = ai_session_rollup_status(&pool).unwrap(); + assert!(status.refreshed_at.is_some(), "refreshed_at must be set"); + assert!(status.row_count > 0); + assert!(status.summary().contains("refreshed")); +} + +/// The key correctness guarantee the design rests on: a REFRESH-based rollup +/// stays exact under DELETE, including when the deleted rows held a session's +/// MIN/MAX timestamp — the exact case where a trigger-maintained rollup would +/// silently corrupt first_seen/last_seen. +#[test] +fn rollup_is_exact_after_deletes_recompute_min_max() { + let (pool, _dir) = test_pool(); + seed_ai_sessions(&pool); + refresh_ai_session_rollup(&pool).unwrap(); + + // Delete the single newest event of every session (the rows holding MAX + // timestamp) plus the oldest of one session (holding MIN). A stale rollup + // would keep the now-deleted extremes; a refresh must recompute them. + { + let conn = pool.get().unwrap(); + // Delete the global newest 12 rows (roughly the MAX-holding rows). + conn.execute( + "DELETE FROM logs WHERE id IN ( + SELECT id FROM logs + WHERE ai_session_id IS NOT NULL + ORDER BY timestamp DESC LIMIT 12 + )", + [], + ) + .unwrap(); + // Delete the global oldest row (a MIN-holding row). + conn.execute( + "DELETE FROM logs WHERE id IN ( + SELECT id FROM logs + WHERE ai_session_id IS NOT NULL + ORDER BY timestamp ASC LIMIT 1 + )", + [], + ) + .unwrap(); + } + + // Stale rollup (not yet refreshed) may now disagree with live — prove the + // refresh restores exactness against a fresh live aggregation. + refresh_ai_session_rollup(&pool).unwrap(); + let live = list_ai_sessions_live(&pool, &default_session_params()).unwrap(); + let rolled = list_ai_sessions(&pool, &default_session_params()).unwrap(); + assert_eq!(live.len(), rolled.len(), "post-delete row count mismatch"); + for (l, r) in live.iter().zip(rolled.iter()) { + assert_eq!(l.ai_session_id, r.ai_session_id); + assert_eq!( + l.first_seen, r.first_seen, + "MIN not recomputed after delete" + ); + assert_eq!(l.last_seen, r.last_seen, "MAX not recomputed after delete"); + assert_eq!( + l.event_count, r.event_count, + "count not recomputed after delete" + ); + } +} + +/// The staging+swap refresh (bead syslog-mcp-rvcz) MUST stay a correct FULL +/// recompute under retention DELETEs — the exact case a watermark-incremental +/// refresh would silently corrupt. This is the test that distinguishes the +/// (correct) staging+swap from the (corrupt) incremental trap: +/// * purge the OLDEST rows of a SURVIVING session -> first_seen must advance +/// to the surviving MIN (an append-keyed incremental would keep the stale +/// deleted minimum); +/// * fully purge ANOTHER session's rows entirely -> its rollup row must be +/// EVICTED (an append-keyed incremental would leave a ghost session); +/// * event_count must equal the live COUNT(*) -> no upward drift. +/// +/// The post-refresh rollup must be byte-for-byte equal to a from-scratch live +/// aggregation over the surviving rows. +#[test] +fn rollup_stays_correct_under_concurrent_retention() { + let (pool, _dir) = test_pool(); + seed_ai_sessions(&pool); + // Initial full materialization (all 12 seeded sessions present). + refresh_ai_session_rollup(&pool).unwrap(); + let sessions_before = list_ai_sessions(&pool, &default_session_params()) + .unwrap() + .len(); + assert!( + sessions_before >= 2, + "need >=2 sessions to exercise eviction" + ); + + // Pick a SURVIVING session and capture its current first_seen + the + // timestamp of its single oldest row (which we will purge). + let (surviving, old_first_seen, oldest_ts): (String, String, String) = { + let conn = pool.get().unwrap(); + conn.query_row( + "SELECT ai_session_id, MIN(timestamp) FROM logs + WHERE ai_session_id IS NOT NULL GROUP BY ai_session_id + ORDER BY COUNT(*) DESC LIMIT 1", + [], + |r| { + let sid: String = r.get(0)?; + let min_ts: String = r.get(1)?; + Ok((sid.clone(), min_ts.clone(), min_ts)) + }, + ) + .unwrap() + }; + + // Pick a DIFFERENT session to fully purge. + let purged: String = { + let conn = pool.get().unwrap(); + conn.query_row( + "SELECT ai_session_id FROM logs + WHERE ai_session_id IS NOT NULL AND ai_session_id != ?1 + LIMIT 1", + params![surviving], + |r| r.get(0), + ) + .unwrap() + }; + assert_ne!(surviving, purged); + + // Retention purge (mimics maintenance.rs deleting oldest/budget rows with + // NO severity exemption): drop the oldest row of the surviving session AND + // every row of the purged session. + { + let conn = pool.get().unwrap(); + let dropped_old = conn + .execute( + "DELETE FROM logs + WHERE ai_session_id = ?1 AND timestamp = ?2", + params![surviving, oldest_ts], + ) + .unwrap(); + assert!( + dropped_old >= 1, + "must purge the surviving session's oldest row" + ); + let dropped_all = conn + .execute("DELETE FROM logs WHERE ai_session_id = ?1", params![purged]) + .unwrap(); + assert!(dropped_all >= 1, "must fully purge the other session"); + } + + // Refresh AFTER the purge. A correct full recompute (staging+swap) restores + // exactness; the incremental trap would not. + refresh_ai_session_rollup(&pool).unwrap(); + + let rolled = list_ai_sessions(&pool, &default_session_params()).unwrap(); + let live = list_ai_sessions_live(&pool, &default_session_params()).unwrap(); + + // (1) Ghost eviction: the fully-purged session must NOT remain in the rollup. + assert!( + rolled.iter().all(|s| s.ai_session_id != purged), + "fully-purged session must be evicted from the rollup (no ghost row)" + ); + assert_eq!( + rolled.len(), + sessions_before - 1, + "exactly one session should have been evicted" + ); + + // (2) first_seen advanced: the surviving session's MIN must move past the + // now-deleted oldest row. + let surv = rolled + .iter() + .find(|s| s.ai_session_id == surviving) + .expect("surviving session must remain in the rollup"); + assert_ne!( + surv.first_seen, old_first_seen, + "first_seen must advance after the oldest row was purged (incremental \ + would keep the stale minimum)" + ); + let live_surv = live + .iter() + .find(|s| s.ai_session_id == surviving) + .expect("surviving session must be in live aggregation"); + assert_eq!( + surv.first_seen, live_surv.first_seen, + "first_seen must equal the surviving MIN(timestamp)" + ); + + // (3) Byte-for-byte equal to a from-scratch live recompute over survivors. + assert_eq!( + rolled.len(), + live.len(), + "row count must match live recompute" + ); + for (r, l) in rolled.iter().zip(live.iter()) { + assert_eq!(r.ai_session_id, l.ai_session_id); + assert_eq!(r.first_seen, l.first_seen, "first_seen drift vs live"); + assert_eq!(r.last_seen, l.last_seen, "last_seen drift vs live"); + assert_eq!( + r.event_count, l.event_count, + "event_count must equal actual COUNT(*) (no drift)" + ); + } +} + +/// Regression for the R1 guard (bead syslog-mcp-rvcz): rows that carry +/// `ai_project` but have NO recognized `ai_tool`/`ai_session_id` (e.g. OTLP +/// logs with only project.path) are counted by the broad `ai_rows_watermark` +/// predicate but correctly EXCLUDED by the rollup's full GROUP BY predicate. +/// The old guard compared `staged` against the broad watermark `src_count`, so +/// for this data shape `staged == 0` while `src_count > 0` and the refresh +/// ERRORED forever. A legitimately-empty rollup must SUCCEED (returning 0 and +/// still stamping the meta/fingerprint), not raise the R1 error. +#[test] +fn rollup_empty_when_only_broad_project_rows_present_succeeds() { + let (pool, _dir) = test_pool(); + + // Insert rows with ai_project set but ai_tool / ai_session_id EMPTY. These + // match the broad watermark predicate (ai_project NOT NULL/!='') but fail + // the full rollup predicate (which also requires ai_tool and ai_session_id + // NOT NULL/!=''), so the staging GROUP BY yields zero groups. + let batch = vec![ + // empty ai_tool + make_ai_entry( + "2026-05-01T00:00:00Z", + "host0", + "", + "/proj/a", + "sess-1", + "otlp event, no tool", + ), + // empty ai_session_id + make_ai_entry( + "2026-05-01T00:01:00Z", + "host0", + "claude", + "/proj/a", + "", + "otlp event, no session", + ), + // both empty + make_ai_entry( + "2026-05-01T00:02:00Z", + "host1", + "", + "/proj/b", + "", + "otlp event, project only", + ), + ]; + insert_logs_batch(&pool, &batch).unwrap(); + + // The watermark sees these rows (broad predicate) so src_count > 0; the + // rollup predicate excludes them all so staging is legitimately empty. + // Pre-fix this raised the R1 error; post-fix it must SUCCEED with 0 rows. + let total = refresh_ai_session_rollup(&pool).unwrap(); + assert_eq!( + total, 0, + "rollup must be legitimately empty (no rollup-eligible rows)" + ); + + // The meta/fingerprint MUST still be stamped on an empty rollup so + // refresh_ai_session_rollup_if_stale can skip subsequent no-op refreshes. + let status = ai_session_rollup_status(&pool).unwrap(); + assert_eq!(status.row_count, 0, "rollup row_count must be 0"); + assert!( + status.refreshed_at.is_some(), + "meta/fingerprint must be stamped even for an empty rollup" + ); +} + +#[test] +fn rollup_read_uses_last_seen_index_no_temp_btree() { + let (pool, _dir) = test_pool(); + seed_ai_sessions(&pool); + refresh_ai_session_rollup(&pool).unwrap(); + // The unbounded rollup read must be served by the last_seen index, NOT a + // temp b-tree sort (the cost that plagued the live aggregation). The query + // mirrors list_ai_sessions_from_rollup. + let plan = query_plan( + &pool, + "SELECT ai_project, ai_tool, ai_session_id, ai_transcript_path, + hostname, first_seen, last_seen, event_count + FROM ai_session_rollup + WHERE 1=1 + ORDER BY last_seen DESC LIMIT 100", + &[], + ); + assert!( + plan.contains("idx_ai_session_rollup_last_seen"), + "rollup read must use the last_seen index; plan was:\n{plan}" + ); + assert!( + !plan.contains("TEMP B-TREE"), + "rollup read must avoid a temp b-tree sort; plan was:\n{plan}" + ); +} + +#[test] +fn rollup_respects_project_and_tool_filters() { + let (pool, _dir) = test_pool(); + seed_ai_sessions(&pool); + refresh_ai_session_rollup(&pool).unwrap(); + + for (project, tool) in [ + (Some("/proj/0".to_string()), None), + (None, Some("codex".to_string())), + (Some("/proj/1".to_string()), Some("claude".to_string())), + ] { + let params = ListAiSessionsParams { + ai_project: project.clone(), + ai_tool: tool.clone(), + ..default_session_params() + }; + let live = list_ai_sessions_live(&pool, ¶ms).unwrap(); + let rolled = list_ai_sessions(&pool, ¶ms).unwrap(); + assert_eq!( + live.iter().map(|s| &s.ai_session_id).collect::>(), + rolled.iter().map(|s| &s.ai_session_id).collect::>(), + "filtered rollup ({project:?},{tool:?}) must match live" + ); + } +} + +#[test] +fn time_windowed_sessions_always_use_live_path() { + let (pool, _dir) = test_pool(); + seed_ai_sessions(&pool); + refresh_ai_session_rollup(&pool).unwrap(); + + // Insert a brand-new event AFTER the rollup was built. A time-windowed + // query must see it live (rollup is stale and must be bypassed). + insert_logs_batch( + &pool, + &[make_ai_entry( + "2026-06-01T12:00:00Z", + "host0", + "codex", + "/proj/0", + "sess-0", + "fresh post-refresh event", + )], + ) + .unwrap(); + + let windowed = ListAiSessionsParams { + since: Some("2026-06-01T00:00:00Z".into()), + until: Some("2026-06-02T00:00:00Z".into()), + ..default_session_params() + }; + let rows = list_ai_sessions(&pool, &windowed).unwrap(); + assert_eq!( + rows.len(), + 1, + "windowed query must see the fresh event live" + ); + assert_eq!(rows[0].last_seen, "2026-06-01T12:00:00Z"); + assert_eq!( + rows[0].event_count, 1, + "windowed count must be live, not rollup" + ); +} + +// --------------------------------------------------------------------------- +// Rollup source-watermark dirty-check (bead cortex-g33v) +// --------------------------------------------------------------------------- + +/// Helper: insert a single non-AI log row (no ai_* fields). Such rows must NOT +/// move the AI watermark, so a refresh that follows them is skipped. +fn insert_plain_log(pool: &DbPool, ts: &str, msg: &str) { + insert_logs_batch(pool, &[make_entry(ts, "host0", "info", msg)]).unwrap(); +} + +#[test] +fn stale_check_first_refresh_runs_then_noop_is_skipped() { + let (pool, _dir) = test_pool(); + seed_ai_sessions(&pool); + + // Never refreshed yet => must refresh. + match refresh_ai_session_rollup_if_stale(&pool).unwrap() { + RollupRefresh::Refreshed { row_count } => assert!(row_count > 0), + RollupRefresh::Skipped => panic!("first refresh must not be skipped"), + } + + // Nothing changed => the expensive re-aggregation must be skipped. + assert_eq!( + refresh_ai_session_rollup_if_stale(&pool).unwrap(), + RollupRefresh::Skipped, + "unchanged source must skip the refresh" + ); +} + +#[test] +fn stale_check_detects_new_ai_row() { + let (pool, _dir) = test_pool(); + seed_ai_sessions(&pool); + refresh_ai_session_rollup_if_stale(&pool).unwrap(); + assert_eq!( + refresh_ai_session_rollup_if_stale(&pool).unwrap(), + RollupRefresh::Skipped + ); + + // A new AI row advances MAX(id) => must refresh. + insert_logs_batch( + &pool, + &[make_ai_entry( + "2026-07-01T00:00:00Z", + "host9", + "codex", + "/proj/new", + "sess-new", + "brand new ai event", + )], + ) + .unwrap(); + assert!( + matches!( + refresh_ai_session_rollup_if_stale(&pool).unwrap(), + RollupRefresh::Refreshed { .. } + ), + "a new AI row must trigger a refresh" + ); + // And the new session is now visible from the rollup path. + let rows = list_ai_sessions(&pool, &default_session_params()).unwrap(); + assert!(rows.iter().any(|s| s.ai_session_id == "sess-new")); +} + +#[test] +fn stale_check_detects_deleted_ai_row() { + let (pool, _dir) = test_pool(); + seed_ai_sessions(&pool); + refresh_ai_session_rollup_if_stale(&pool).unwrap(); + assert_eq!( + refresh_ai_session_rollup_if_stale(&pool).unwrap(), + RollupRefresh::Skipped + ); + + // Deleting an AI row changes COUNT(*) (and likely MAX(id)) => must refresh. + { + let conn = pool.get().unwrap(); + let deleted = conn + .execute( + "DELETE FROM logs WHERE id IN ( + SELECT id FROM logs WHERE ai_session_id IS NOT NULL LIMIT 1 + )", + [], + ) + .unwrap(); + assert_eq!(deleted, 1, "test must delete exactly one AI row"); + } + assert!( + matches!( + refresh_ai_session_rollup_if_stale(&pool).unwrap(), + RollupRefresh::Refreshed { .. } + ), + "a deleted AI row must trigger a refresh" + ); +} + +#[test] +fn stale_check_ignores_non_ai_rows() { + let (pool, _dir) = test_pool(); + seed_ai_sessions(&pool); + refresh_ai_session_rollup_if_stale(&pool).unwrap(); + + // Plain syslog rows (the overwhelming majority of ingest) must NOT force a + // re-aggregation — that is the whole point of an AI-scoped watermark. + insert_plain_log(&pool, "2026-07-01T00:00:00Z", "ordinary syslog line"); + insert_plain_log(&pool, "2026-07-01T00:00:01Z", "another ordinary line"); + assert_eq!( + refresh_ai_session_rollup_if_stale(&pool).unwrap(), + RollupRefresh::Skipped, + "non-AI ingest must not trigger a rollup refresh" + ); +} + +#[test] +fn stale_check_watermark_is_index_only_no_table_scan() { + // The watermark fingerprint must be cheap: served from the partial index + // idx_logs_ai_project_time (WHERE ai_project IS NOT NULL), NOT a full scan + // of `logs`. That index-only cost is the whole reason the dirty-check is + // worth running on every cadence tick. Mirrors ai_rows_watermark's query. + let (pool, _dir) = test_pool(); + seed_ai_sessions(&pool); + let plan = query_plan( + &pool, + "SELECT COUNT(*), COALESCE(MAX(id), 0) FROM logs + WHERE ai_project IS NOT NULL AND ai_project != ''", + &[], + ); + assert!( + plan.contains("idx_logs_ai_project_time"), + "watermark must use the AI partial index; plan was:\n{plan}" + ); + assert!( + !plan.contains("SCAN logs\n") && !plan.ends_with("SCAN logs"), + "watermark must not full-scan logs; plan was:\n{plan}" + ); +} + +#[test] +fn stale_check_skip_keeps_rollup_correct_vs_live() { + // A skipped refresh must leave the rollup serving results identical to a + // fresh live aggregation (i.e. skipping never serves stale data when the + // source genuinely did not change). + let (pool, _dir) = test_pool(); + seed_ai_sessions(&pool); + refresh_ai_session_rollup_if_stale(&pool).unwrap(); + assert_eq!( + refresh_ai_session_rollup_if_stale(&pool).unwrap(), + RollupRefresh::Skipped + ); + + let live = list_ai_sessions_live(&pool, &default_session_params()).unwrap(); + let rolled = list_ai_sessions(&pool, &default_session_params()).unwrap(); + assert_eq!(live.len(), rolled.len()); + for (l, r) in live.iter().zip(rolled.iter()) { + assert_eq!(l.ai_session_id, r.ai_session_id); + assert_eq!(l.last_seen, r.last_seen); + assert_eq!(l.event_count, r.event_count); + } +} + +// --------------------------------------------------------------------------- +// RAG v1 tests +// --------------------------------------------------------------------------- + +fn make_app_entry(ts: &str, host: &str, severity: &str, app: &str, msg: &str) -> LogBatchEntry { + LogBatchEntry { + timestamp: ts.to_string(), + hostname: host.to_string(), + facility: None, + severity: severity.to_string(), + app_name: Some(app.to_string()), + process_id: None, + message: msg.to_string(), + raw: msg.to_string(), + source_ip: "10.0.0.1:514".to_string(), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + } +} + +#[test] +fn similar_incidents_clusters_returns_clusters_for_matching_logs() { + let (pool, _dir) = test_pool(); + + let logs = vec![ + make_app_entry( + "2024-01-15T10:00:00Z", + "web-01", + "err", + "nginx", + "upstream connect error timeout", + ), + make_app_entry( + "2024-01-15T10:05:00Z", + "web-01", + "crit", + "nginx", + "upstream connect error connection refused", + ), + ]; + insert_logs_batch(&pool, &logs).unwrap(); + + let params = SimilarIncidentsParams { + query: "upstream".into(), + host: None, + app: None, + severity_min: None, + since: None, + until: None, + window_minutes: Some(30), + limit: Some(10), + }; + let result = similar_incidents_clusters(&pool, ¶ms).unwrap(); + assert!(!result.clusters.is_empty(), "expected at least one cluster"); + let cluster = &result.clusters[0]; + assert_eq!(cluster.hostname, "web-01"); + assert_eq!(cluster.app_name.as_deref(), Some("nginx")); + assert!(cluster.log_count >= 2); + // "crit" is more severe than "err" + assert_eq!(cluster.severity_peak, "crit"); +} + +#[test] +fn similar_incidents_clusters_filters_by_hostname() { + let (pool, _dir) = test_pool(); + + let logs = vec![ + make_app_entry( + "2024-01-15T10:00:00Z", + "web-01", + "err", + "nginx", + "upstream connect error", + ), + make_app_entry( + "2024-01-15T10:01:00Z", + "web-02", + "err", + "nginx", + "upstream connect error", + ), + ]; + insert_logs_batch(&pool, &logs).unwrap(); + + let params = SimilarIncidentsParams { + query: "upstream".into(), + host: Some("web-01".into()), + ..Default::default() + }; + let result = similar_incidents_clusters(&pool, ¶ms).unwrap(); + assert!(result.clusters.iter().all(|c| c.hostname == "web-01")); +} + +#[test] +fn similar_incidents_applies_fts_before_candidate_cap() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[make_app_entry( + "2024-01-01T00:00:00Z", + "web-01", + "err", + "nginx", + "historicalincidentneedle connection refused", + )], + ) + .unwrap(); + + // Populate exactly the old raw-log candidate cap with newer nonmatches. + // A recency cap applied before FTS excludes the historical match above. + let conn = pool.get().unwrap(); + conn.execute_batch( + "WITH digits(d) AS ( + VALUES (0),(1),(2),(3),(4),(5),(6),(7),(8),(9) + ), + rows(n) AS ( + SELECT a.d + 10*b.d + 100*c.d + 1000*d.d + 10000*e.d + FROM digits a + CROSS JOIN digits b + CROSS JOIN digits c + CROSS JOIN digits d + CROSS JOIN digits e + ) + INSERT INTO logs + (timestamp, hostname, severity, app_name, message, raw, source_ip) + SELECT '2024-01-02T00:00:00Z', 'web-01', 'info', 'nginx', + 'routine health check', 'routine health check', '10.0.0.1:514' + FROM rows;", + ) + .unwrap(); + drop(conn); + + let result = similar_incidents_clusters( + &pool, + &SimilarIncidentsParams { + query: "historicalincidentneedle".into(), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.clusters.len(), 1); + assert_eq!(result.clusters[0].window_start, "2024-01-01T00:00:00Z"); + assert_eq!(result.clusters[0].log_count, 1); +} + +#[test] +fn incident_context_summary_returns_window_stats() { + let (pool, _dir) = test_pool(); + + let logs = vec![ + make_app_entry( + "2024-02-01T08:00:00Z", + "db-01", + "err", + "postgres", + "FATAL: out of shared memory", + ), + make_app_entry( + "2024-02-01T08:01:00Z", + "db-01", + "info", + "postgres", + "database system is ready", + ), + ]; + insert_logs_batch(&pool, &logs).unwrap(); + + let params = IncidentContextParams { + since: "2024-02-01T07:00:00Z".into(), + until: "2024-02-01T09:00:00Z".into(), + host: None, + app: None, + query: None, + severity_min: Some("err".into()), + limit: Some(10), + }; + let result = incident_context_summary(&pool, ¶ms).unwrap(); + assert_eq!(result.total_logs, 2); + assert!(!result.by_severity.is_empty()); + // Only the "err" row should be in error_logs (not "info") + assert_eq!(result.error_logs.len(), 1); + assert_eq!(result.error_logs[0].message, "FATAL: out of shared memory"); +} + +#[test] +fn incident_context_summary_empty_window_returns_zero() { + let (pool, _dir) = test_pool(); + + let params = IncidentContextParams { + since: "2020-01-01T00:00:00Z".into(), + until: "2020-01-02T00:00:00Z".into(), + ..Default::default() + }; + let result = incident_context_summary(&pool, ¶ms).unwrap(); + assert_eq!(result.total_logs, 0); + assert!(result.error_logs.is_empty()); + assert!(result.ai_sessions.is_empty()); +} + +#[test] +fn incident_context_summary_filters_error_logs_by_fts_query() { + let (pool, _dir) = test_pool(); + insert_logs_batch( + &pool, + &[ + make_app_entry( + "2024-02-01T08:00:00Z", + "db-01", + "err", + "postgres", + "shared memory exhausted", + ), + make_app_entry( + "2024-02-01T08:01:00Z", + "db-01", + "err", + "postgres", + "connection pool exhausted", + ), + ], + ) + .unwrap(); + + let result = incident_context_summary( + &pool, + &IncidentContextParams { + since: "2024-02-01T07:00:00Z".into(), + until: "2024-02-01T09:00:00Z".into(), + query: Some("memory".into()), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.total_logs, 2, "window aggregates remain unfiltered"); + assert_eq!(result.error_logs.len(), 1); + assert_eq!(result.error_logs[0].message, "shared memory exhausted"); +} + +#[test] +fn incident_context_window_queries_force_timestamp_index() { + let (pool, _dir) = test_pool(); + let plan = query_plan( + &pool, + "SELECT COUNT(*) + FROM logs INDEXED BY idx_logs_timestamp + WHERE (ai_project IS NULL OR ai_project = '') + AND timestamp BETWEEN ?1 AND ?2", + &[ + rusqlite::types::Value::Text("2024-02-01T07:00:00Z".into()), + rusqlite::types::Value::Text("2024-02-01T09:00:00Z".into()), + ], + ); + assert!( + plan.contains("idx_logs_timestamp"), + "incident context window scan should be timestamp-index driven; got:\n{plan}" + ); +} + +// ─────────────────────────────────────────────────────────────────────────── +// Performance benchmark harness (Issue 4 / bead cortex-2vre). +// +// Builds a synthetic on-disk SQLite DB with a realistic row count and times +// `get_stats` and `list_ai_sessions` before/after the optimization work. +// +// IGNORED by default — it builds millions of rows and takes minutes, so it +// must never run in the normal `cargo nextest` suite. Run explicitly: +// +// CORTEX_BENCH_ROWS=10000000 cargo test --lib \ +// db::queries::tests::bench_stats_and_sessions -- --ignored --nocapture +// +// Row count is controlled by CORTEX_BENCH_ROWS (default 5_000_000). +// ─────────────────────────────────────────────────────────────────────────── + +/// Insert `n` synthetic log rows through the live schema (FTS + inventory + +/// counter triggers all fire), in large transactions for throughput. ~20% of +/// rows carry AI session fields spread across many (project, tool, session) +/// groups so the sessions query has realistic cardinality. +fn bench_seed_rows(pool: &DbPool, n: usize) { + use std::time::Instant; + let started = Instant::now(); + const CHUNK: usize = 50_000; + let mut inserted = 0usize; + while inserted < n { + let this = CHUNK.min(n - inserted); + let mut conn = pool.get().unwrap(); + let tx = conn.transaction().unwrap(); + { + let mut stmt = tx + .prepare_cached( + "INSERT INTO logs (timestamp, hostname, facility, severity, app_name, + process_id, message, raw, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path) + VALUES (?1,?2,?3,?4,?5,?6,?7,?8,?9,?10,?11,?12,?13,?14)", + ) + .unwrap(); + for i in 0..this { + let x = inserted + i; + // Strictly-increasing timestamp per row (base + x seconds, full + // date rollover). Monotonic in `x` => each session's MAX(ts) is + // globally unique (no last_seen ties), so the rollup vs live + // top-N comparison is deterministic at the LIMIT boundary. + let base = chrono::DateTime::parse_from_rfc3339("2026-01-01T00:00:00Z").unwrap(); + let dt = base + chrono::TimeDelta::seconds(x as i64); + let ts = dt.format("%Y-%m-%dT%H:%M:%SZ").to_string(); + let recv = ts.clone(); + let host = format!("host{:02}", x % 25); + let app = format!("app{:02}", x % 60); + let msg = format!("synthetic log line {x} some error retry connection text"); + let is_ai = x.is_multiple_of(5); + let (tool, proj, sess, tpath): ( + Option, + Option, + Option, + Option, + ) = if is_ai { + let proj = format!("/proj/{}", x % 40); + let tool = if x.is_multiple_of(2) { + "codex" + } else { + "claude" + } + .to_string(); + // ~20k distinct sessions => realistic group cardinality. + let sess = format!("sess-{}", x % 20_000); + let tpath = format!("{proj}/{sess}.jsonl"); + (Some(tool), Some(proj), Some(sess), Some(tpath)) + } else { + (None, None, None, None) + }; + stmt.execute(rusqlite::params![ + ts, + host, + Option::::None, + "info", + app, + Option::::None, + msg, + "raw", + recv, + "10.0.0.1:514", + tool, + proj, + sess, + tpath, + ]) + .unwrap(); + } + } + tx.commit().unwrap(); + inserted += this; + } + eprintln!( + "[bench] seeded {inserted} rows in {:.1}s", + started.elapsed().as_secs_f64() + ); +} + +/// Median of N timed runs of `f`, in milliseconds. One warm-up run first. +fn bench_median_ms(runs: usize, mut f: impl FnMut()) -> f64 { + use std::time::Instant; + f(); // warm-up + let mut samples: Vec = Vec::with_capacity(runs); + for _ in 0..runs { + let t = Instant::now(); + f(); + samples.push(t.elapsed().as_secs_f64() * 1000.0); + } + samples.sort_by(|a, b| a.partial_cmp(b).unwrap()); + samples[samples.len() / 2] +} + +#[test] +#[ignore = "performance benchmark; builds millions of rows. Run with --ignored."] +fn bench_stats_and_sessions() { + let rows: usize = std::env::var("CORTEX_BENCH_ROWS") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(5_000_000); + + // Optional persistent DB path so a seeded DB can be reused across runs + // (seeding 10M rows takes ~13 min). When unset, use a throwaway tempdir. + let _guard_dir; + let (pool, cfg) = if let Ok(path) = std::env::var("CORTEX_BENCH_DB") { + let db_path = std::path::PathBuf::from(&path); + let fresh = !db_path.exists(); + let cfg = test_storage_config(db_path); + let pool = init_pool(&cfg).unwrap(); + if fresh { + bench_seed_rows(&pool, rows); + } else { + eprintln!("[bench] reusing existing DB at {path} (skipping seed)"); + } + (pool, cfg) + } else { + let dir = tempfile::tempdir().unwrap(); + let cfg = test_storage_config(dir.path().join("test.db")); + let pool = init_pool(&cfg).unwrap(); + bench_seed_rows(&pool, rows); + _guard_dir = dir; // keep alive + (pool, cfg) + }; + + let ground_truth: i64 = { + let conn = pool.get().unwrap(); + conn.query_row("SELECT COUNT(*) FROM logs", [], |r| r.get(0)) + .unwrap() + }; + eprintln!("[bench] ground-truth COUNT(*) FROM logs = {ground_truth}"); + + // --- stats (default: FTS diagnostic skipped) --- + let mut last_stats = None; + let stats_ms = bench_median_ms(5, || { + last_stats = Some(get_stats(&pool, &cfg).unwrap()); + }); + let stats = last_stats.unwrap(); + eprintln!( + "[bench] get_stats (default, FTS skipped): {stats_ms:.1} ms (total_logs={})", + stats.total_logs + ); + assert_eq!( + stats.total_logs, ground_truth, + "stats total_logs must equal ground-truth COUNT(*)" + ); + + // --- stats with FTS diagnostic ON (the expensive COUNT(*) FROM logs_fts) --- + let stats_fts_ms = bench_median_ms(3, || { + let _ = get_stats_with_options(&pool, &cfg, true).unwrap(); + }); + eprintln!("[bench] get_stats (FTS diagnostic ON): {stats_fts_ms:.1} ms"); + + // --- sessions BEFORE: live aggregation (GROUP BY + temp-btree sort) --- + let params = ListAiSessionsParams { + ai_project: None, + ai_tool: None, + host: None, + since: None, + until: None, + limit: Some(100), + }; + let mut live_rows = 0usize; + let sessions_live_ms = bench_median_ms(5, || { + live_rows = list_ai_sessions_live(&pool, ¶ms).unwrap().len(); + }); + eprintln!( + "[bench] BEFORE list_ai_sessions_live(limit=100): {sessions_live_ms:.1} ms ({live_rows} rows)" + ); + + // --- refresh cost (background cadence; not on the request path) --- + let mut rollup_total = 0usize; + let refresh_ms = bench_median_ms(3, || { + rollup_total = refresh_ai_session_rollup(&pool).unwrap(); + }); + eprintln!( + "[bench] refresh_ai_session_rollup: {refresh_ms:.1} ms ({rollup_total} session rows total)" + ); + + // --- sessions AFTER: indexed read from the rollup materialization --- + let mut rollup_rows = 0usize; + let sessions_rollup_ms = bench_median_ms(5, || { + rollup_rows = list_ai_sessions(&pool, ¶ms).unwrap().len(); + }); + eprintln!( + "[bench] AFTER list_ai_sessions(rollup, limit=100): {sessions_rollup_ms:.1} ms ({rollup_rows} rows)" + ); + + // Correctness: the rollup-served top-N must equal the live top-N. Both + // paths order by `last_seen DESC` only, so rows that TIE on last_seen may + // appear in different relative order between the two plans. Compare in a + // tie-order-independent way: (a) the multiset of last_seen ordering keys + // must be identical, and (b) the per-session (last_seen, event_count) facts + // must match for every returned session. + let live = list_ai_sessions_live(&pool, ¶ms).unwrap(); + let rollup = list_ai_sessions(&pool, ¶ms).unwrap(); + assert_eq!(live.len(), rollup.len(), "rollup/live row count mismatch"); + let mut live_keys: Vec<&String> = live.iter().map(|s| &s.last_seen).collect(); + let mut rollup_keys: Vec<&String> = rollup.iter().map(|s| &s.last_seen).collect(); + live_keys.sort(); + rollup_keys.sort(); + assert_eq!( + live_keys, rollup_keys, + "rollup/live last_seen ordering-key multisets differ" + ); + let live_facts: std::collections::HashMap<_, _> = live + .iter() + .map(|s| { + ( + (&s.ai_project, &s.ai_tool, &s.ai_session_id, &s.hostname), + (&s.last_seen, s.event_count), + ) + }) + .collect(); + for r in &rollup { + let key = (&r.ai_project, &r.ai_tool, &r.ai_session_id, &r.hostname); + match live_facts.get(&key) { + Some((last_seen, count)) => { + assert_eq!(*last_seen, &r.last_seen, "last_seen mismatch for {key:?}"); + assert_eq!(*count, r.event_count, "event_count mismatch for {key:?}"); + } + None => panic!("rollup returned session not in live top-N: {key:?}"), + } + } + + let speedup = sessions_live_ms / sessions_rollup_ms.max(0.001); + eprintln!( + "[bench] SUMMARY rows={rows} \ + stats_default_ms={stats_ms:.1} stats_fts_on_ms={stats_fts_ms:.1} \ + sessions_BEFORE_live_ms={sessions_live_ms:.1} \ + sessions_AFTER_rollup_ms={sessions_rollup_ms:.1} \ + refresh_ms={refresh_ms:.1} sessions_speedup={speedup:.1}x" + ); +} + +/// full-review QM2: the 15-column log projection is written inline at ~14 +/// sites across queries.rs / analytics.rs / ingest.rs, and `map_row` / +/// `map_row_offset` / `map_row_with_raw` read columns BY ORDINAL POSITION — +/// reordering or inserting a column at one site without updating the readers +/// silently mis-maps fields with no compile error. This drift test extracts +/// every projection that ends in `metadata_json` from the source text and +/// asserts it carries the canonical column order. (`map_row_with_raw` selects +/// `..., metadata_json, raw`; the canonical prefix still applies.) +#[test] +fn inline_log_projections_match_map_row_column_order() { + // Two canonical shapes exist: `map_row` (15 cols) and `map_row_with_raw` + // (16 cols, `raw` between `message` and `received_at`). + const CANON: &str = "id timestamp hostname facility severity app_name process_id message \ + received_at source_ip ai_tool ai_project ai_session_id ai_transcript_path metadata_json"; + const CANON_WITH_RAW: &str = "id timestamp hostname facility severity app_name process_id \ + message raw received_at source_ip ai_tool ai_project ai_session_id ai_transcript_path \ + metadata_json"; + let canon_tokens: Vec<&str> = CANON.split_whitespace().collect(); + let canon_raw_tokens: Vec<&str> = CANON_WITH_RAW.split_whitespace().collect(); + + let sources = [ + ("queries.rs", include_str!("queries.rs")), + ("analytics.rs", include_str!("analytics.rs")), + ("ingest.rs", include_str!("ingest.rs")), + ]; + let re = regex::Regex::new( + r"SELECT\s+((?:[a-zA-Z_][a-zA-Z_0-9]*\.)?id[\sa-zA-Z_0-9,.\\]*?metadata_json)", + ) + .unwrap(); + + let mut checked = 0usize; + for (name, src) in sources { + for cap in re.captures_iter(src) { + let projection = &cap[1]; + let tokens: Vec = projection + .split([',', '\\']) + .map(|t| t.trim()) + .filter(|t| !t.is_empty()) + .map(|t| { + // Strip any table alias prefix ("l.id" -> "id"). + t.rsplit('.').next().unwrap_or(t).to_string() + }) + .collect(); + assert!( + tokens == canon_tokens || tokens == canon_raw_tokens, + "{name}: inline log projection diverges from map_row / \ + map_row_with_raw column order — update the projection AND the \ + row readers together:\n{projection}\ngot: {tokens:?}" + ); + checked += 1; + } + } + assert!( + checked >= 10, + "expected to find at least 10 inline projections; the extraction regex \ + may have rotted (found {checked})" + ); +} + +#[test] +fn lint_flags_unquoted_infix_hyphen_term() { + let err = validate_fts_query("smoke-test").unwrap_err().to_string(); + assert!( + err.contains("NOT operator"), + "should explain hyphen trap: {err}" + ); + assert!( + err.contains("--grep") || err.contains("\"smoke-test\""), + "should suggest a fix: {err}" + ); +} + +#[test] +fn lint_accepts_quoted_phrase() { + // Already-quoted hyphenated phrase is valid FTS5 and must pass. + assert!(validate_fts_query("\"smoke-test\"").is_ok()); +} + +#[test] +fn lint_accepts_normal_boolean_query() { + assert!(validate_fts_query("error AND nginx").is_ok()); +} + +#[test] +fn lint_leaves_leading_hyphen_not_term_alone() { + // `-nginx` is an intentional FTS5 NOT, not the hyphenated-word trap. + assert!(validate_fts_query("error -nginx").is_ok()); +} + +#[test] +fn lint_flags_unbalanced_quote() { + let err = validate_fts_query("\"oops").unwrap_err().to_string(); + assert!(err.contains("unbalanced quote"), "{err}"); +} + +#[test] +fn lint_flags_unquoted_hyphen_term_alongside_a_quoted_phrase() { + // The hyphen check is per-term: a quoted phrase elsewhere must not mask an + // unquoted hyphenated term (regression for a query-wide quote gate). + let err = validate_fts_query("\"disk full\" smoke-test") + .unwrap_err() + .to_string(); + assert!(err.contains("NOT operator"), "{err}"); +} diff --git a/crates/shared/cortex/storage-sqlite/src/skill_events.rs b/crates/shared/cortex/storage-sqlite/src/skill_events.rs new file mode 100644 index 00000000..793080df --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/skill_events.rs @@ -0,0 +1,177 @@ +//! `ai_skill_events` insert + list query layer. Table/columns are defined in +//! migration 38 (`src/db/pool.rs`). Extraction happens in +//! `crate::inputs`; this module only persists and reads back +//! already-extracted events. + +use anyhow::Result; +use rusqlite::{Transaction, params}; +use serde::{Deserialize, Serialize}; + +use crate::inputs::ExtractedSkillEvent; +pub use cortex_domain::SkillEventEntry as AiSkillEventEntry; + +use super::pool::DbPool; + +#[derive(Debug, Clone)] +pub struct SkillEventInsert { + pub log_id: i64, + pub ai_tool: String, + pub ai_project: Option, + pub ai_session_id: Option, + pub hostname: String, + pub timestamp: String, + pub event: ExtractedSkillEvent, +} + +/// Insert `events` inside an existing transaction with `INSERT OR IGNORE` +/// (idempotent on the `UNIQUE(log_id, skill_name, event_kind, evidence_kind)` +/// constraint). Returns the number of rows actually inserted (excludes +/// ignored duplicates) via SQLite `changes()` summed per statement. +pub(crate) fn insert_skill_events_in_tx( + tx: &Transaction<'_>, + events: &[SkillEventInsert], +) -> Result { + if events.is_empty() { + return Ok(0); + } + let mut stmt = tx.prepare_cached( + "INSERT OR IGNORE INTO ai_skill_events ( + log_id, ai_tool, ai_project, ai_session_id, hostname, timestamp, + skill_name, skill_plugin, event_kind, evidence_kind + ) VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, ?9, ?10)", + )?; + let mut inserted = 0usize; + for item in events { + let changed = stmt.execute(params![ + item.log_id, + item.ai_tool, + item.ai_project, + item.ai_session_id, + item.hostname, + item.timestamp, + item.event.skill_name, + item.event.skill_plugin, + item.event.event_kind.as_str(), + item.event.evidence_kind.as_str(), + ])?; + inserted += changed; + } + Ok(inserted) +} + +/// Pool-acquiring wrapper for callers outside an existing transaction (e.g. +/// the backfill service, which owns its own chunked transaction boundary). +pub fn insert_skill_events(pool: &DbPool, events: &[SkillEventInsert]) -> Result { + let mut conn = pool.get()?; + let _write_guard = crate::write_lock(); + let tx = conn.transaction()?; + let inserted = insert_skill_events_in_tx(&tx, events)?; + tx.commit()?; + Ok(inserted) +} + +#[derive(Debug, Clone, Default)] +pub struct AiSkillEventParams { + pub skill: Option, + pub plugin: Option, + pub tool: Option, + pub project: Option, + pub session_id: Option, + pub hostname: Option, + pub from: Option, + pub to: Option, + pub limit: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct ListSkillEventsResult { + pub total: usize, + pub truncated: bool, + pub events: Vec, +} + +const DEFAULT_LIMIT: u32 = 50; +const MAX_LIMIT: u32 = 500; + +/// List `ai_skill_events` rows newest-first, applying every non-`None` +/// filter in `params` as an `AND`-ed equality/range clause. `limit` is +/// clamped to `[1, 500]`; `truncated` is `true` when more rows matched than +/// were returned (probed via `LIMIT + 1`, mirroring `list_ai_tools`'s +/// truncation-detection pattern in `src/db/queries.rs`). +pub fn list_skill_events( + pool: &DbPool, + params: &AiSkillEventParams, +) -> Result { + let conn = pool.get()?; + let limit = params.limit.unwrap_or(DEFAULT_LIMIT).clamp(1, MAX_LIMIT) as usize; + + let mut sql = String::from( + "SELECT id, log_id, ai_tool, ai_project, ai_session_id, hostname, timestamp, + skill_name, skill_plugin, event_kind, evidence_kind + FROM ai_skill_events WHERE 1 = 1", + ); + let mut bindings: Vec = Vec::new(); + let mut idx = 1usize; + + macro_rules! bind_eq { + ($column:literal, $value:expr) => { + if let Some(value) = $value { + sql.push_str(&format!(" AND {} = ?{idx}", $column)); + bindings.push(rusqlite::types::Value::Text(value.clone())); + idx += 1; + } + }; + } + bind_eq!("skill_name", ¶ms.skill); + bind_eq!("skill_plugin", ¶ms.plugin); + bind_eq!("ai_tool", ¶ms.tool); + bind_eq!("ai_project", ¶ms.project); + bind_eq!("ai_session_id", ¶ms.session_id); + bind_eq!("hostname", ¶ms.hostname); + if let Some(from) = ¶ms.from { + sql.push_str(&format!(" AND timestamp >= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(from.clone())); + idx += 1; + } + if let Some(to) = ¶ms.to { + sql.push_str(&format!(" AND timestamp <= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(to.clone())); + idx += 1; + } + let _ = idx; + sql.push_str(&format!( + " ORDER BY timestamp DESC, id DESC LIMIT {}", + limit + 1 + )); + + let mut stmt = conn.prepare(&sql)?; + let mut rows = stmt + .query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + Ok(AiSkillEventEntry { + id: row.get(0)?, + log_id: row.get(1)?, + ai_tool: row.get(2)?, + ai_project: row.get(3)?, + ai_session_id: row.get(4)?, + hostname: row.get(5)?, + timestamp: row.get(6)?, + skill_name: row.get(7)?, + skill_plugin: row.get(8)?, + event_kind: row.get(9)?, + evidence_kind: row.get(10)?, + }) + })? + .collect::>>()?; + + let truncated = rows.len() > limit; + rows.truncate(limit); + Ok(ListSkillEventsResult { + total: rows.len(), + truncated, + events: rows, + }) +} + +#[cfg(test)] +#[path = "skill_events_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/skill_events_tests.rs b/crates/shared/cortex/storage-sqlite/src/skill_events_tests.rs new file mode 100644 index 00000000..02bb5531 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/skill_events_tests.rs @@ -0,0 +1,156 @@ +use super::*; +use crate::config::StorageConfig; +use crate::inputs::{ExtractedSkillEvent, SkillEventKind, SkillEvidenceKind}; +use crate::pool::init_pool; + +fn test_pool() -> (crate::DbPool, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let pool = init_pool(&StorageConfig::for_test(db_path)).unwrap(); + (pool, dir) +} + +fn insert_log_row(pool: &crate::DbPool, hostname: &str, timestamp: &str) -> i64 { + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO logs (timestamp, hostname, severity, message, raw, source_ip) + VALUES (?1, ?2, 'info', 'msg', 'raw', 'transcript://claude_project')", + rusqlite::params![timestamp, hostname], + ) + .unwrap(); + conn.last_insert_rowid() +} + +fn sample_event(skill_name: &str) -> ExtractedSkillEvent { + ExtractedSkillEvent { + skill_name: skill_name.to_string(), + skill_plugin: Some("cortex".to_string()), + event_kind: SkillEventKind::ClaudeAttribution, + evidence_kind: SkillEvidenceKind::StructuredJsonField, + } +} + +#[test] +fn insert_and_list_round_trips() { + let (pool, _dir) = test_pool(); + let log_id = insert_log_row(&pool, "devhost", "2026-06-01T00:00:00.000Z"); + let insert = SkillEventInsert { + log_id, + ai_tool: "claude".to_string(), + ai_project: Some("cortex".to_string()), + ai_session_id: Some("sess-1".to_string()), + hostname: "devhost".to_string(), + timestamp: "2026-06-01T00:00:00.000Z".to_string(), + event: sample_event("cortex-troubleshoot"), + }; + let inserted = insert_skill_events(&pool, &[insert]).unwrap(); + assert_eq!(inserted, 1); + + let result = list_skill_events(&pool, &AiSkillEventParams::default()).unwrap(); + assert_eq!(result.total, 1); + assert_eq!(result.events[0].skill_name, "cortex-troubleshoot"); + assert_eq!(result.events[0].skill_plugin.as_deref(), Some("cortex")); + assert_eq!(result.events[0].event_kind, "claude_attribution"); + assert_eq!(result.events[0].evidence_kind, "structured_json_field"); + assert_eq!(result.events[0].log_id, log_id); +} + +#[test] +fn insert_or_ignore_is_idempotent_on_duplicate() { + let (pool, _dir) = test_pool(); + let log_id = insert_log_row(&pool, "devhost", "2026-06-01T00:00:00.000Z"); + let insert = SkillEventInsert { + log_id, + ai_tool: "claude".to_string(), + ai_project: None, + ai_session_id: None, + hostname: "devhost".to_string(), + timestamp: "2026-06-01T00:00:00.000Z".to_string(), + event: sample_event("cortex-troubleshoot"), + }; + assert_eq!( + insert_skill_events(&pool, std::slice::from_ref(&insert)).unwrap(), + 1 + ); + assert_eq!(insert_skill_events(&pool, &[insert]).unwrap(), 0); + + let result = list_skill_events(&pool, &AiSkillEventParams::default()).unwrap(); + assert_eq!(result.total, 1); +} + +#[test] +fn insert_succeeds_without_project_or_session_id() { + let (pool, _dir) = test_pool(); + let log_id = insert_log_row(&pool, "devhost", "2026-06-01T00:00:00.000Z"); + let insert = SkillEventInsert { + log_id, + ai_tool: "codex".to_string(), + ai_project: None, + ai_session_id: None, + hostname: "devhost".to_string(), + timestamp: "2026-06-01T00:00:00.000Z".to_string(), + event: ExtractedSkillEvent { + skill_name: "rustarr".to_string(), + skill_plugin: None, + event_kind: SkillEventKind::CodexSkillBlock, + evidence_kind: SkillEvidenceKind::TranscriptContent, + }, + }; + assert_eq!(insert_skill_events(&pool, &[insert]).unwrap(), 1); + let result = list_skill_events(&pool, &AiSkillEventParams::default()).unwrap(); + assert_eq!(result.events[0].ai_project, None); + assert_eq!(result.events[0].ai_session_id, None); +} + +#[test] +fn list_filters_by_skill_project_and_tool() { + let (pool, _dir) = test_pool(); + let log_id_a = insert_log_row(&pool, "devhost", "2026-06-01T00:00:00.000Z"); + let log_id_b = insert_log_row(&pool, "nashost", "2026-06-01T01:00:00.000Z"); + insert_skill_events( + &pool, + &[ + SkillEventInsert { + log_id: log_id_a, + ai_tool: "claude".to_string(), + ai_project: Some("cortex".to_string()), + ai_session_id: Some("sess-a".to_string()), + hostname: "devhost".to_string(), + timestamp: "2026-06-01T00:00:00.000Z".to_string(), + event: sample_event("cortex-troubleshoot"), + }, + SkillEventInsert { + log_id: log_id_b, + ai_tool: "codex".to_string(), + ai_project: Some("axon".to_string()), + ai_session_id: Some("sess-b".to_string()), + hostname: "nashost".to_string(), + timestamp: "2026-06-01T01:00:00.000Z".to_string(), + event: sample_event("axon-deploy"), + }, + ], + ) + .unwrap(); + + let result = list_skill_events( + &pool, + &AiSkillEventParams { + project: Some("cortex".to_string()), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(result.total, 1); + assert_eq!(result.events[0].skill_name, "cortex-troubleshoot"); + + let result = list_skill_events( + &pool, + &AiSkillEventParams { + tool: Some("codex".to_string()), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(result.total, 1); + assert_eq!(result.events[0].ai_tool, "codex"); +} diff --git a/crates/shared/cortex/storage-sqlite/src/skill_incident_evidence.rs b/crates/shared/cortex/storage-sqlite/src/skill_incident_evidence.rs new file mode 100644 index 00000000..e6b0aef8 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/skill_incident_evidence.rs @@ -0,0 +1,357 @@ +//! Investigation evidence-bundle layer for skill incidents. Expands a +//! `SkillIncident` (grouped/scored in `src/db/skill_incidents.rs`) into a +//! bounded, truncation-flagged evidence bundle: the underlying skill events, +//! the transcript rows that triggered anchor signals, transcript context +//! before/after, and nearby non-AI logs split into tool-failure/ +//! user-correction/error subsets. Mirrors `investigate_ai_incidents` in +//! `src/db/queries.rs` but keyed on skill usage instead of abuse-term +//! anchors. + +use anyhow::Result; +use serde::{Deserialize, Serialize}; + +use cortex_domain::skill_signal_detectors::{detect_tool_failure, detect_user_correction}; + +use super::models::LogEntry; +use super::pool::DbPool; +use super::queries::map_row; +use super::skill_events::AiSkillEventEntry; +use super::skill_incidents::{AiSkillIncidentParams, SkillIncident, search_ai_skill_incidents}; + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct AiSkillInvestigateParams { + pub incident_id: Option, + pub skill: Option, + pub plugin: Option, + pub ai_tool: Option, + pub ai_project: Option, + pub since: Option, + pub until: Option, + /// Max incidents to investigate. Default 3, clamp 1..=10. + pub limit: Option, + /// Incident grouping window minutes. Default 10, clamp 1..=120. + pub window_minutes: Option, + /// Correlation window minutes around incident. Default 5, clamp 1..=120. + pub correlation_window_minutes: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct SkillIncidentEvidence { + pub incident: SkillIncident, + /// The `ai_skill_events` rows in this group, capped at 25. + pub skill_events: Vec, + pub skill_events_truncated: bool, + /// Transcript rows that triggered an anchor signal, capped at 50. + pub signal_anchors: Vec, + pub signal_anchors_truncated: bool, + /// Same-session transcript entries before the first skill event, capped 20. + pub transcript_before: Vec, + pub transcript_before_truncated: bool, + /// Same-session transcript entries after the last skill event, capped 20. + pub transcript_after: Vec, + pub transcript_after_truncated: bool, + /// Subset of nearby_logs matching tool-failure phrases, capped 25. + pub nearby_tool_failures: Vec, + pub nearby_tool_failures_truncated: bool, + /// Subset of nearby_logs matching user-correction phrases, capped 25. + pub nearby_user_corrections: Vec, + pub nearby_user_corrections_truncated: bool, + /// Non-AI syslog/Docker logs in the correlation window, capped 50. + pub nearby_logs: Vec, + pub nearby_logs_truncated: bool, + /// Subset of nearby_logs with severity warning or above, capped 25. + pub nearby_errors: Vec, + pub nearby_errors_truncated: bool, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiSkillInvestigateResult { + pub evidence: Vec, + pub total_incidents: usize, + pub truncated: bool, +} + +/// Row mapper for `ai_skill_events` rows, matching the 11-column SELECT used +/// throughout this module. Kept as a standalone fn (rather than duplicating +/// the inline closure in `skill_events::list_skill_events`) so the ordering +/// only needs to be maintained in one place. +fn map_skill_event_row(row: &rusqlite::Row) -> rusqlite::Result { + Ok(AiSkillEventEntry { + id: row.get(0)?, + log_id: row.get(1)?, + ai_tool: row.get(2)?, + ai_project: row.get(3)?, + ai_session_id: row.get(4)?, + hostname: row.get(5)?, + timestamp: row.get(6)?, + skill_name: row.get(7)?, + skill_plugin: row.get(8)?, + event_kind: row.get(9)?, + evidence_kind: row.get(10)?, + }) +} + +pub fn investigate_ai_skill_incidents( + pool: &DbPool, + params: &AiSkillInvestigateParams, +) -> Result { + const SKILL_EVENTS_CAP: usize = 25; + const SIGNAL_ANCHORS_CAP: usize = 50; + const TRANSCRIPT_CAP: usize = 20; + const NEARBY_CAP: usize = 50; + const NEARBY_SUBSET_CAP: usize = 25; + + let limit = params.limit.unwrap_or(3).clamp(1, 10) as usize; + let corr_mins = i64::from(params.correlation_window_minutes.unwrap_or(5).clamp(1, 120)); + + // `incident_id` is passed straight through to `AiSkillIncidentParams`, + // which filters the full computed incident set (bounded only by + // `SKILL_INCIDENT_CANDIDATE_CAP` events, not an incident-count cap) + // before its own priority-ranked truncation. This guarantees an exact + // incident_id lookup finds its target regardless of priority rank — + // routing it through a fixed-size top-N candidate window (as a prior + // version of this code did) could silently miss incidents ranked + // below that window. + let incident_result = search_ai_skill_incidents( + pool, + &AiSkillIncidentParams { + skill: params.skill.clone(), + plugin: params.plugin.clone(), + ai_tool: params.ai_tool.clone(), + ai_project: params.ai_project.clone(), + ai_session_id: None, + hostname: None, + since: params.since.clone(), + until: params.until.clone(), + incident_id: params.incident_id.clone(), + limit: Some(limit as u32), + window_minutes: params.window_minutes, + signals: Vec::new(), + min_score: None, + }, + )?; + let total_incidents = incident_result.total_incidents; + let truncated = incident_result.truncated; + let mut incidents = incident_result.incidents; + incidents.truncate(limit); + + let conn = pool.get()?; + let mut evidence = Vec::with_capacity(incidents.len()); + + for incident in incidents { + // ── Skill events for this group ───────────────────────────────────── + let (skill_events, skill_events_truncated) = if incident.skill_event_ids.is_empty() { + (Vec::new(), false) + } else { + let placeholders: Vec = (1..=incident.skill_event_ids.len()) + .map(|i| format!("?{i}")) + .collect(); + let sql = format!( + "SELECT id, log_id, ai_tool, ai_project, ai_session_id, hostname, timestamp, + skill_name, skill_plugin, event_kind, evidence_kind + FROM ai_skill_events WHERE id IN ({}) ORDER BY timestamp ASC", + placeholders.join(",") + ); + let mut stmt = conn.prepare(&sql)?; + let rows: Vec = stmt + .query_map( + rusqlite::params_from_iter( + incident + .skill_event_ids + .iter() + .map(|id| rusqlite::types::Value::Integer(*id)), + ), + map_skill_event_row, + )? + .collect::>>()?; + let truncated = rows.len() > SKILL_EVENTS_CAP; + let mut out = rows; + out.truncate(SKILL_EVENTS_CAP); + (out, truncated) + }; + + // ── Signal anchor log rows ────────────────────────────────────────── + let (signal_anchors, signal_anchors_truncated) = if incident.anchor_log_ids.is_empty() { + (Vec::new(), false) + } else { + let placeholders: Vec = (1..=incident.anchor_log_ids.len()) + .map(|i| format!("?{i}")) + .collect(); + let sql = format!( + "SELECT id, timestamp, hostname, facility, severity, app_name, + process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs WHERE id IN ({}) ORDER BY timestamp ASC", + placeholders.join(",") + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt + .query_map( + rusqlite::params_from_iter( + incident + .anchor_log_ids + .iter() + .map(|id| rusqlite::types::Value::Integer(*id)), + ), + map_row, + )? + .collect::>>()?; + let truncated = rows.len() > SIGNAL_ANCHORS_CAP; + let mut out = rows; + out.truncate(SIGNAL_ANCHORS_CAP); + (out, truncated) + }; + + // ── Transcript before/after (same pattern as investigate_ai_incidents) ── + let (transcript_before, transcript_before_truncated) = { + let mut stmt = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, app_name, + process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs + WHERE ai_session_id = ?1 AND ai_project = ?2 AND ai_tool = ?3 + AND timestamp < ?4 + ORDER BY timestamp DESC + LIMIT 21", + )?; + let rows = stmt + .query_map( + rusqlite::params![ + &incident.session_id, + &incident.project, + &incident.tool, + &incident.first_seen, + ], + map_row, + )? + .collect::>>()?; + let truncated = rows.len() > TRANSCRIPT_CAP; + let mut out = rows; + out.truncate(TRANSCRIPT_CAP); + out.reverse(); + (out, truncated) + }; + + let (transcript_after, transcript_after_truncated) = { + let mut stmt = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, app_name, + process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs + WHERE ai_session_id = ?1 AND ai_project = ?2 AND ai_tool = ?3 + AND timestamp > ?4 + ORDER BY timestamp ASC + LIMIT 21", + )?; + let rows = stmt + .query_map( + rusqlite::params![ + &incident.session_id, + &incident.project, + &incident.tool, + &incident.last_seen, + ], + map_row, + )? + .collect::>>()?; + let truncated = rows.len() > TRANSCRIPT_CAP; + let mut out = rows; + out.truncate(TRANSCRIPT_CAP); + (out, truncated) + }; + + // ── Nearby non-AI logs in the correlation window ──────────────────── + let (nearby_logs, nearby_logs_truncated) = { + let win_from = chrono::DateTime::parse_from_rfc3339(&incident.first_seen) + .map(|dt| { + (dt.with_timezone(&chrono::Utc) - chrono::Duration::minutes(corr_mins)) + .format("%Y-%m-%dT%H:%M:%S%.3fZ") + .to_string() + }) + .unwrap_or_else(|_| incident.first_seen.clone()); + let win_to = chrono::DateTime::parse_from_rfc3339(&incident.last_seen) + .map(|dt| { + (dt.with_timezone(&chrono::Utc) + chrono::Duration::minutes(corr_mins)) + .format("%Y-%m-%dT%H:%M:%S%.3fZ") + .to_string() + }) + .unwrap_or_else(|_| incident.last_seen.clone()); + + let mut stmt = conn.prepare( + "SELECT id, timestamp, hostname, facility, severity, app_name, + process_id, message, received_at, source_ip, + ai_tool, ai_project, ai_session_id, ai_transcript_path, metadata_json + FROM logs + WHERE timestamp >= ?1 AND timestamp <= ?2 AND hostname = ?3 + ORDER BY timestamp ASC + LIMIT 51", + )?; + let rows = stmt + .query_map( + rusqlite::params![win_from, win_to, &incident.hostname], + map_row, + )? + .collect::>>()?; + let truncated = rows.len() > NEARBY_CAP; + let mut out = rows; + out.truncate(NEARBY_CAP); + (out, truncated) + }; + + // ── Derived subsets: tool failures, user corrections, errors ──────── + let mut nearby_tool_failures: Vec = nearby_logs + .iter() + .filter(|e| detect_tool_failure(&e.message)) + .cloned() + .collect(); + let nearby_tool_failures_truncated = nearby_tool_failures.len() > NEARBY_SUBSET_CAP; + nearby_tool_failures.truncate(NEARBY_SUBSET_CAP); + + let mut nearby_user_corrections: Vec = nearby_logs + .iter() + .filter(|e| detect_user_correction(&e.message)) + .cloned() + .collect(); + let nearby_user_corrections_truncated = nearby_user_corrections.len() > NEARBY_SUBSET_CAP; + nearby_user_corrections.truncate(NEARBY_SUBSET_CAP); + + let error_sevs = ["emergency", "alert", "critical", "error", "warning"]; + let mut nearby_errors: Vec = nearby_logs + .iter() + .filter(|e| error_sevs.contains(&e.severity.as_str())) + .cloned() + .collect(); + let nearby_errors_truncated = nearby_errors.len() > NEARBY_SUBSET_CAP; + nearby_errors.truncate(NEARBY_SUBSET_CAP); + + evidence.push(SkillIncidentEvidence { + incident, + skill_events, + skill_events_truncated, + signal_anchors, + signal_anchors_truncated, + transcript_before, + transcript_before_truncated, + transcript_after, + transcript_after_truncated, + nearby_tool_failures, + nearby_tool_failures_truncated, + nearby_user_corrections, + nearby_user_corrections_truncated, + nearby_logs, + nearby_logs_truncated, + nearby_errors, + nearby_errors_truncated, + }); + } + + Ok(AiSkillInvestigateResult { + evidence, + total_incidents, + truncated, + }) +} + +#[cfg(test)] +#[path = "skill_incident_evidence_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/skill_incident_evidence_tests.rs b/crates/shared/cortex/storage-sqlite/src/skill_incident_evidence_tests.rs new file mode 100644 index 00000000..12483f8d --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/skill_incident_evidence_tests.rs @@ -0,0 +1,482 @@ +use super::*; +use crate::config::StorageConfig; +use crate::pool::init_pool; +use crate::skill_incidents::{AiSkillIncidentParams, search_ai_skill_incidents}; +use crate::{DbPool, LogBatchEntry, insert_logs_batch}; + +fn test_pool() -> (DbPool, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let pool = init_pool(&StorageConfig::for_test(db_path)).unwrap(); + (pool, dir) +} + +fn make_ai_entry( + ts: &str, + host: &str, + tool: &str, + project: &str, + session_id: &str, + message: &str, +) -> LogBatchEntry { + LogBatchEntry { + timestamp: ts.to_string(), + hostname: host.to_string(), + facility: Some("local0".to_string()), + severity: "info".to_string(), + app_name: Some("ai-transcript".to_string()), + process_id: None, + message: message.to_string(), + raw: message.to_string(), + source_ip: "127.0.0.1:514".to_string(), + docker_checkpoint: None, + ai_tool: Some(tool.to_string()), + ai_project: Some(project.to_string()), + ai_session_id: Some(session_id.to_string()), + ai_transcript_path: Some(format!("{project}/{session_id}.jsonl")), + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + } +} + +#[allow(clippy::too_many_arguments)] +fn insert_skill_event( + pool: &DbPool, + log_id: i64, + ai_tool: &str, + ai_project: &str, + ai_session_id: &str, + hostname: &str, + timestamp: &str, + skill_name: &str, + skill_plugin: Option<&str>, +) { + // Note (PR 2 eng review sync): `skill_path`/`metadata_json` were dropped + // from `ai_skill_events` before PR 2 shipped — neither extractor ever set + // them. This INSERT reflects the shipped column set. + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO ai_skill_events + (log_id, ai_tool, ai_project, ai_session_id, hostname, timestamp, + skill_name, skill_plugin, event_kind, evidence_kind, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 'skill_invoked', 'transcript', ?6)", + rusqlite::params![ + log_id, + ai_tool, + ai_project, + ai_session_id, + hostname, + timestamp, + skill_name, + skill_plugin, + ], + ) + .unwrap(); +} + +#[test] +fn investigate_ai_skill_incidents_bundle_has_bounded_collections_and_truncation_flags() { + let (pool, _dir) = test_pool(); + + let skill_log = make_ai_entry( + "2026-01-01T00:00:00Z", + "devhost", + "codex", + "/tmp/project-d", + "sess-d", + "loaded skill lavra:lavra-plan", + ); + let before_log = make_ai_entry( + "2026-01-01T00:00:00.000Z", + "devhost", + "codex", + "/tmp/project-d", + "sess-d", + "user asked to plan the feature", + ); + let correction_log = make_ai_entry( + "2026-01-01T00:02:00Z", + "devhost", + "codex", + "/tmp/project-d", + "sess-d", + "that's not what I asked, wrong file", + ); + let failure_log = make_ai_entry( + "2026-01-01T00:03:00Z", + "devhost", + "codex", + "/tmp/project-d", + "sess-d", + "command exited with exit code 1", + ); + insert_logs_batch(&pool, &[before_log, skill_log, correction_log, failure_log]).unwrap(); + + let log_ids: Vec = { + let conn = pool.get().unwrap(); + let mut stmt = conn + .prepare("SELECT id FROM logs ORDER BY timestamp ASC, id ASC") + .unwrap(); + stmt.query_map([], |row| row.get::<_, i64>(0)) + .unwrap() + .collect::>>() + .unwrap() + }; + // log_ids: [before, skill, correction, failure] in timestamp order. + insert_skill_event( + &pool, + log_ids[1], + "codex", + "/tmp/project-d", + "sess-d", + "devhost", + "2026-01-01T00:00:00Z", + "lavra:lavra-plan", + Some("lavra"), + ); + + let result = investigate_ai_skill_incidents( + &pool, + &AiSkillInvestigateParams { + skill: Some("lavra:lavra-plan".into()), + limit: Some(3), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.evidence.len(), 1); + let bundle = &result.evidence[0]; + assert_eq!(bundle.incident.skill_name, "lavra:lavra-plan"); + assert!(!bundle.skill_events.is_empty()); + assert!(!bundle.skill_events_truncated); + assert!(!bundle.signal_anchors.is_empty()); + assert!(!bundle.signal_anchors_truncated); + // transcript_before should include the pre-skill "user asked to plan" row. + assert!( + bundle + .transcript_before + .iter() + .any(|e| e.message.contains("user asked to plan")) + ); + assert!(!bundle.transcript_before_truncated); + assert!(!bundle.transcript_after_truncated); + // The correction log should land in nearby_user_corrections; the failure + // log should land in nearby_tool_failures. + assert!( + bundle + .nearby_user_corrections + .iter() + .any(|e| e.message.contains("wrong file")) + ); + assert!( + bundle + .nearby_tool_failures + .iter() + .any(|e| e.message.contains("exit code 1")) + ); + assert!(!bundle.nearby_logs_truncated); + assert!(!bundle.nearby_errors_truncated); +} + +#[test] +fn investigate_ai_skill_incidents_exact_incident_id_can_target_outside_top_page() { + let (pool, _dir) = test_pool(); + let mut entries = Vec::new(); + for i in 0..12 { + entries.push(make_ai_entry( + &format!("2026-01-01T00:{i:02}:00Z"), + "host-a", + "codex", + "/tmp/project-e", + &format!("sess-e-{i:02}"), + "loaded skill lavra:lavra-plan", + )); + } + insert_logs_batch(&pool, &entries).unwrap(); + let log_ids: Vec = { + let conn = pool.get().unwrap(); + let mut stmt = conn.prepare("SELECT id FROM logs ORDER BY id ASC").unwrap(); + stmt.query_map([], |row| row.get::<_, i64>(0)) + .unwrap() + .collect::>>() + .unwrap() + }; + for (i, log_id) in log_ids.iter().enumerate() { + insert_skill_event( + &pool, + *log_id, + "codex", + "/tmp/project-e", + &format!("sess-e-{i:02}"), + "host-a", + &format!("2026-01-01T00:{i:02}:00Z"), + "lavra:lavra-plan", + Some("lavra"), + ); + } + + let listed = search_ai_skill_incidents( + &pool, + &AiSkillIncidentParams { + skill: Some("lavra:lavra-plan".into()), + limit: Some(12), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(listed.incidents.len(), 12); + let target_id = listed.incidents.last().unwrap().incident_id.clone(); + + let top_page = investigate_ai_skill_incidents( + &pool, + &AiSkillInvestigateParams { + skill: Some("lavra:lavra-plan".into()), + limit: Some(3), + ..Default::default() + }, + ) + .unwrap(); + assert!( + !top_page + .evidence + .iter() + .any(|b| b.incident.incident_id == target_id) + ); + + let exact = investigate_ai_skill_incidents( + &pool, + &AiSkillInvestigateParams { + incident_id: Some(target_id.clone()), + skill: Some("lavra:lavra-plan".into()), + limit: Some(1), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(exact.evidence.len(), 1); + assert_eq!(exact.evidence[0].incident.incident_id, target_id); +} + +/// Regression test for a bug where an exact `incident_id` lookup routed +/// through `search_ai_skill_incidents` with `limit: Some(100)` and then +/// filtered client-side for the matching id — if the target incident +/// ranked below the top 100 by priority score, investigation silently +/// returned empty evidence for an incident that actually existed. This +/// constructs 100 higher-scored decoy incidents plus one lower-scored +/// target so the target provably ranks outside any top-100 window, then +/// asserts the exact lookup still finds it. +#[test] +fn investigate_ai_skill_incidents_exact_incident_id_beyond_top_100_candidates() { + let (pool, _dir) = test_pool(); + + fn log_ids_for_session(pool: &DbPool, session_id: &str) -> Vec { + let conn = pool.get().unwrap(); + let mut stmt = conn + .prepare("SELECT id FROM logs WHERE ai_session_id = ?1 ORDER BY timestamp ASC, id ASC") + .unwrap(); + stmt.query_map([session_id], |row| row.get::<_, i64>(0)) + .unwrap() + .collect::>>() + .unwrap() + } + + // 100 decoy groups, each scored higher than baseline via a + // user_correction_after_skill anchor, so every decoy outranks the target. + for i in 0..100 { + let session_id = format!("sess-decoy-{i:03}"); + let skill_log = make_ai_entry( + "2026-01-01T00:00:00Z", + "host-a", + "codex", + "/tmp/project-g", + &session_id, + "loaded skill lavra:lavra-plan", + ); + let correction_log = make_ai_entry( + "2026-01-01T00:00:30Z", + "host-a", + "codex", + "/tmp/project-g", + &session_id, + "that's not what I asked, wrong file", + ); + insert_logs_batch(&pool, &[skill_log, correction_log]).unwrap(); + let ids = log_ids_for_session(&pool, &session_id); + insert_skill_event( + &pool, + ids[0], + "codex", + "/tmp/project-g", + &session_id, + "host-a", + "2026-01-01T00:00:00Z", + "lavra:lavra-plan", + Some("lavra"), + ); + } + + // Target group: baseline score only (no anchor signal), guaranteeing it + // ranks last among the 101 total matching incidents. + let target_session_id = "sess-target".to_string(); + let target_skill_log = make_ai_entry( + "2026-01-01T00:00:00Z", + "host-a", + "codex", + "/tmp/project-g", + &target_session_id, + "loaded skill lavra:lavra-plan", + ); + insert_logs_batch(&pool, &[target_skill_log]).unwrap(); + let target_log_ids = log_ids_for_session(&pool, &target_session_id); + insert_skill_event( + &pool, + target_log_ids[0], + "codex", + "/tmp/project-g", + &target_session_id, + "host-a", + "2026-01-01T00:00:00Z", + "lavra:lavra-plan", + Some("lavra"), + ); + + let target_lookup = search_ai_skill_incidents( + &pool, + &AiSkillIncidentParams { + skill: Some("lavra:lavra-plan".into()), + ai_session_id: Some(target_session_id.clone()), + limit: Some(1), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(target_lookup.incidents.len(), 1); + let target_id = target_lookup.incidents[0].incident_id.clone(); + + let top100 = search_ai_skill_incidents( + &pool, + &AiSkillIncidentParams { + skill: Some("lavra:lavra-plan".into()), + limit: Some(100), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!(top100.total_incidents, 101, "100 decoys + 1 target"); + assert_eq!(top100.incidents.len(), 100); + assert!( + !top100 + .incidents + .iter() + .any(|inc| inc.incident_id == target_id), + "test setup invariant: target must rank outside the top 100" + ); + + // The regression check: an exact incident_id lookup must still find the + // target even though it ranks outside the top-100 candidate window. + let exact = investigate_ai_skill_incidents( + &pool, + &AiSkillInvestigateParams { + incident_id: Some(target_id.clone()), + skill: Some("lavra:lavra-plan".into()), + limit: Some(1), + ..Default::default() + }, + ) + .unwrap(); + assert_eq!( + exact.evidence.len(), + 1, + "exact incident_id lookup must find an incident ranked outside the top 100" + ); + assert_eq!(exact.evidence[0].incident.incident_id, target_id); +} + +/// Regression test for a bug where the `nearby_logs` query only filtered by +/// timestamp range, with no hostname scope, so an incident on one host could +/// pull in unrelated log rows from a different host in the same time window. +#[test] +fn investigate_ai_skill_incidents_nearby_logs_scoped_to_incident_hostname() { + let (pool, _dir) = test_pool(); + + let skill_log = make_ai_entry( + "2026-01-01T00:00:00Z", + "host-a", + "codex", + "/tmp/project-h", + "sess-h", + "loaded skill lavra:lavra-plan", + ); + insert_logs_batch(&pool, &[skill_log]).unwrap(); + let log_id: i64 = { + let conn = pool.get().unwrap(); + conn.query_row("SELECT id FROM logs LIMIT 1", [], |row| row.get(0)) + .unwrap() + }; + insert_skill_event( + &pool, + log_id, + "codex", + "/tmp/project-h", + "sess-h", + "host-a", + "2026-01-01T00:00:00Z", + "lavra:lavra-plan", + Some("lavra"), + ); + + // Unrelated non-AI log on a DIFFERENT host, within the correlation window. + let other_host_log = LogBatchEntry { + timestamp: "2026-01-01T00:01:00Z".to_string(), + hostname: "host-b".to_string(), + facility: Some("local0".to_string()), + severity: "error".to_string(), + app_name: Some("nginx".to_string()), + process_id: None, + message: "connection refused".to_string(), + raw: "connection refused".to_string(), + source_ip: "10.0.0.5:514".to_string(), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + }; + insert_logs_batch(&pool, &[other_host_log]).unwrap(); + + let result = investigate_ai_skill_incidents( + &pool, + &AiSkillInvestigateParams { + skill: Some("lavra:lavra-plan".into()), + limit: Some(1), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.evidence.len(), 1); + let bundle = &result.evidence[0]; + assert!( + bundle.nearby_logs.iter().all(|e| e.hostname == "host-a"), + "nearby_logs leaked a cross-host row: {:?}", + bundle.nearby_logs + ); + assert!( + !bundle + .nearby_logs + .iter() + .any(|e| e.message.contains("connection refused")), + "cross-host log should not appear in nearby_logs" + ); +} diff --git a/crates/shared/cortex/storage-sqlite/src/skill_incidents.rs b/crates/shared/cortex/storage-sqlite/src/skill_incidents.rs new file mode 100644 index 00000000..7cad528e --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/skill_incidents.rs @@ -0,0 +1,393 @@ +//! Skill-usage incident grouping and scoring. Groups `ai_skill_events` rows +//! into `SkillIncident`s by `(skill_name, skill_plugin, tool, project, +//! session_id, hostname, window_bucket)`, scans nearby transcript logs for +//! the five deterministic anchor signals in +//! `cortex_domain::skill_signal_detectors`, and scores/sorts the resulting +//! groups. Mirrors the abuse-incident grouping query (`search_ai_incidents` +//! in `src/db/queries.rs`) but keyed on skill usage instead of abuse-term +//! anchors. The investigation evidence-bundle layer that expands a +//! `SkillIncident` lives in the sibling module +//! `src/db/skill_incident_evidence.rs`. + +use anyhow::Result; +use serde::{Deserialize, Serialize}; +use std::collections::HashMap; + +use cortex_domain::skill_signal_detectors::{ + detect_ignored_instruction, detect_overlong_loop, detect_scope_or_source_confusion, + detect_tool_failure, detect_user_correction, +}; +pub use cortex_domain::{SkillIncident, SkillSignalCounts}; + +use super::pool::DbPool; + +// --------------------------------------------------------------------------- +// Skill incident grouping +// --------------------------------------------------------------------------- + +#[derive(Debug, Clone, Default, Serialize, Deserialize)] +pub struct AiSkillIncidentParams { + pub skill: Option, + pub plugin: Option, + pub ai_tool: Option, + pub ai_project: Option, + pub ai_session_id: Option, + pub hostname: Option, + pub since: Option, + pub until: Option, + /// Exact incident_id match. When set, filters the full computed incident + /// set (bounded only by `SKILL_INCIDENT_CANDIDATE_CAP`, not `limit`) + /// before the priority-ranked truncation, so a match ranked below + /// `limit` is still found. + pub incident_id: Option, + /// Max incidents to return. Default 20, clamp 1..=100. + pub limit: Option, + /// Grouping window in minutes. Default 10, clamp 1..=120. + pub window_minutes: Option, + /// Restrict to incidents containing at least one of these signal + /// categories. Empty = no filter (all incidents). + pub signals: Vec, + /// Minimum `priority_score` (inclusive). `None` = no filter. + pub min_score: Option, +} + +#[derive(Debug, Clone, Serialize, Deserialize)] +pub struct AiSkillIncidentResult { + pub incidents: Vec, + pub total_incidents: usize, + pub candidate_event_rows: usize, + pub candidate_cap: usize, + pub candidate_window_truncated: bool, + pub truncated: bool, +} + +const SKILL_INCIDENT_CANDIDATE_CAP: usize = 10_000; + +/// Grouping key for skill incidents: `(skill_name, skill_plugin, tool, +/// project, session_id, hostname, window_bucket)`. +/// `window_bucket = unix_secs / window_secs * window_secs` (floor to window +/// boundary), mirroring `search_ai_incidents`'s abuse-incident grouping. +pub fn search_ai_skill_incidents( + pool: &DbPool, + params: &AiSkillIncidentParams, +) -> Result { + let conn = pool.get()?; + let limit = params.limit.unwrap_or(20).clamp(1, 100) as usize; + let window_secs = i64::from(params.window_minutes.unwrap_or(10).clamp(1, 120)) * 60; + + // ── Fetch candidate skill events (bounded, same capped-window pattern as + // search_ai_incidents' FTS candidate fetch) ───────────────────────────── + struct SkillEventRow { + id: i64, + timestamp: String, + hostname: String, + tool: String, + project: String, + session_id: String, + skill_name: String, + skill_plugin: Option, + } + + let mut sql = String::from( + "SELECT id, timestamp, hostname, ai_tool, ai_project, ai_session_id, + skill_name, skill_plugin + FROM ai_skill_events + WHERE 1 = 1", + ); + let mut bindings: Vec = Vec::new(); + let mut idx = 1usize; + if let Some(skill) = ¶ms.skill { + sql.push_str(&format!(" AND skill_name = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(skill.clone())); + idx += 1; + } + if let Some(plugin) = ¶ms.plugin { + sql.push_str(&format!(" AND skill_plugin = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(plugin.clone())); + idx += 1; + } + if let Some(tool) = ¶ms.ai_tool { + sql.push_str(&format!(" AND ai_tool = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(tool.clone())); + idx += 1; + } + if let Some(project) = ¶ms.ai_project { + sql.push_str(&format!(" AND ai_project = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(project.clone())); + idx += 1; + } + if let Some(session_id) = ¶ms.ai_session_id { + sql.push_str(&format!(" AND ai_session_id = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(session_id.clone())); + idx += 1; + } + if let Some(hostname) = ¶ms.hostname { + sql.push_str(&format!(" AND hostname = ?{idx}")); + bindings.push(rusqlite::types::Value::Text(hostname.clone())); + idx += 1; + } + if let Some(from) = ¶ms.since { + sql.push_str(&format!(" AND timestamp >= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(from.clone())); + idx += 1; + } + if let Some(to) = ¶ms.until { + sql.push_str(&format!(" AND timestamp <= ?{idx}")); + bindings.push(rusqlite::types::Value::Text(to.clone())); + } + let _ = idx; + sql.push_str(&format!( + " ORDER BY timestamp ASC LIMIT {}", + SKILL_INCIDENT_CANDIDATE_CAP + 1 + )); + + let mut stmt = conn.prepare(&sql)?; + let candidate_events: Vec = stmt + .query_map(rusqlite::params_from_iter(bindings.iter()), |row| { + Ok(SkillEventRow { + id: row.get(0)?, + timestamp: row.get(1)?, + hostname: row.get(2)?, + tool: row.get(3)?, + project: row.get(4)?, + session_id: row.get(5)?, + skill_name: row.get(6)?, + skill_plugin: row.get(7)?, + }) + })? + .collect::>>()?; + + let candidate_window_truncated = candidate_events.len() > SKILL_INCIDENT_CANDIDATE_CAP; + let raw_candidate_count = candidate_events.len(); + + // ── Group by (skill_name, skill_plugin, tool, project, session_id, + // hostname, window_bucket) ─────────────────────────────────────────────── + type GroupKey = (String, Option, String, String, String, String, i64); + let mut groups: HashMap> = HashMap::new(); + + for row in candidate_events.iter().take(SKILL_INCIDENT_CANDIDATE_CAP) { + let bucket = chrono::DateTime::parse_from_rfc3339(&row.timestamp) + .map(|dt| (dt.timestamp() / window_secs) * window_secs) + .unwrap_or(0); + let key = ( + row.skill_name.clone(), + row.skill_plugin.clone(), + row.tool.clone(), + row.project.clone(), + row.session_id.clone(), + row.hostname.clone(), + bucket, + ); + groups.entry(key).or_default().push(row); + } + + // ── For each group, fetch nearby transcript logs in the session/window to + // detect anchor signals, then score ─────────────────────────────────────── + let mut incidents: Vec = Vec::with_capacity(groups.len()); + for ((skill_name, skill_plugin, tool, project, session_id, hostname, _bucket), events) in groups + { + let first_seen = events + .first() + .map(|e| e.timestamp.clone()) + .unwrap_or_default(); + let last_seen = events + .last() + .map(|e| e.timestamp.clone()) + .unwrap_or_default(); + let duration_secs = { + let t0 = chrono::DateTime::parse_from_rfc3339(&first_seen) + .map(|dt| dt.timestamp()) + .unwrap_or(0); + let t1 = chrono::DateTime::parse_from_rfc3339(&last_seen) + .map(|dt| dt.timestamp()) + .unwrap_or(0); + (t1 - t0).max(0) + }; + + // Window bounds for anchor detection: from first skill event to + // window_secs after the last one (anchors that follow the skill). + let win_from = first_seen.clone(); + let win_to = chrono::DateTime::parse_from_rfc3339(&last_seen) + .map(|dt| { + (dt.with_timezone(&chrono::Utc) + chrono::Duration::seconds(window_secs)) + .format("%Y-%m-%dT%H:%M:%S%.3fZ") + .to_string() + }) + .unwrap_or_else(|_| last_seen.clone()); + + let mut anchor_stmt = conn.prepare( + "SELECT id, message FROM logs + WHERE ai_session_id = ?1 AND ai_project = ?2 AND ai_tool = ?3 + AND timestamp >= ?4 AND timestamp <= ?5 + ORDER BY timestamp ASC + LIMIT 500", + )?; + let anchor_rows: Vec<(i64, String)> = anchor_stmt + .query_map( + rusqlite::params![session_id, project, tool, win_from, win_to], + |row| Ok((row.get::<_, i64>(0)?, row.get::<_, String>(1)?)), + )? + .collect::>>()?; + + let mut counts = SkillSignalCounts::default(); + let mut anchor_log_ids: Vec = Vec::new(); + let tool_call_rows = anchor_rows.len(); + let mut has_correction_or_frustration = false; + + for (id, message) in &anchor_rows { + let mut hit = false; + if detect_user_correction(message) { + counts.user_correction_after_skill += 1; + has_correction_or_frustration = true; + hit = true; + } + if detect_tool_failure(message) { + counts.tool_failure_after_skill += 1; + hit = true; + } + if detect_scope_or_source_confusion(message) { + counts.scope_or_source_confusion += 1; + hit = true; + } + if detect_ignored_instruction(message) { + counts.ignored_skill_or_policy_instruction += 1; + hit = true; + } + if hit { + anchor_log_ids.push(*id); + } + } + if detect_overlong_loop(events.len(), tool_call_rows, has_correction_or_frustration) { + counts.overlong_loop_after_skill += 1; + } + + anchor_log_ids.sort_unstable(); + anchor_log_ids.dedup(); + + let mut signals_present: Vec = Vec::new(); + if counts.user_correction_after_skill > 0 { + signals_present.push("user_correction_after_skill".to_string()); + } + if counts.tool_failure_after_skill > 0 { + signals_present.push("tool_failure_after_skill".to_string()); + } + if counts.scope_or_source_confusion > 0 { + signals_present.push("scope_or_source_confusion".to_string()); + } + if counts.ignored_skill_or_policy_instruction > 0 { + signals_present.push("ignored_skill_or_policy_instruction".to_string()); + } + if counts.overlong_loop_after_skill > 0 { + signals_present.push("overlong_loop_after_skill".to_string()); + } + signals_present.sort(); + + // ── Locked scoring formula ────────────────────────────────────────── + let signal_variety = signals_present.len() as f64; + let priority_score = events.len() as f64 * 2.0 + + counts.user_correction_after_skill as f64 * 15.0 + + counts.tool_failure_after_skill as f64 * 8.0 + + counts.scope_or_source_confusion as f64 * 12.0 + + counts.ignored_skill_or_policy_instruction as f64 * 12.0 + + counts.overlong_loop_after_skill as f64 * 10.0 + + signal_variety * 5.0; + + let priority_label = if priority_score < 15.0 { + "low" + } else if priority_score < 35.0 { + "medium" + } else if priority_score < 60.0 { + "high" + } else { + "critical" + } + .to_string(); + + let mut skill_event_ids: Vec = events.iter().map(|e| e.id).collect(); + skill_event_ids.sort_unstable(); + + // ── Stable incident ID: same DefaultHasher pattern as + // search_ai_incidents (src/db/queries.rs), extended with skill + // name/plugin and sorted skill event ids. + let incident_id = { + use std::collections::hash_map::DefaultHasher; + use std::hash::{Hash, Hasher}; + let mut h = DefaultHasher::new(); + skill_name.hash(&mut h); + skill_plugin.hash(&mut h); + tool.hash(&mut h); + project.hash(&mut h); + session_id.hash(&mut h); + hostname.hash(&mut h); + for id in &anchor_log_ids { + id.hash(&mut h); + } + for id in &skill_event_ids { + id.hash(&mut h); + } + format!("skill-inc-{:016x}", h.finish()) + }; + + incidents.push(SkillIncident { + incident_id, + skill_name, + skill_plugin, + tool, + project, + session_id, + hostname, + first_seen, + last_seen, + duration_secs, + skill_event_count: events.len(), + skill_event_ids, + anchor_log_ids, + signal_counts: counts, + signals_present, + priority_score, + priority_label, + window_minutes: (window_secs / 60) as u32, + }); + } + + // ── Post-grouping filters: incident_id, signals, min_score ────────────── + if let Some(incident_id) = ¶ms.incident_id { + incidents.retain(|inc| &inc.incident_id == incident_id); + } + if !params.signals.is_empty() { + incidents.retain(|inc| { + inc.signals_present + .iter() + .any(|s| params.signals.contains(s)) + }); + } + if let Some(min_score) = params.min_score { + incidents.retain(|inc| inc.priority_score >= min_score); + } + + // Sort by priority_score descending, then last_seen descending. Uses + // total_cmp (not partial_cmp/unwrap_or(Equal)) — a total order even if a + // NaN score ever appears. + incidents.sort_by(|a, b| { + b.priority_score + .total_cmp(&a.priority_score) + .then_with(|| b.last_seen.cmp(&a.last_seen)) + }); + + let total_incidents = incidents.len(); + let truncated = total_incidents > limit || candidate_window_truncated; + incidents.truncate(limit); + + Ok(AiSkillIncidentResult { + incidents, + total_incidents, + candidate_event_rows: raw_candidate_count.min(SKILL_INCIDENT_CANDIDATE_CAP), + candidate_cap: SKILL_INCIDENT_CANDIDATE_CAP, + candidate_window_truncated, + truncated, + }) +} + +#[cfg(test)] +#[path = "skill_incidents_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/skill_incidents_tests.rs b/crates/shared/cortex/storage-sqlite/src/skill_incidents_tests.rs new file mode 100644 index 00000000..ef34f701 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/skill_incidents_tests.rs @@ -0,0 +1,308 @@ +use super::*; +use crate::config::StorageConfig; +use crate::pool::init_pool; +use crate::{DbPool, LogBatchEntry, insert_logs_batch}; + +fn test_pool() -> (DbPool, tempfile::TempDir) { + let dir = tempfile::tempdir().unwrap(); + let db_path = dir.path().join("test.db"); + let pool = init_pool(&StorageConfig::for_test(db_path)).unwrap(); + (pool, dir) +} + +fn make_ai_entry( + ts: &str, + host: &str, + tool: &str, + project: &str, + session_id: &str, + message: &str, +) -> LogBatchEntry { + LogBatchEntry { + timestamp: ts.to_string(), + hostname: host.to_string(), + facility: Some("local0".to_string()), + severity: "info".to_string(), + app_name: Some("ai-transcript".to_string()), + process_id: None, + message: message.to_string(), + raw: message.to_string(), + source_ip: "127.0.0.1:514".to_string(), + docker_checkpoint: None, + ai_tool: Some(tool.to_string()), + ai_project: Some(project.to_string()), + ai_session_id: Some(session_id.to_string()), + ai_transcript_path: Some(format!("{project}/{session_id}.jsonl")), + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + } +} + +#[allow(clippy::too_many_arguments)] +fn insert_skill_event( + pool: &DbPool, + log_id: i64, + ai_tool: &str, + ai_project: &str, + ai_session_id: &str, + hostname: &str, + timestamp: &str, + skill_name: &str, + skill_plugin: Option<&str>, +) { + // Note (PR 2 eng review sync): `skill_path`/`metadata_json` were dropped + // from `ai_skill_events` before PR 2 shipped — neither extractor ever set + // them. This INSERT reflects the shipped column set. + let conn = pool.get().unwrap(); + conn.execute( + "INSERT INTO ai_skill_events + (log_id, ai_tool, ai_project, ai_session_id, hostname, timestamp, + skill_name, skill_plugin, event_kind, evidence_kind, created_at) + VALUES (?1, ?2, ?3, ?4, ?5, ?6, ?7, ?8, 'skill_invoked', 'transcript', ?6)", + rusqlite::params![ + log_id, + ai_tool, + ai_project, + ai_session_id, + hostname, + timestamp, + skill_name, + skill_plugin, + ], + ) + .unwrap(); +} + +#[test] +fn search_ai_skill_incidents_groups_by_skill_session_window_and_scores() { + let (pool, _dir) = test_pool(); + + // Skill event log row. + let skill_log = make_ai_entry( + "2026-01-01T00:00:00Z", + "devhost", + "codex", + "/home/jmagar/workspace/cortex", + "sess-skill-1", + "loaded skill lavra:lavra-plan", + ); + // Correction anchor shortly after, same session. + let correction_log = make_ai_entry( + "2026-01-01T00:02:00Z", + "devhost", + "codex", + "/home/jmagar/workspace/cortex", + "sess-skill-1", + "That's not what I asked for, please redo it.", + ); + insert_logs_batch(&pool, &[skill_log, correction_log]).unwrap(); + + let log_ids: Vec = { + let conn = pool.get().unwrap(); + let mut stmt = conn.prepare("SELECT id FROM logs ORDER BY id ASC").unwrap(); + stmt.query_map([], |row| row.get::<_, i64>(0)) + .unwrap() + .collect::>>() + .unwrap() + }; + assert_eq!(log_ids.len(), 2); + + insert_skill_event( + &pool, + log_ids[0], + "codex", + "/home/jmagar/workspace/cortex", + "sess-skill-1", + "devhost", + "2026-01-01T00:00:00Z", + "lavra:lavra-plan", + Some("lavra"), + ); + + let result = search_ai_skill_incidents( + &pool, + &AiSkillIncidentParams { + skill: Some("lavra:lavra-plan".into()), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.incidents.len(), 1, "expected one grouped incident"); + let incident = &result.incidents[0]; + assert_eq!(incident.skill_name, "lavra:lavra-plan"); + assert_eq!(incident.skill_plugin.as_deref(), Some("lavra")); + assert_eq!(incident.tool, "codex"); + assert_eq!(incident.project, "/home/jmagar/workspace/cortex"); + assert_eq!(incident.session_id, "sess-skill-1"); + assert_eq!(incident.hostname, "devhost"); + assert_eq!(incident.skill_event_count, 1); + assert_eq!(incident.signal_counts.user_correction_after_skill, 1); + assert!( + incident + .signals_present + .contains(&"user_correction_after_skill".to_string()) + ); + // score = skill_event_count*2 + user_correction_count*15 + signal_variety*5 + // = 1*2 + 1*15 + 1*5 = 22 -> "medium" (>=15, <35) + assert!((incident.priority_score - 22.0).abs() < f64::EPSILON); + assert_eq!(incident.priority_label, "medium"); + assert!(!incident.incident_id.is_empty()); + assert!(incident.incident_id.starts_with("skill-inc-")); +} + +#[test] +fn search_ai_skill_incidents_sorts_by_score_with_total_cmp() { + let (pool, _dir) = test_pool(); + + // Two independent sessions -> two incidents with different scores. + // Session A: skill event only, no negative signal (low score). + let a_skill = make_ai_entry( + "2026-01-01T00:00:00Z", + "devhost", + "codex", + "/tmp/project-a", + "sess-a", + "loaded skill lavra:lavra-plan", + ); + // Session B: skill event + correction + tool failure (higher score). + let b_skill = make_ai_entry( + "2026-01-01T00:00:00Z", + "devhost", + "codex", + "/tmp/project-b", + "sess-b", + "loaded skill lavra:lavra-plan", + ); + let b_correction = make_ai_entry( + "2026-01-01T00:01:00Z", + "devhost", + "codex", + "/tmp/project-b", + "sess-b", + "you said you would run the tests but you didn't", + ); + let b_failure = make_ai_entry( + "2026-01-01T00:02:00Z", + "devhost", + "codex", + "/tmp/project-b", + "sess-b", + "command exited with exit code 1", + ); + insert_logs_batch(&pool, &[a_skill, b_skill, b_correction, b_failure]).unwrap(); + + let log_ids: Vec = { + let conn = pool.get().unwrap(); + let mut stmt = conn.prepare("SELECT id FROM logs ORDER BY id ASC").unwrap(); + stmt.query_map([], |row| row.get::<_, i64>(0)) + .unwrap() + .collect::>>() + .unwrap() + }; + insert_skill_event( + &pool, + log_ids[0], + "codex", + "/tmp/project-a", + "sess-a", + "devhost", + "2026-01-01T00:00:00Z", + "lavra:lavra-plan", + Some("lavra"), + ); + insert_skill_event( + &pool, + log_ids[1], + "codex", + "/tmp/project-b", + "sess-b", + "devhost", + "2026-01-01T00:00:00Z", + "lavra:lavra-plan", + Some("lavra"), + ); + + let result = search_ai_skill_incidents( + &pool, + &AiSkillIncidentParams { + skill: Some("lavra:lavra-plan".into()), + limit: Some(10), + ..Default::default() + }, + ) + .unwrap(); + + assert_eq!(result.incidents.len(), 2); + // Highest score first (session B). + assert_eq!(result.incidents[0].session_id, "sess-b"); + assert_eq!(result.incidents[1].session_id, "sess-a"); + assert!(result.incidents[0].priority_score > result.incidents[1].priority_score); + // Regression guard: scores must be a total order even in pathological + // cases (NaN would break partial_cmp/unwrap_or(Equal) but not total_cmp). + let mut scores = [f64::NAN, 3.0, 1.0, f64::NAN, 2.0]; + scores.sort_by(|a, b| b.total_cmp(a)); + assert_eq!( + scores.len(), + 5, + "total_cmp sort must not panic or drop elements on NaN" + ); +} + +#[test] +fn search_ai_skill_incidents_min_score_and_signals_filters() { + let (pool, _dir) = test_pool(); + let skill_log = make_ai_entry( + "2026-01-01T00:00:00Z", + "devhost", + "claude", + "/tmp/project-c", + "sess-c", + "loaded skill lavra:lavra-plan", + ); + insert_logs_batch(&pool, &[skill_log]).unwrap(); + let log_id: i64 = { + let conn = pool.get().unwrap(); + conn.query_row("SELECT id FROM logs LIMIT 1", [], |row| row.get(0)) + .unwrap() + }; + insert_skill_event( + &pool, + log_id, + "claude", + "/tmp/project-c", + "sess-c", + "devhost", + "2026-01-01T00:00:00Z", + "lavra:lavra-plan", + Some("lavra"), + ); + + // min_score above what a bare skill-event-only incident can reach (score=2) excludes it. + let filtered = search_ai_skill_incidents( + &pool, + &AiSkillIncidentParams { + skill: Some("lavra:lavra-plan".into()), + min_score: Some(10.0), + ..Default::default() + }, + ) + .unwrap(); + assert!(filtered.incidents.is_empty()); + + // signals filter for a category with zero hits also excludes it. + let filtered_by_signal = search_ai_skill_incidents( + &pool, + &AiSkillIncidentParams { + skill: Some("lavra:lavra-plan".into()), + signals: vec!["tool_failure_after_skill".into()], + ..Default::default() + }, + ) + .unwrap(); + assert!(filtered_by_signal.incidents.is_empty()); +} diff --git a/crates/shared/cortex/storage-sqlite/src/stream_health.rs b/crates/shared/cortex/storage-sqlite/src/stream_health.rs new file mode 100644 index 00000000..13b57b39 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/stream_health.rs @@ -0,0 +1,140 @@ +//! Per-stream last-seen rollup for stream-silence alerting. +//! +//! `stream_last_seen` (migration 43) holds one row per `(hostname, +//! source_kind)` with the newest `received_at` observed for that stream. The +//! notification evaluator refreshes it each cycle from a bounded window of +//! recent rows, then alerts on entries whose age crosses +//! `stream_silence_threshold_secs` — "this stream used to produce logs and +//! stopped". The logs table itself cannot answer that cheaply: source kind +//! lives inside `metadata_json`, so a direct newest-row-per-stream query +//! would scan and JSON-parse the whole window every cycle. +//! +//! The rollup tracks ALL kinds it observes; the alert query applies the +//! configured kind allowlist. Entries older than the forget horizon are +//! pruned so decommissioned streams stop being alert-eligible. + +use anyhow::Result; +use rusqlite::Connection; + +/// Window used to seed an empty rollup on the evaluator's first cycle. +/// Bounded so the one-time seed scan stays predictable; streams already +/// silent for longer than this at seed time never enter the rollup and +/// therefore never alert (documented tradeoff vs. a startup-blocking +/// backfill over the full retention window). +pub const STREAM_SEED_WINDOW_SECS: u64 = 86_400; + +/// Classify a log row's source kind. Single source of truth shared with +/// `ingest_health::ingest_source_kind_health` — prefix-mapped synthetic +/// sources first, then the denormalised `metadata_json.source_kind`. +const SOURCE_KIND_CASE: &str = "CASE + WHEN ai_transcript_path IS NOT NULL OR source_ip LIKE 'transcript://%' THEN 'transcript' + WHEN source_ip LIKE 'docker://%' THEN 'docker-stream' + WHEN source_ip LIKE 'docker-event://%' THEN 'docker-event' + WHEN source_ip LIKE 'agent-command://%' THEN 'agent-command' + WHEN source_ip LIKE 'shell-history://%' THEN 'shell-history' + WHEN source_ip LIKE 'file-tail://%' THEN 'file-tail' + ELSE json_extract(metadata_json, '$.source_kind') + END"; + +/// A stream that was active in the past but has gone silent. +#[derive(Debug, Clone)] +pub struct SilentStream { + pub hostname: String, + pub source_kind: String, + pub last_seen_at: String, + pub age_secs: u64, +} + +/// True when the rollup has no rows yet (fresh migration) — the caller +/// should refresh with [`STREAM_SEED_WINDOW_SECS`] instead of the cycle +/// window. +pub fn stream_last_seen_is_empty(conn: &Connection) -> Result { + let count: i64 = conn.query_row("SELECT COUNT(*) FROM stream_last_seen", [], |row| { + row.get(0) + })?; + Ok(count == 0) +} + +/// Fold the newest `received_at` per `(hostname, source_kind)` from the last +/// `window_secs` seconds of logs into the rollup. Monotonic: a conflict only +/// moves `last_seen_at` forward, so overlapping or stale windows can never +/// regress an entry. +pub fn refresh_stream_last_seen(conn: &Connection, window_secs: u64) -> Result { + let sql = format!( + "INSERT INTO stream_last_seen (hostname, source_kind, last_seen_at) + SELECT hostname, kind, MAX(received_at) + FROM ( + SELECT hostname, {SOURCE_KIND_CASE} AS kind, received_at + FROM logs + WHERE received_at >= strftime('%Y-%m-%dT%H:%M:%fZ', 'now', printf('-%d seconds', ?1)) + ) + WHERE kind IS NOT NULL AND kind != '' + GROUP BY hostname, kind + ON CONFLICT (hostname, source_kind) DO UPDATE SET + last_seen_at = CASE + WHEN excluded.last_seen_at > stream_last_seen.last_seen_at + THEN excluded.last_seen_at + ELSE stream_last_seen.last_seen_at + END" + ); + let changed = conn.execute(&sql, rusqlite::params![window_secs as i64])?; + Ok(changed) +} + +/// Streams whose newest row is older than `threshold_secs` but younger than +/// `forget_secs`, restricted to the configured kind allowlist. The forget +/// bound keeps long-dead streams from re-alerting after every dedup window +/// until pruning catches up. +pub fn silent_streams( + conn: &Connection, + kinds: &[String], + threshold_secs: u64, + forget_secs: u64, +) -> Result> { + if kinds.is_empty() { + return Ok(Vec::new()); + } + let placeholders = (1..=kinds.len()) + .map(|i| format!("?{i}")) + .collect::>() + .join(", "); + // Threshold and forget are trusted u64 config values, inlined because + // SQLite does not allow SELECT aliases in WHERE; the sender-controlled + // kind strings stay bound as parameters. + let age_expr = "CAST(strftime('%s','now') AS INTEGER) - \ + CAST(strftime('%s', last_seen_at) AS INTEGER)"; + let sql = format!( + "SELECT hostname, source_kind, last_seen_at, {age_expr} AS age_secs + FROM stream_last_seen + WHERE ({age_expr}) > {threshold_secs} AND ({age_expr}) < {forget_secs} + AND source_kind IN ({placeholders}) + ORDER BY hostname ASC, source_kind ASC" + ); + let mut stmt = conn.prepare(&sql)?; + let rows = stmt + .query_map(rusqlite::params_from_iter(kinds.iter()), |row| { + Ok(SilentStream { + hostname: row.get(0)?, + source_kind: row.get(1)?, + last_seen_at: row.get(2)?, + age_secs: row.get::<_, i64>(3)?.max(0) as u64, + }) + })? + .collect::>>()?; + Ok(rows) +} + +/// Drop rollup entries older than the forget horizon. +pub fn prune_stream_last_seen(conn: &Connection, forget_secs: u64) -> Result { + let deleted = conn.execute( + "DELETE FROM stream_last_seen + WHERE CAST(strftime('%s','now') AS INTEGER) - + CAST(strftime('%s', last_seen_at) AS INTEGER) >= ?1", + rusqlite::params![forget_secs as i64], + )?; + Ok(deleted) +} + +#[cfg(test)] +#[path = "stream_health_tests.rs"] +mod tests; diff --git a/crates/shared/cortex/storage-sqlite/src/stream_health_tests.rs b/crates/shared/cortex/storage-sqlite/src/stream_health_tests.rs new file mode 100644 index 00000000..014ca620 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/src/stream_health_tests.rs @@ -0,0 +1,167 @@ +use rusqlite::Connection; + +use super::{ + prune_stream_last_seen, refresh_stream_last_seen, silent_streams, stream_last_seen_is_empty, +}; + +fn conn_with_schema() -> Connection { + let conn = Connection::open_in_memory().expect("in-memory db"); + conn.execute_batch( + "CREATE TABLE stream_last_seen ( + hostname TEXT NOT NULL, + source_kind TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + PRIMARY KEY (hostname, source_kind) + ) WITHOUT ROWID; + CREATE TABLE logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + hostname TEXT NOT NULL, + source_ip TEXT NOT NULL DEFAULT '', + ai_transcript_path TEXT, + metadata_json TEXT, + received_at TEXT NOT NULL + );", + ) + .expect("schema"); + conn +} + +/// Insert a log row `age_secs` in the past. +fn insert_log( + conn: &Connection, + hostname: &str, + source_ip: &str, + metadata: Option<&str>, + age_secs: i64, +) { + conn.execute( + "INSERT INTO logs (hostname, source_ip, metadata_json, received_at) + VALUES (?1, ?2, ?3, strftime('%Y-%m-%dT%H:%M:%fZ', 'now', printf('-%d seconds', ?4)))", + rusqlite::params![hostname, source_ip, metadata, age_secs], + ) + .expect("insert log"); +} + +fn rollup_entry(conn: &Connection, hostname: &str, kind: &str) -> Option { + conn.query_row( + "SELECT last_seen_at FROM stream_last_seen WHERE hostname = ?1 AND source_kind = ?2", + rusqlite::params![hostname, kind], + |row| row.get(0), + ) + .ok() +} + +fn seed_rollup(conn: &Connection, hostname: &str, kind: &str, age_secs: i64) { + conn.execute( + "INSERT INTO stream_last_seen (hostname, source_kind, last_seen_at) + VALUES (?1, ?2, strftime('%Y-%m-%dT%H:%M:%fZ', 'now', printf('-%d seconds', ?3)))", + rusqlite::params![hostname, kind, age_secs], + ) + .expect("seed rollup"); +} + +#[test] +fn refresh_classifies_prefixes_and_metadata_kinds() { + let conn = conn_with_schema(); + insert_log(&conn, "nashost", "docker://nashost/plex/stdout", None, 10); + insert_log( + &conn, + "devhost", + "192.0.2.6:1234", + Some(r#"{"source_kind":"agent-docker"}"#), + 10, + ); + insert_log(&conn, "edgehost", "192.0.2.8:99", None, 10); // no kind — skipped + + refresh_stream_last_seen(&conn, 3600).expect("refresh"); + + assert!(rollup_entry(&conn, "nashost", "docker-stream").is_some()); + assert!(rollup_entry(&conn, "devhost", "agent-docker").is_some()); + let count: i64 = conn + .query_row("SELECT COUNT(*) FROM stream_last_seen", [], |r| r.get(0)) + .unwrap(); + assert_eq!(count, 2, "kindless row must not create an entry"); +} + +#[test] +fn refresh_is_monotonic_and_window_bounded() { + let conn = conn_with_schema(); + seed_rollup(&conn, "nashost", "syslog-tcp", 30); + // An older row inside the window must not regress the newer entry. + insert_log( + &conn, + "nashost", + "1.2.3.4:1", + Some(r#"{"source_kind":"syslog-tcp"}"#), + 600, + ); + // A row outside the window must be invisible to the refresh. + insert_log( + &conn, + "backuphost", + "1.2.3.5:1", + Some(r#"{"source_kind":"syslog-tcp"}"#), + 7200, + ); + + refresh_stream_last_seen(&conn, 3600).expect("refresh"); + + let kept = rollup_entry(&conn, "nashost", "syslog-tcp").expect("entry"); + let age: i64 = conn + .query_row( + "SELECT CAST(strftime('%s','now') AS INTEGER) - CAST(strftime('%s', ?1) AS INTEGER)", + [&kept], + |r| r.get(0), + ) + .unwrap(); + assert!(age < 120, "newer rollup value must survive, got age {age}s"); + assert!( + rollup_entry(&conn, "backuphost", "syslog-tcp").is_none(), + "row outside window must not enter the rollup" + ); +} + +#[test] +fn silent_streams_applies_threshold_forget_and_kind_bounds() { + let conn = conn_with_schema(); + seed_rollup(&conn, "nashost", "agent-docker", 7200); // silent 2h — alertable + seed_rollup(&conn, "devhost", "agent-docker", 60); // fresh — not silent + seed_rollup(&conn, "backuphost", "agent-docker", 700_000); // past forget — ignored + seed_rollup(&conn, "nashost", "shell-history", 7200); // silent but kind not listed + + let kinds = vec!["agent-docker".to_string()]; + let silent = silent_streams(&conn, &kinds, 3600, 604_800).expect("query"); + + assert_eq!(silent.len(), 1, "exactly one alertable stream: {silent:?}"); + assert_eq!(silent[0].hostname, "nashost"); + assert_eq!(silent[0].source_kind, "agent-docker"); + assert!(silent[0].age_secs > 3600 && silent[0].age_secs < 8000); +} + +#[test] +fn silent_streams_empty_kinds_returns_nothing() { + let conn = conn_with_schema(); + seed_rollup(&conn, "nashost", "agent-docker", 7200); + let silent = silent_streams(&conn, &[], 3600, 604_800).expect("query"); + assert!(silent.is_empty()); +} + +#[test] +fn prune_drops_only_forgotten_entries() { + let conn = conn_with_schema(); + seed_rollup(&conn, "nashost", "agent-docker", 700_000); + seed_rollup(&conn, "devhost", "agent-docker", 60); + + let deleted = prune_stream_last_seen(&conn, 604_800).expect("prune"); + assert_eq!(deleted, 1); + assert!(rollup_entry(&conn, "devhost", "agent-docker").is_some()); + assert!(rollup_entry(&conn, "nashost", "agent-docker").is_none()); +} + +#[test] +fn is_empty_reflects_rollup_population() { + let conn = conn_with_schema(); + assert!(stream_last_seen_is_empty(&conn).unwrap()); + seed_rollup(&conn, "nashost", "agent-docker", 60); + assert!(!stream_last_seen_is_empty(&conn).unwrap()); +} diff --git a/crates/shared/cortex/storage-sqlite/tests/fixtures/schema-43.sql b/crates/shared/cortex/storage-sqlite/tests/fixtures/schema-43.sql new file mode 100644 index 00000000..99bc8c37 --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/tests/fixtures/schema-43.sql @@ -0,0 +1,100 @@ +-- Synthetic Cortex schema-43 upgrade fixture. +-- Generated from the migration contract in src/db/pool.rs, never from a live DB. +-- All values are deterministic and contain no host, user, credential, or secret data. + +PRAGMA foreign_keys = OFF; + +CREATE TABLE schema_migrations ( + version INTEGER PRIMARY KEY, + applied_at TEXT NOT NULL DEFAULT '2026-01-01T00:00:00.000Z' +); +WITH RECURSIVE versions(version) AS ( + SELECT 1 + UNION ALL + SELECT version + 1 FROM versions WHERE version < 43 +) +INSERT INTO schema_migrations(version, applied_at) +SELECT version, '2026-01-01T00:00:00.000Z' FROM versions; + +CREATE TABLE logs ( + id INTEGER PRIMARY KEY AUTOINCREMENT, + timestamp TEXT NOT NULL, + hostname TEXT NOT NULL, + facility TEXT, + severity TEXT NOT NULL, + app_name TEXT, + process_id TEXT, + message TEXT NOT NULL, + raw TEXT NOT NULL, + received_at TEXT NOT NULL DEFAULT '2026-01-01T00:00:00.000Z', + source_ip TEXT NOT NULL DEFAULT '', + ai_tool TEXT, + ai_project TEXT, + ai_session_id TEXT, + ai_transcript_path TEXT, + metadata_json TEXT +); +INSERT INTO logs ( + id, timestamp, hostname, facility, severity, app_name, process_id, + message, raw, received_at, source_ip, ai_tool, ai_project, + ai_session_id, ai_transcript_path, metadata_json +) VALUES ( + 1, + '2026-01-01T00:00:00.000Z', + 'fixture-host', + 'user', + 'info', + 'fixture-app', + '1', + 'synthetic legacy log', + '<14>synthetic legacy log', + '2026-01-01T00:00:00.000Z', + '192.0.2.1', + 'fixture-tool', + 'fixture-project', + 'fixture-session', + 'fixture://transcript/session.jsonl', + '{"source_kind":"fixture"}' +); + +CREATE TABLE ai_session_rollup ( + ai_project TEXT NOT NULL, + ai_tool TEXT NOT NULL, + ai_session_id TEXT NOT NULL, + hostname TEXT NOT NULL, + ai_transcript_path TEXT, + first_seen TEXT NOT NULL, + last_seen TEXT NOT NULL, + event_count INTEGER NOT NULL, + PRIMARY KEY (ai_project, ai_tool, ai_session_id, hostname) +); +CREATE INDEX idx_ai_session_rollup_last_seen + ON ai_session_rollup(last_seen DESC); +INSERT INTO ai_session_rollup ( + ai_project, ai_tool, ai_session_id, hostname, ai_transcript_path, + first_seen, last_seen, event_count +) VALUES ( + 'fixture-project', 'fixture-tool', 'fixture-session', 'fixture-host', + 'fixture://transcript/session.jsonl', + '2026-01-01T00:00:00.000Z', '2026-01-01T00:00:00.000Z', 1 +); + +CREATE TABLE ai_session_rollup_meta ( + id INTEGER PRIMARY KEY CHECK (id = 1), + refreshed_at TEXT, + row_count INTEGER NOT NULL DEFAULT 0, + source_row_count INTEGER NOT NULL DEFAULT 0, + source_max_id INTEGER NOT NULL DEFAULT 0 +); +INSERT INTO ai_session_rollup_meta ( + id, refreshed_at, row_count, source_row_count, source_max_id +) VALUES (1, '2026-01-01T00:00:00.000Z', 1, 1, 1); + +CREATE TABLE stream_last_seen ( + hostname TEXT NOT NULL, + source_kind TEXT NOT NULL, + last_seen_at TEXT NOT NULL, + PRIMARY KEY (hostname, source_kind) +) WITHOUT ROWID; +INSERT INTO stream_last_seen(hostname, source_kind, last_seen_at) +VALUES ('fixture-host', 'fixture', '2026-01-01T00:00:00.000Z'); diff --git a/crates/shared/cortex/storage-sqlite/tests/public_api.rs b/crates/shared/cortex/storage-sqlite/tests/public_api.rs new file mode 100644 index 00000000..8bea945a --- /dev/null +++ b/crates/shared/cortex/storage-sqlite/tests/public_api.rs @@ -0,0 +1,57 @@ +use cortex_storage_sqlite::{ + KNOWN_SCHEMA_VERSION, LogBatchEntry, StorageConfig, fetch_patterns, init_pool, + insert_logs_batch, read_schema_version_info, tail_logs, +}; + +#[test] +fn independent_consumer_initializes_schema_and_round_trips_logs() { + let dir = tempfile::tempdir().unwrap(); + let config = StorageConfig { + db_path: dir.path().join("consumer.db"), + pool_size: 1, + wal_mode: false, + ..StorageConfig::default() + }; + + let pool = init_pool(&config).unwrap(); + let schema = read_schema_version_info(&pool).unwrap(); + assert_eq!(KNOWN_SCHEMA_VERSION, 47); + assert_eq!(schema.version, KNOWN_SCHEMA_VERSION); + assert_eq!(schema.known_version, KNOWN_SCHEMA_VERSION); + + let entry = LogBatchEntry { + timestamp: "2026-08-18T12:00:00Z".into(), + hostname: "dookie".into(), + facility: Some("daemon".into()), + severity: "info".into(), + app_name: Some("consumer-test".into()), + process_id: Some("1".into()), + message: "storage consumer round trip".into(), + raw: "storage consumer round trip".into(), + source_ip: "127.0.0.1:514".into(), + docker_checkpoint: None, + ai_tool: None, + ai_project: None, + ai_session_id: None, + ai_transcript_path: None, + metadata_json: None, + http_status: None, + auth_outcome: None, + dns_blocked: None, + event_action: None, + parse_error: None, + }; + insert_logs_batch(&pool, &[entry]).unwrap(); + + let rows = tail_logs(&pool, Some("dookie"), None, None, None, 10).unwrap(); + assert_eq!(rows.len(), 1); + assert_eq!(rows[0].message, "storage consumer round trip"); + assert_eq!(rows[0].hostname, "dookie"); + + let (patterns, scanned, truncated) = + fetch_patterns(&pool, None, None, None, None, None, 100, 10).unwrap(); + assert_eq!(scanned, 1); + assert!(!truncated); + assert_eq!(patterns.len(), 1); + assert_eq!(patterns[0].count, 1); +} diff --git a/crates/shared/operations/fleet/src/fanout_tests.rs b/crates/shared/operations/fleet/src/fanout_tests.rs index ef0bea94..9803e9b4 100644 --- a/crates/shared/operations/fleet/src/fanout_tests.rs +++ b/crates/shared/operations/fleet/src/fanout_tests.rs @@ -79,10 +79,7 @@ async fn fanout_classifies_failures_timeouts_and_partial_success() { .run(targets(4), CancellationToken::new(), |host, _| async move { match host.id().as_str() { "host1" => Err("driver failed"), - "host2" => { - tokio::time::sleep(Duration::from_millis(30)).await; - Ok("late") - } + "host2" => std::future::pending::>().await, _ => Ok("ok"), } }) diff --git a/crates/shared/self-update/src/transaction_async.rs b/crates/shared/self-update/src/transaction_async.rs index d4552c18..74bfbcdb 100644 --- a/crates/shared/self-update/src/transaction_async.rs +++ b/crates/shared/self-update/src/transaction_async.rs @@ -76,3 +76,76 @@ where ) })? } + +#[cfg(test)] +mod tests { + use std::sync::{Arc, Condvar, Mutex, mpsc}; + use std::time::Duration; + + use tokio::sync::oneshot; + + use super::*; + + #[tokio::test(flavor = "current_thread")] + async fn blocking_transaction_keeps_executor_responsive_while_work_is_held() { + let gate = Arc::new((Mutex::new(false), Condvar::new())); + let worker_gate = Arc::clone(&gate); + let watchdog_gate = Arc::clone(&gate); + let (entered_tx, entered_rx) = oneshot::channel(); + let (watchdog_arm_tx, watchdog_arm_rx) = mpsc::channel(); + + // The watchdog only bounds a broken implementation that executes the + // closure on this current-thread runtime. It is armed by the closure, + // so blocking-pool startup latency cannot consume the timeout budget. + let watchdog = std::thread::spawn(move || { + watchdog_arm_rx + .recv() + .expect("blocking closure should arm watchdog"); + let (lock, condvar) = &*watchdog_gate; + let released = lock.lock().unwrap(); + let (mut released, timeout) = condvar + .wait_timeout_while(released, Duration::from_secs(30), |released| !*released) + .unwrap(); + if timeout.timed_out() && !*released { + *released = true; + condvar.notify_all(); + true + } else { + false + } + }); + + let transaction = tokio::spawn(async move { + blocking_transaction(PathBuf::from("test-state"), move || { + let _ = entered_tx.send(()); + watchdog_arm_tx + .send(()) + .expect("watchdog thread should still be waiting"); + let (lock, condvar) = &*worker_gate; + let released = lock.lock().unwrap(); + drop(condvar.wait_while(released, |released| !*released).unwrap()); + Ok(()) + }) + .await + }); + + entered_rx + .await + .expect("blocking closure should start on a worker"); + + // This task must run while the blocking closure is still held. If the + // closure ever runs on the current-thread executor, only the watchdog + // can release it and the assertion below will fail. + assert_eq!(tokio::spawn(async { 42_u8 }).await.unwrap(), 42); + + let (lock, condvar) = &*gate; + *lock.lock().unwrap() = true; + condvar.notify_all(); + + transaction.await.unwrap().unwrap(); + assert!( + !watchdog.join().unwrap(), + "blocking transaction ran on the async executor instead of a blocking worker" + ); + } +} diff --git a/crates/shared/self-update/tests/transaction.rs b/crates/shared/self-update/tests/transaction.rs index 6945e475..b907c7df 100644 --- a/crates/shared/self-update/tests/transaction.rs +++ b/crates/shared/self-update/tests/transaction.rs @@ -133,34 +133,6 @@ async fn supported_source_mode_is_applied_only_during_final_install() { } } -#[tokio::test(flavor = "current_thread")] -async fn install_yields_the_async_executor_while_transaction_work_blocks() { - use std::sync::Arc; - use std::sync::atomic::{AtomicBool, Ordering}; - - let temp = tempdir().unwrap(); - let executable = temp.path().join("example"); - let state = temp.path().join("update.json"); - let old = b"#!/bin/sh\necho 'example 1.0.0'\n"; - let new = b"#!/bin/sh\necho 'example 2.0.0'\n"; - std::fs::write(&executable, old).unwrap(); - let updater = Updater::new( - UpdateLayout::new(&executable, &state), - UpdatePolicy::default(), - ); - let artifact = validated(&updater, new, "2.0.0").await; - let progressed = Arc::new(AtomicBool::new(false)); - let task_progressed = Arc::clone(&progressed); - let unrelated_task = tokio::spawn(async move { - task_progressed.store(true, Ordering::SeqCst); - }); - - updater.install(artifact, "1.0.0").await.unwrap(); - - assert!(progressed.load(Ordering::SeqCst)); - unrelated_task.await.unwrap(); -} - #[tokio::test] async fn oversized_previous_version_is_rejected_before_backup_or_swap() { let temp = tempdir().unwrap(); @@ -598,6 +570,16 @@ async fn validated( version: &str, ) -> soma_self_update::ValidatedArtifact { use std::os::unix::fs::PermissionsExt; + + // These integration tests exercise transaction semantics, not validator + // process concurrency. Running many temporary executable validators in + // parallel can make Linux report transient ETXTBSY while another test is + // still executing its own staged artifact, even though every staging path + // is unique. Serialize only the stage+validate window; install/recovery + // operations and the explicit concurrent-live-stage assertions remain + // parallel and independently exercised. + static VALIDATION_LOCK: tokio::sync::Mutex<()> = tokio::sync::Mutex::const_new(()); + let _validation_guard = VALIDATION_LOCK.lock().await; if let Ok(metadata) = std::fs::metadata(updater.layout().executable()) { let mode = metadata.permissions().mode(); if mode & 0o111 == 0 { diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index 50a848f0..05e042e9 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -1,7 +1,7 @@ --- title: "Architecture" created: 2026-05-15 -updated: 2026-08-17 +updated: 2026-08-18 doc_type: "guide" status: "active" owner: "soma" @@ -12,7 +12,7 @@ scope: "soma" source_of_truth: false upstream_refs: - "docs/PATTERNS.md" -last_reviewed: "2026-08-17" +last_reviewed: "2026-08-18" --- # Architecture @@ -63,6 +63,7 @@ crates/ cli-core/ ← reusable terminal/output/confirmation mechanics codemode/ ← reusable Code Mode runtime and runner support cortex/ + domain/ <- reusable Cortex semantic contracts and deterministic incident rules ingest-core/ ← reusable Cortex normalization/signature/metadata safety primitives http-api/ ← reusable API response/error/probe helpers http-server/ ← reusable Axum lifecycle/middleware helpers @@ -84,9 +85,12 @@ crates/ Shared crates are reusable building blocks below the Soma product layer and must not depend back on `apps/soma` or `crates/soma/**`. Namespaced capability families may also live below `crates/shared/` when they are intentionally reusable across -products. The first Cortex extraction proof lives at -`crates/shared/cortex/ingest-core`; its full extraction contract and target -composition are tracked in [`docs/cortex-extraction/`](cortex-extraction/README.md). +products. The Cortex shared family currently contains `cortex-domain` and +`cortex-ingest-core`. The former owns storage/transport-neutral semantic +contracts and deterministic incident rules; the latter owns ingest +normalization/signature/metadata safety primitives. Their extraction contract +and target composition are tracked in +[`docs/cortex-extraction/`](cortex-extraction/README.md). Two pieces sit outside the client → application → shim pattern: diff --git a/docs/cortex-extraction/MODEL-CLASSIFICATION.md b/docs/cortex-extraction/MODEL-CLASSIFICATION.md new file mode 100644 index 00000000..5225ac8c --- /dev/null +++ b/docs/cortex-extraction/MODEL-CLASSIFICATION.md @@ -0,0 +1,408 @@ +--- +title: "Cortex Model Classification" +created: 2026-08-18 +updated: 2026-08-18 +doc_type: "report" +status: "active" +owner: "soma" +audience: + - "contributors" + - "agents" +scope: "family" +source_of_truth: true +last_reviewed: "2026-08-18" +--- + +# Cortex model classification + +This inventory classifies every public type declared in the Cortex donor `src/app/models/*.rs` surface at commit `7edf23fadb94650c2d2a2f9c80111fb44319eea8`. The classification is about ownership, not about where the donor happens to define the type today. + +## Decision rules + +- **semantic contract**: meaning survives replacement of SQLite, HTTP/MCP/CLI, and process/runtime implementations; these are eligible for `cortex-domain`. +- **storage/query projection**: persistence statistics, database-maintenance state, or query-shaped rows whose ownership belongs with the storage adapter/application query layer. +- **transport DTO/policy**: request/response envelopes, pagination/filter input, surface-specific policy, and response-navigation metadata. +- **runtime/collector state**: OS/process/collector implementation state that belongs to the runtime capability producing it. + +Current totals: **255 public types**: 65 semantic, 165 transport, 23 storage/query, 2 runtime. Wave 1 extracts the stable semantic subset that is already useful without later storage/transport crates; semantic types still embedded in response-only aggregates remain assigned to the domain boundary but can move at cutover without changing this ownership decision. + +## Complete type inventory + +## `ai_hook_incidents.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `AiHookIncidentRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `HookSignalCounts` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `HookIncident` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `AiHookIncidentResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AiHookInvestigateRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `HookIncidentEvidence` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `HookIncidentSummary` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `AiHookInvestigateResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | + +## `ai_incidents.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `AiIncidentRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AiIncidentResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AbuseIncident` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `AiInvestigateRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `IncidentEvidence` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `AiInvestigateResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AiAssessRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AiAssessEvidenceSummary` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `AiAssessResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AbuseAssessRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AbuseAssessResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AiCorrelateRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AiCorrelationAnchor` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `AiCorrelateResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `TopicCorrelateRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `ResolvedTopicEntity` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `TopicExpansionEntity` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `TopicTimelineEntry` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `TopicCorrelateResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `CorrelatedLogRow` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `GraphSessionCorrelation` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | + +## `ai_inventory.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `UsageBlocksRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `UsageBlock` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `UsageBlocksResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `ProjectContextRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `ProjectContextResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `ListAiToolsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AiToolEntry` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `ListAiToolsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `ListAiProjectsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AiProjectEntry` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `ListAiProjectsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | + +## `ai_mcp_incidents.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `AiMcpIncidentRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `McpSignalCounts` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `McpIncident` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `AiMcpIncidentResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AiMcpInvestigateRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `McpIncidentEvidence` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `McpIncidentSummary` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `AiMcpInvestigateResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | + +## `ai_sessions.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `ListSessionsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `ListSessionsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AiSessionEntry` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `SearchSessionsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `SearchedSessionEntry` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `SearchSessionsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AbuseSearchRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AbuseMatch` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `AbuseSearchResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | + +## `ai_skill_incidents.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `AiSkillIncidentRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `SkillSignalCounts` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `SkillIncident` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `AiSkillIncidentResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AiSkillInvestigateRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `SkillIncidentEvidence` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `SkillIncidentSummary` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `AiSkillInvestigateResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | + +## `context.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `ContextRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `ContextResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `GetLogRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `GetLogResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `FeedLogsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `FeedLogsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `LogEntryWithRaw` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `IngestRateRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `IngestRateResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `IngestRateBuckets` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `IngestRatePerHost` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `SilentHostsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `SilentHostsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `SilentHostEntry` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `ClockSkewRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `ClockSkewResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `ClockSkewEntry` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `AnomaliesRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AnomaliesResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AnomalyEntry` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `CompareRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `CompareResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `RangeSummary` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | + +## `core.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `RequestActor` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `AiCorrelateLimitPolicy` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AiLimitPolicy` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `DbMaintenanceStatus` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `DbCheckpointResult` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `DbVacuumResult` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `DbIntegrityResult` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `DbIntegrityJobStarted` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `MaintenanceJobStatus` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `DbBackupRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `DbBackupResult` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `ServiceLogsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `ServiceLogsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `ServiceJournalEntry` | runtime/collector state | Assigned to runtime/capability owner; excluded from domain. | +| `AiWatchStatusReport` | runtime/collector state | Assigned to runtime/capability owner; excluded from domain. | +| `IncidentRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `IncidentResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `IncidentEvent` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `LogEntry` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `HostStateRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `HostStateResponse` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `FleetStateRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `FleetStateHostRow` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `FleetStateSummary` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `FleetStateResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `CorrelateStateRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `CorrelateStateWindow` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `CorrelateStateHostEntry` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `CorrelateStateResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | + +## `graph.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `GraphEntityLookupRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `GraphAroundRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `GraphExplainRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `GraphProjectionStatusResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `GraphRebuildStatsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `GraphRebuildResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `GraphEntity` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `GraphEntityCandidate` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `GraphRelationship` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `GraphEntitySummary` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `GraphEvidence` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `GraphEvidenceLookupRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `GraphSourceLogSummary` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `GraphEvidenceLookupResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `GraphNextQuery` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `GraphResponseMetadata` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `GraphEntityLookupResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `GraphAroundResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `GraphExplainResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `GraphIncidentNarrative` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `GraphNarrativeChain` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | + +## `hook_assess.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `HookAssessRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `HookAssessResult` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `HookAssessResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | + +## `hook_events.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `HookBackfillRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `HookBackfillResult` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `ListHookEventsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `HookEventEntry` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `ListHookEventsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | + +## `investigation.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `InvestigationVersionResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `InvestigationEnvelope` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `InvestigationMetadata` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `InvestigationBudget` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `InvestigationBudgetUsed` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `InvestigationClaimType` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `InvestigationClaim` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `AppEntitySummary` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `AppRelationshipSummary` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `AppEvidenceSummary` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `AppLogSummary` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `AppGraphResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AppGraphEntityResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AppGraphEvidenceResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AskInvestigationRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AskInvestigationResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | + +## `log_query.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `SearchLogsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `FilterLogsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `SearchLogsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `TailLogsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `ErrorSummaryEntry` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `GetErrorsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `GetErrorsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `HostEntry` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `ListHostsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `HomelabMapRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `HomelabMapResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `HomelabMapGraphAnswer` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `HomelabMapGraphTarget` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `HomelabMapAnswerRow` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `HomelabMapAnswerTruncation` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `HomelabMapNextQuery` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `HomelabMapProofQuery` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `TopologyFinding` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `TopologyFindingEntity` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `TopologyFindingEvidence` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `HomelabMapSummary` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `HomelabMapNode` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `HomelabMapSourceIp` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `HomelabMapApp` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `CortexOverlaySummary` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `CorrelateEventsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `CorrelatedHost` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `CorrelateEventsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | + +## `mcp_assess.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `McpAssessRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `McpAssessResult` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `McpAssessResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | + +## `mcp_events.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `McpBackfillRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `McpBackfillResult` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `ListMcpEventsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `McpEventEntry` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `ListMcpEventsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | + +## `ops.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `UnaddressedErrorsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `UnaddressedErrorsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `ErrorSignatureEntry` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `AckErrorRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AckErrorResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `UnackErrorRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `UnackErrorResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `NotificationsRecentRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `LlmInvocationsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AiCheckpointsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AiParseErrorsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AiPruneCheckpointsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `DbIntegrityRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `DbCheckpointRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `DbVacuumRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | + +## `rag.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `SimilarIncidentsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `CorrelatedSession` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `IncidentCluster` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `SimilarIncidentsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `IncidentContextRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `SeverityCount` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `AppLogCount` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `IncidentContextResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | + +## `skill_assess.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `SkillAssessRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `SkillAssessResult` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `SkillAssessResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | + +## `skill_events.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `SkillBackfillRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `SkillBackfillResult` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `ListSkillEventsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `SkillEventEntry` | semantic contract | Extracted to `cortex-domain` with product/storage conversions omitted. | +| `ListSkillEventsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | + +## `stats.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `DbStats` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `ListAppsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `ListAppsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AppEntry` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `ListSourceIpsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `ListSourceIpsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `SourceIpHostBreakdown` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `SourceIpEntry` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `TimelineRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `TimelineResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `TimelinePoint` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | +| `PatternsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `PatternsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `PatternEntry` | storage/query projection | Assigned to `cortex-storage-sqlite` / application query adapter; excluded from domain. | + +## `surface.rs` + +| Type | Classification | Wave 1 disposition | +|---|---|---| +| `AnalysisRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AnalysisResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `CorrelateRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `CorrelateResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `StateRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `StateResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `StatsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `StatsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `IngestRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `IngestResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AlertsRequest` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | +| `AlertsResponse` | transport DTO/policy | Assigned to API/MCP/CLI/application boundary; excluded from domain. | + +## Public re-exports not captured by declarations + +`ops.rs` additionally re-exports `FileTailAddRequest`, `FileTailOp`, `FileTailRequest`, `FileTailResponse`, `FileTailSource`, and `FileTailStatus` from `crate::filetail`. The request/response/op shapes are transport DTOs and the source/status values are runtime/collector state. None belong in `cortex-domain`. + +## Boundary violations found in the donor + +- **53** `impl From` mappings live beside public models. The extracted domain owns none of them; Wave 2 assigns them to `cortex-storage-sqlite`. +- `HostStateResponse`, `CorrelateStateHostEntry`, `GraphSessionCorrelation`, and topic-correlation payloads expose raw `db::Heartbeat* types. Wave 1 introduces domain-owned heartbeat contracts and uses those in extracted semantic aggregates. +- MCP and skill incident evidence expose raw `Vec` / `Vec`. Extracted evidence uses domain-owned `McpEventEntry` / `SkillEventEntry`. +- `AiWatchStatusReport` exposes `crate::scanner::AiIndexingHealth`; `DbStats::from` reads a `crate::receiver` process counter; `ops.rs` re-exports `crate::filetail`; `surface.rs` embeds `crate::config::NotificationsConfig`; and `log_query.rs` imports `crate::inventory::schema::* . Those stay with runtime, transport, or inventory owners. + +## Error taxonomy ownership + +The donor `ServiceError` mixes semantic failures with storage/runtime classification. `cortex-domain` extracts only `DomainError::InvalidInput` and `DomainError::NotFound`. SQLite busy/timeout, constraint violations, row-not-found persistence details, pool starvation, and opaque `anyhow` failures stay in application/storage adapters, which translate them at the surface boundary. + +## Completeness check + +The inventory was generated against all `pub struct`, `pub enum`, and `pub type` declarations under the donor `src/app/models/*.rs`; all **255** declarations are represented exactly once above. The six `filetail` re-exports are recorded separately because they are not declarations in that directory. diff --git a/docs/cortex-extraction/PROGRESS.md b/docs/cortex-extraction/PROGRESS.md index 0f9354ec..d441cb48 100644 --- a/docs/cortex-extraction/PROGRESS.md +++ b/docs/cortex-extraction/PROGRESS.md @@ -1,7 +1,7 @@ --- title: "Cortex Extraction Progress" created: 2026-08-17 -updated: 2026-08-17 +updated: 2026-08-18 doc_type: "report" status: "active" owner: "soma" @@ -10,7 +10,7 @@ audience: - "agents" scope: "family" source_of_truth: true -last_reviewed: "2026-08-17" +last_reviewed: "2026-08-18" --- # Cortex Extraction Progress @@ -22,6 +22,11 @@ evidence exists on the branch or in the linked lane PR. - Donor baseline: `7edf23fadb94650c2d2a2f9c80111fb44319eea8` - Soma integration branch: `feat/cortex-shared-extraction` +- Foundation PR: [#363](https://github.com/dinglebear-ai/soma/pull/363) +- Wave 1 lane: `feat/cortex-domain-extraction` +- Wave 1/Wave 2 integration PR: [#364](https://github.com/dinglebear-ai/soma/pull/364) +- Runner-capacity decision: Wave 2 is intentionally batched into #364 rather than opening another stacked lane while the shared Rust runner pool is saturated. +- Public model ownership inventory: [MODEL-CLASSIFICATION.md](MODEL-CLASSIFICATION.md) - Working topology: [SPEC.md](SPEC.md) - Normative rules: [CONTRACTS.md](CONTRACTS.md) - Verification gates: [VERIFICATION.md](VERIFICATION.md) @@ -47,25 +52,26 @@ evidence exists on the branch or in the linked lane PR. ## Wave 1: Domain seam -- [ ] Classify every public `app/models/**` type as semantic contract, storage projection, transport DTO, or runtime state. -- [ ] Introduce `cortex-domain` with only storage/transport-neutral contracts. -- [ ] Move service error taxonomy/invariants that truly belong to domain. -- [ ] Relocate `From` mappings out of the domain dependency direction. -- [ ] Remove raw DB, scanner, receiver-counter, filetail, and runtime-config types from public domain responses. -- [ ] Add serialization/parity fixtures for user-visible response models. -- [ ] Add independent consumer tests and README/rustdoc. -- [ ] Pass architecture/all-features gates. +- [x] Classify every public `app/models/**` type as semantic contract, storage projection, transport DTO, or runtime state. +- [x] Introduce `cortex-domain` with only storage/transport-neutral contracts. +- [x] Move service error taxonomy/invariants that truly belong to domain. +- [x] Relocate `From` mappings out of the domain dependency direction. +- [x] Remove raw DB, scanner, receiver-counter, filetail, and runtime-config types from public domain responses. +- [x] Add serialization/parity fixtures for user-visible response models. +- [x] Add independent consumer tests and README/rustdoc. +- [x] Pass architecture/all-features gates. ## Wave 2: SQLite storage adapter -- [ ] Create `cortex-storage-sqlite`. -- [ ] Move pool initialization and SQLite configuration. -- [ ] Move migrations with exact migration-order/version parity tests. -- [ ] Move query, FTS, retention, storage-budget, incident/event, graph, and observatory persistence. -- [ ] Implement domain/application repository ports without exposing raw row types upward. -- [ ] Preserve single-writer/maintenance coordination semantics. -- [ ] Add temporary-database consumer fixtures. -- [ ] Pass donor DB suite plus workspace gates. +- [x] Create `cortex-storage-sqlite`. +- [x] Move pool initialization and SQLite configuration. +- [x] Move migrations with exact migration-order/version parity tests. +- [x] Move query, FTS, retention, storage-budget, incident/event, graph, and observatory persistence. +- [x] Implement domain/application repository ports without exposing raw row types upward. +- [x] Preserve single-writer/maintenance coordination semantics. +- [x] Add temporary-database consumer fixtures. +- [x] Keep donor-parity oversized modules warning-visible with narrow path-specific PATTERNS budgets rather than a crate-wide exemption. +- [x] Pass donor DB suite plus workspace gates. Final SQLite suite: 441 passed, 1 intentionally ignored; external-consumer test: 1/1 passed; workspace all-features check passed; workspace Nextest: 3,528/3,528 passed with 4 skipped. ## Wave 3: Ingest engines @@ -81,7 +87,7 @@ evidence exists on the branch or in the linked lane PR. ## Wave 4: Inventory, observatory, and agent -- [ ] Create `cortex-inventory` and move normalized inventory/cache/collector behavior. +- [ ] Create `cortex-inventory` and move normalized inventory/cache/collector behavior. Pure snapshot schema/limits are staged early in Wave 2 so SQLite graph projection depends downward on a stable contract; collectors/cache/orchestration remain Wave 4 work. - [ ] Feature-gate service-specific collectors where practical. - [ ] Create `cortex-observatory` with persistence ports. - [ ] Move identity, attribution, classification, lifecycle, and projector behavior. @@ -121,6 +127,9 @@ evidence exists on the branch or in the linked lane PR. - [ ] Run complete Cortex donor behavior/surface suite against composed Soma workspace crates. - [ ] Run live-safe smoke tests that do not mutate homelab state. - [ ] Remove obsolete duplicated donor modules. +- [ ] Split the six Wave 2 parity-preserved oversized SQLite modules along stable seams and remove their transitional PATTERNS budgets. +- [ ] Re-audit D11 after all extraction/cutover work: no raw SQLite/internal projection row type may escape the storage public API. +- [ ] Re-audit D12 after all extraction/cutover work: `cortex-storage-sqlite` must still contain zero `dead_code` suppressions. - [ ] Prove no business logic remains duplicated between Cortex app and shared crates. - [ ] Sweep all docs, examples, manifests, CI, release metadata, and dependency references. - [ ] Re-run full Soma workspace gates. diff --git a/docs/cortex-extraction/README.md b/docs/cortex-extraction/README.md index faf5917a..e4725b23 100644 --- a/docs/cortex-extraction/README.md +++ b/docs/cortex-extraction/README.md @@ -1,7 +1,7 @@ --- title: "Cortex Shared-Crate Extraction" created: 2026-08-17 -updated: 2026-08-17 +updated: 2026-08-18 doc_type: "guide" status: "active" owner: "soma" @@ -10,7 +10,7 @@ audience: - "agents" scope: "family" source_of_truth: true -last_reviewed: "2026-08-17" +last_reviewed: "2026-08-18" --- # Cortex Shared-Crate Extraction @@ -39,6 +39,7 @@ source reference for parity work. | [SPEC.md](SPEC.md) | Target crate architecture, dependency graph, runtime composition, and migration sequence. | | [CONTRACTS.md](CONTRACTS.md) | Rules every extracted crate and adapter must satisfy. | | [SOURCE-INVENTORY.md](SOURCE-INVENTORY.md) | Current Cortex modules, coupling hotspots, and planned destinations. | +| [MODEL-CLASSIFICATION.md](MODEL-CLASSIFICATION.md) | Complete ownership classification for all 255 public donor model declarations. | | [PROGRESS.md](PROGRESS.md) | Dedicated lane-by-lane extraction tracker and definition of done. | | [VERIFICATION.md](VERIFICATION.md) | Required build, test, docs, architecture, parity, and smoke gates. | | [REVIEW.md](REVIEW.md) | Review passes, findings, resolutions, and final evidence for this extraction branch. | @@ -46,16 +47,26 @@ source reference for parity work. ## Current implementation -The first proof crate is -`crates/shared/cortex/ingest-core` (package `cortex-ingest-core`). It extracts -Cortex's message normalization/signature logic and bounded metadata redaction. -It deliberately does not know about SQLite, Axum, RMCP, Labby auth, process -runtime, or deployment. +The shared family now has four implemented/staged crates. `cortex-ingest-core` +contains message normalization/signature logic, bounded metadata redaction, and +the canonical ingest source-kind vocabulary. `cortex-domain` contains all 65 +donor public model declarations classified as semantic contracts plus the pure +incident/signal, heartbeat, observatory-identity, and graph-confidence policy. +`cortex-storage-sqlite` owns the donor SQLite pool, migrations, queries/FTS, +retention/storage budget, event/incident persistence, graph projection, and +observatory persistence behind explicit storage ports. `cortex-inventory` is +staged early with only the pure snapshot schema/limits needed by storage graph +projection; collectors/cache/orchestration remain Wave 4 work. -This proof establishes the pattern future Cortex crates must copy: workspace -package inheritance, `layer = "shared"` architecture metadata, explicit -features, `publish = false` during stabilization, README and crate-level docs, -ported donor tests, and tests that exercise only the public consumer API. +The domain lane records ownership for all 255 public donor model declarations. +Exact semantic duplicates discovered during storage extraction are re-used from +`cortex-domain`; storage/query projections with genuinely different join fields +or serde behavior stay explicit. None of the lower crates depends upward on +Cortex application/runtime namespaces. All four use workspace package +inheritance, `layer = "shared"` architecture metadata, explicit features, and +`publish = false` during stabilization; the completed Wave 0/1 crates and the +Wave 2 storage adapter carry donor/public-consumer tests appropriate to their +boundaries. ## Completion condition diff --git a/docs/cortex-extraction/REVIEW.md b/docs/cortex-extraction/REVIEW.md index b53fccfb..fa1f1680 100644 --- a/docs/cortex-extraction/REVIEW.md +++ b/docs/cortex-extraction/REVIEW.md @@ -1,7 +1,7 @@ --- title: "Cortex Extraction Review Log" created: 2026-08-17 -updated: 2026-08-17 +updated: 2026-08-18 doc_type: "report" status: "active" owner: "soma" @@ -10,7 +10,7 @@ audience: - "agents" scope: "family" source_of_truth: true -last_reviewed: "2026-08-17" +last_reviewed: "2026-08-18" --- # Cortex Extraction Review Log @@ -254,3 +254,355 @@ and Cargo warns that `incus-client` and `codex-app-server-client` both have an example output named `basic`. None requires a Cortex behavior change, so this branch records them without folding unrelated fleet-policy migration, schema regeneration, or example renaming into the extraction. + +## Review 3: Wave 1 domain seam + +### Finding C1: the donor model module has four different owners + +**Severity:** P1 if copied wholesale. + +The donor exposes 255 public model declarations from one application module, but +the declarations do not share an architectural owner. The complete classification +records 65 semantic contracts, 165 transport DTO/policy types, 23 storage/query +projections, and 2 runtime/collector state types. + +**Resolution:** [MODEL-CLASSIFICATION.md](MODEL-CLASSIFICATION.md) classifies all +255 declarations exactly once. All 65 semantic donor declarations are represented +in `cortex-domain`; no type classified as semantic remains unowned. + +### Finding C2: storage types leaked through otherwise semantic contracts + +**Severity:** P1 for a reusable domain crate. + +The donor keeps 53 `impl From` mappings beside public models and also +exposes raw heartbeat, MCP-event, and skill-event database types from semantic +aggregates. + +**Resolution:** `cortex-domain` owns no database-row conversion. It introduces +domain-owned heartbeat contracts, uses `McpEventEntry` / `SkillEventEntry` in +evidence bundles, and assigns row-to-domain mapping to the Wave 2 SQLite adapter. +The donor remains unchanged until cutover, so extraction does not alter the live +Cortex product while dependency direction is being repaired. + +### Finding C3: ServiceError mixes domain meaning with adapter failures + +**Severity:** P1 if moved unchanged. + +`ServiceError` combines invalid/not-found semantic outcomes with SQLite busy, +timeout, constraint, row, pool, and opaque runtime errors. + +**Resolution:** the domain crate exposes only `DomainError::InvalidInput` and +`DomainError::NotFound`. Storage/application adapters retain operational error +classification and translate those failures at their surface boundaries. + +### Finding C4: deterministic finding engines are domain behavior + +**Severity:** P2 if left coupled to the monolithic application module. + +The incident, hook, MCP, and skill finding engines are pure deterministic rule +evaluation. They query no database and invoke no model, but donor location under +`app/` obscured that property. + +**Resolution:** all four engines move with their donor parity tests. Their only +adaptations are crate-local imports and replacing raw database event arguments +with domain event contracts. Existing evidence-id, conservative-confidence, +determinism, and unknown/open-question behavior remains covered. + +### Finding C5: copied comments violated Soma ASCII source hygiene + +**Severity:** P2 CI failure if left unresolved. + +Donor comments used typographic punctuation and box-drawing characters. + +**Resolution:** Rust source comments are normalized to ASCII spellings while code +and runtime strings remain unchanged. The domain source tree is ASCII-clean. + +### Finding C6: transport envelopes are not domain contracts + +**Severity:** P2 architecture drift. + +Request/response envelopes, surface limit policy, graph response-navigation +metadata, maintenance/query result projections, and collector implementation +state were tempting to move because many are serde-only. Their semantics are +still surface, storage, or runtime-specific. + +**Resolution:** these types remain explicitly assigned to later API/MCP/CLI, +application/query, SQLite, inventory, or runtime lanes in the model inventory. +At the Wave 1 checkpoint the domain manifest contained only `serde`, +`serde_json`, and `thiserror`. Wave 2 adds `chrono` solely for pure heartbeat +time/skew policy; it still has no storage, transport, auth, scanner, collector, +or runtime dependency. + +### Finding C7: fanout timeout fixture raced two short timers under load + +**Severity:** P1 for a trustworthy all-features gate. + +The first final workspace Nextest run reached 3,037 passing tests but exposed a +pre-existing flake in `soma-fleet::fanout_classifies_failures_timeouts_and_partial_success`. +The fixture raced a 10 ms timeout against a 30 ms Tokio sleep. Under heavy +parallel test/compile load both timers can become ready before the runtime polls +them again, allowing the inner sleep result to win and incorrectly making the +fixture report three successes instead of two. + +**Resolution:** replace the intentionally late branch with a permanently pending +future. That leaves the scheduler timeout as its only possible terminal path and +tests the behavior the fixture actually claims to test without wall-clock +racing. The targeted case and all 40 `soma-fleet` tests pass, the corrected case +passed 500 consecutive stress executions, and the subsequent full workspace +Nextest run passed 3,038/3,038. Production fanout logic is unchanged. + +## Wave 1 final verification + +- Cargo metadata registers `cortex-domain` as workspace member 42. +- All 255 donor public model declarations are classified exactly once; all 65 + semantic donor declarations are represented in the domain crate, and normalized + shape comparison reports 65/65 matches after the documented adapter substitutions. +- `cargo check -p cortex-domain --all-features` and the final + `cargo check --workspace --all-features` passed. +- `cargo clippy -p cortex-domain --all-targets --all-features -- -D warnings` passed. +- `cargo test -p cortex-domain --all-features` passed 42 unit/parity tests and 2 + independent-consumer integration tests. +- `RUSTDOCFLAGS="-D warnings" cargo doc -p cortex-domain --no-deps --all-features` + passed with only the known fleet-required renamed-lint warning, which rustdoc + explicitly exempts from `-D warnings`. +- `cargo nextest run --workspace --all-features` passed 3,038/3,038 runnable tests + with 3 skipped after resolving the surfaced fanout fixture race. +- `cargo xtask check-architecture` passed with 42 workspace packages and 92 + internal edges; `check-test-siblings` passed with 24 checked source trees. +- ASCII hygiene, coupled-file ownership, generated/docs checks, and the Python + platform gates pass. +- The exact fleet contract implementation pinned by Soma CI at + `ac57c3208cf92d71c5971bb936df51c400cb1ccf` reports `fleet contract valid`. +- Full `cargo deny check` reports advisories, bans, licenses, and sources all ok; + the stacked lockfile contains patched `h2 0.4.16`. +- The crate source and manifest contain no database/pool, HTTP/MCP, auth, scanner, + receiver, file-tail, config, or product-runtime dependency. +- The Rust source tree is ASCII-clean after comment-only normalization. + +## Review 4: Wave 2 SQLite boundary + +### Finding D1: donor SQLite dependency versions cannot coexist unchanged in Soma + +**Severity:** P1 integration blocker. + +Cortex donor storage uses `rusqlite 0.39`/`r2d2_sqlite 0.34`, while Soma already +links SQLite through `rusqlite 0.40`. Cargo permits only one crate with the +`links = "sqlite3"` native linkage in this workspace. + +**Resolution:** retain donor storage behavior while aligning the adapter to +`rusqlite 0.40` and `r2d2_sqlite 0.35`, the matching pool adapter release. The +extracted storage crate compiles cleanly against that pair; migration and donor +DB tests remain the behavioral guard for this integration-only version change. + +### Finding D2: copied DB modules contained forty upward product references + +**Severity:** P1 architecture violation. + +The mechanical DB extraction initially referenced application signal detectors, +scanner event types, inventory runtime schema, enrichment `SourceKind`, agent +Docker constants, observatory identity helpers, and application heartbeat/error +policy. Preserving those imports would turn the storage crate into a disguised +copy of the Cortex monolith. + +**Resolution:** introduce storage-neutral normalized event inputs; move pure +incident detectors, heartbeat policy, observatory identity keys, and graph +confidence math into `cortex-domain`; move canonical ingest source-kind values +into `cortex-ingest-core`; and stage the pure inventory snapshot schema/limits in +`cortex-inventory`. The tracked upward-reference scan now reports zero matches +for app, scanner, inventory runtime, enrichment, agent, observatory identity, +normalization, or the old db namespace. + +### Finding D3: storage reintroduced domain-owned semantic response types + +**Severity:** P1 boundary drift. + +The copied DB model module still defined semantic types already classified and +extracted in Wave 1. Keeping independent copies would allow storage and domain +wire shapes to diverge while presenting both as canonical Cortex concepts. + +**Resolution:** exact duplicates now reuse the domain contracts directly: +`LogEntry`, `AbuseIncident`, `AiAbuseMatch` (as the domain `AbuseMatch`), +`SeverityCount`, and `AppLogCount`; the hook/MCP/skill event entries; the +hook/MCP/skill incident and signal-count contracts; and exact graph entity / +entity-candidate rows. Query projections whose field names, join payload, or +serde behavior intentionally differ remain storage-owned until the application +facade maps them. A mechanical same-name scan now finds zero structs defined in +both crates. The error-signature read path was tightened further: the raw +`SignatureRow` adapter type was removed and storage now returns +`cortex_domain::ErrorSignatureEntry` directly while keeping +`normalizer_version` only as a persistence key/input. + +### Finding D4: later-wave storage consumers looked like dead code after extraction + +**Severity:** P2 lint/API-design risk. + +Several donor DB capabilities are consumed by application/runtime modules that +have not moved yet: error-signature scanning, notification outbox/firings, LLM +invocation persistence, stream health, observatory paging, pattern clustering, +and PRAGMA diagnostics. As crate-private functions they became dead-code +warnings; deleting them would break later product parity, while a blanket lint +allowance would hide real stale code. + +**Resolution:** make the donor-used persistence capabilities deliberate public +storage ports and remove obsolete monolith-only crate-root reexports. PRAGMA +interpolation was tightened to a closed `PragmaName` enum rather than exposing +arbitrary identifiers. Pure graph-confidence math was removed from SQLite and +relocated to `cortex-domain`. With the temporary unused-import allowance +removed, `cargo check -p cortex-storage-sqlite` completes with zero warnings. + +### Finding D5: donor DB tests crossed extraction boundaries + +**Severity:** P1 if silently dropped or copied with fake product namespaces. + +The first extracted test compile found three harness gaps: the frozen schema-43 +fixture still used its donor-relative path, query tests required a test-only +`regex` dependency, and one maintenance test called the application +notification rule evaluator after proving the storage-budget invariant. + +**Resolution:** copy the immutable schema-43 fixture into the storage crate and +adjust only its relative include path; add `regex` as a dev dependency; and +keep the maintenance test scoped to persistence behavior (external disk pressure +blocks writes without deleting retained rows). Notification firing remains an +application-layer policy test for its later extraction wave rather than creating +a fake `notifications::rules` namespace inside storage. The frozen storage +suite subsequently passes 440 tests with one intentionally ignored benchmark, +plus the independent temporary-database consumer test. + +### Finding D6: the mechanical DB copy did not satisfy Soma sibling-test layout + +**Severity:** P1 repository-contract failure. + +Registering the SQLite source tree with the repository sibling checker exposed +16 modules without focused `_tests.rs` siblings and two extra donor test modules +whose names did not correspond to source files. The large donor suites covered +much of this code indirectly, but the extraction would still violate Soma's explicit source/test ownership convention. + +**Resolution:** add focused tests for resolver vocabulary/observations/adapters, +inventory SQL helpers, storage configuration, OTLP rows, ingest health, and the +Agent Observatory projection lookup/SQL/ref/type/tie-break/counter helpers. The +existing observatory model suite was renamed to the real `agent_observatory.rs` +sibling. `queries_graph_tests.rs` remains a deliberate second split suite for +`queries.rs` and is documented as such in the orphan-exemption list. The gate +now passes with 26 checked source trees. The added counter test exercises a real +atomic projection write and verifies event/error counter updates. + +### Finding D7: public extraction docs linked private implementation helpers + +**Severity:** P2 strict-rustdoc failure. + +Making storage APIs public caused seven inherited donor doc comments to create +rustdoc links to private constants/helpers. Widening those private helpers only +for documentation would have enlarged the API for the wrong reason. + +**Resolution:** rewrite the seven links as plain code names/descriptions while +keeping implementation visibility private. Strict rustdoc then passes for all +four Cortex shared crates; the only emitted warning is the known fleet-required +`missing_crate_level_docs` rename diagnostic, which explicitly ignores +`-D warnings`. + +### Finding D8: module-wide dead-code suppression hid extraction state + +**Severity:** P2 reviewability/API-risk. + +Four copied modules (`graph`, Agent Observatory, OTLP metrics, and OTLP traces) +still carried donor-level `#![allow(dead_code)]` attributes. Those blanket +suppressions made it impossible for strict linting to distinguish intentionally +exported later-wave storage ports from genuinely orphaned extraction code. + +**Resolution:** remove all four module-wide suppressions and rerun strict Clippy. +The only newly surfaced dead code was four Agent Observatory cursor/health +persistence helpers; those are real later-wave storage capabilities, so they are +now explicit documented public ports rather than lint-hidden internals. A later +exact-head review removed the remaining narrow allowances after verifying that +their public/test contract reachability did not require suppression; see D12. + +### Finding D9: projection cursor initialization bypassed the global SQLite writer lock + +**Severity:** P1 single-writer contract violation. + +`projection_cursor` lazily initializes a missing cursor with `INSERT OR IGNORE`, +but unlike the adjacent cursor-advance and health-write functions it did not +acquire the process-wide `write_lock()`. A future Observatory projector could +therefore perform this SQLite write outside the adapter's single-writer +coordination path. + +**Resolution:** acquire `write_lock()` before cursor initialization, expose the +cursor/health helpers as deliberate storage ports, and add a temporary-database +round-trip test covering cursor initialization/advance plus health attempt +accumulation. The focused regression test and strict all-target Clippy pass. + +### Finding D10: donor Unicode fixtures violated Soma source hygiene + +**Severity:** P1 repository-contract failure. + +The exact-head ASCII gate found non-ASCII math notation in extracted confidence comments plus Unicode behavior fixtures in domain, inventory, and SQLite tests. The runtime values were legitimate test inputs, so blindly transliterating them would have weakened parity coverage. + +**Resolution:** rewrite mathematical comments/formulas with ASCII notation and encode intentional Unicode fixture values with Rust Unicode escapes. This keeps the runtime strings byte-for-byte equivalent while satisfying the source policy. The ASCII gate passes, and the affected domain/inventory parity suites remain green. + +### Finding D11: an opaque pattern source row escaped the storage boundary + +**Severity:** P1 adapter-boundary violation. + +`PatternSourceRow` was publicly re-exported even though callers could not read its +private fields. Application code therefore had to accept a SQLite-shaped +intermediate value from `fetch_pattern_rows` only to hand it back to +`cluster_pattern_rows`, contradicting the Wave 2 rule that raw storage rows do +not escape upward. + +**Resolution:** add the public `fetch_patterns` storage port, keep +`PatternSourceRow`, `fetch_pattern_rows`, and `cluster_pattern_rows` +crate-private, and extend both the focused analytics suite and the independent +consumer test through the new public port. The application-facing result now +contains only `PatternEntry` values, scanned-row count, and truncation state. + +### Finding D12: narrow dead-code suppressions outlived the extraction + +**Severity:** P2 reviewability/API hygiene. + +Five targeted `dead_code` allowances remained after the module-wide cleanup: a +resolver helper, reserved resolver variants, two notification row fields, and a +timeline diagnostic field. All five are now reachable through public storage +contracts or focused tests, so suppressing the lint no longer documents a real +exception. + +**Resolution:** remove every remaining `dead_code` allowance from +`cortex-storage-sqlite` and rerun strict all-target Clippy. The crate now needs +zero dead-code suppression. + +### Finding D13: donor-parity modules exceeded Soma hard file-size limits + +**Severity:** P1 repository-contract failure. + +The fresh `Soma Contracts` CI run correctly rejected six mechanically extracted +SQLite donor modules whose effective line counts exceeded twice Soma's ordinary +Rust module target: `analytics.rs`, `graph.rs`, `graph_inventory.rs`, +`maintenance.rs`, `pool.rs`, and `queries.rs`. A six-module behavioral split at +the end of the extraction lane would materially increase parity risk and make +the already-green donor/storage evidence stale. + +**Resolution:** use the repository's existing path-specific transitional size +budget mechanism rather than a blanket exemption. Each of the six modules gets +a tight warning-visible budget sized just above half of its current extracted +size, so current donor parity can land while any meaningful further growth still +hard-fails. A focused xtask regression test pins all six exact paths and proves a +neighboring storage module remains on the ordinary 350-line target. The modules +remain explicitly scheduled for real decomposition before final cutover and +publication; the policy exception must shrink or disappear as those seams land. + +## Wave 2 final verification + +- `cargo fmt --all -- --check` passed. +- Strict Clippy passed for `cortex-domain`, `cortex-ingest-core`, `cortex-inventory`, and `cortex-storage-sqlite` with `--all-targets --all-features -- -D warnings`. +- Strict rustdoc passed for all four crates with only the fleet-required renamed-lint notice that explicitly ignores `-D warnings`. +- The final frozen `cortex-storage-sqlite` suite passed 441 runnable tests with one intentionally ignored benchmark; its external-consumer integration test passed 1/1. +- `cargo check --workspace --all-features` passed across all 44 workspace packages. +- `cargo nextest run --workspace --all-features --no-fail-fast` passed 3,528/3,528 runnable tests with 4 skipped. +- `cargo xtask check-architecture` passed with 44 workspace packages and 95 internal edges. +- `cargo xtask check-test-siblings` passed with 26 checked source trees after adding focused Wave 2 siblings. +- ASCII hygiene, documentation generation/policy, and coupled-file ownership checks passed. +- Full `cargo deny check` passed. +- The exact fleet contract pinned at `ac57c3208cf92d71c5971bb936df51c400cb1ccf` reports `fleet contract valid`. +- The storage source has zero tracked upward references to application/scanner/inventory-runtime/enrichment/agent/observatory-identity/normalization/legacy-db namespaces, and no same-name semantic structs remain duplicated between `cortex-domain` and `cortex-storage-sqlite`. +- Donor DB inventory comparison finds all 83 donor Rust files represented in storage; the only donor-relative paths absent are the renamed observatory model test and `graph_confidence.rs` plus its tests, which intentionally moved to `cortex-domain`. +- No `dead_code` suppression remains anywhere in the extracted storage crate. +- Critical integration pins on the rebased head are `futures 0.3.34`, `h2 0.4.16`, `rusqlite 0.40.2`, and `r2d2_sqlite 0.35.0`. diff --git a/docs/cortex-extraction/SPEC.md b/docs/cortex-extraction/SPEC.md index 32adf129..eeda36d8 100644 --- a/docs/cortex-extraction/SPEC.md +++ b/docs/cortex-extraction/SPEC.md @@ -1,7 +1,7 @@ --- title: "Cortex Shared-Crate Extraction Specification" created: 2026-08-17 -updated: 2026-08-17 +updated: 2026-08-18 doc_type: "spec" status: "active" owner: "soma" @@ -10,7 +10,7 @@ audience: - "agents" scope: "family" source_of_truth: true -last_reviewed: "2026-08-17" +last_reviewed: "2026-08-18" --- # Cortex Shared-Crate Extraction Specification @@ -52,13 +52,14 @@ apps/cortex | +--> cortex-domain +--> cortex-ingest-core + +--> cortex-inventory (pure snapshot contract) cortex-ingest --------> cortex-domain + cortex-ingest-core + ports cortex-agent ---------> cortex-domain + cortex-ingest-core + ports - cortex-inventory -----> cortex-domain + cortex-inventory -----> standalone snapshot contract in Wave 2; collector behavior remains Wave 4 cortex-observatory ---> cortex-domain - cortex-domain --------> general shared/external primitives only - cortex-ingest-core ---> serde_json + sha2 only in wave 0 + cortex-domain --------> serde + serde_json + thiserror + chrono (pure time policy) + cortex-ingest-core ---> serde + serde_json + sha2 ``` The exact dependency graph may become even narrower as ports are introduced. It @@ -79,7 +80,9 @@ configuration. Own stable semantic types and invariants shared across Cortex capabilities. It may define traits/ports needed to invert dependencies, but it does not know SQLite row structs, Axum extractors, RMCP protocol DTOs, filesystem layout, or -process globals. +process globals. Wave 1 classifies all 255 donor public model declarations and +extracts all 65 semantic contracts. Database-row mappings remain owned by the +SQLite adapter, and transport/runtime projections remain outside the domain API. ### cortex-storage-sqlite diff --git a/docs/cortex-extraction/VERIFICATION.md b/docs/cortex-extraction/VERIFICATION.md index 2020fcf9..1d68652c 100644 --- a/docs/cortex-extraction/VERIFICATION.md +++ b/docs/cortex-extraction/VERIFICATION.md @@ -1,7 +1,7 @@ --- title: "Cortex Extraction Verification" created: 2026-08-17 -updated: 2026-08-17 +updated: 2026-08-18 doc_type: "guide" status: "active" owner: "soma" @@ -10,7 +10,7 @@ audience: - "agents" scope: "family" source_of_truth: true -last_reviewed: "2026-08-17" +last_reviewed: "2026-08-18" --- # Cortex Extraction Verification @@ -36,6 +36,51 @@ The crate test command includes the donor unit tests and `tests/public_api.rs`, which acts as an external consumer and proves no Cortex product/runtime dependency is needed. +## Wave 1 domain gates + +Run the domain lane with strict lint/doc policy and the repository source +hygiene/test-layout contracts: + +```bash +cargo fmt --all --check +cargo clippy -p cortex-domain --all-targets --all-features -- -D warnings +cargo test -p cortex-domain --all-features +RUSTDOCFLAGS="-D warnings" cargo doc -p cortex-domain --no-deps --all-features +cargo xtask check-test-siblings +cargo xtask run-ascii-check +``` + +The model-ownership review must also prove that every public donor declaration +under `src/app/models/*.rs` is classified exactly once and that every type +classified as a semantic contract is represented in `cortex-domain`. The Wave 1 +baseline is 255 public declarations, including 65 semantic contracts. Source and +manifest review must show no dependency on database/pool, HTTP/MCP, auth, +scanner, receiver, file-tail, config, or product runtime types. + +## Wave 2 SQLite storage gates + +The storage lane must prove both donor database behavior and the reusable public +boundary. Run: + +```bash +cargo fmt --all --check +cargo check -p cortex-storage-sqlite --all-features +cargo clippy -p cortex-storage-sqlite --all-targets --all-features -- -D warnings +cargo test -p cortex-storage-sqlite --all-features --no-fail-fast +RUSTDOCFLAGS="-D warnings" cargo doc -p cortex-storage-sqlite --no-deps --all-features +cargo test -p cortex-domain -p cortex-ingest-core -p cortex-inventory +cargo xtask check-test-siblings +cargo xtask check-architecture +``` + +The SQLite suite includes the pinned schema-43 migration fixture and must finish +at `KNOWN_SCHEMA_VERSION = 47`. `tests/public_api.rs` initializes a temporary +database and round-trips a log using only public storage APIs. Source review must +show no upward references to Cortex app, scanner, inventory runtime, enrichment, +agent, or observatory implementation namespaces. The integration adaptation to +Soma uses `rusqlite 0.40` with `r2d2_sqlite 0.35`; donor behavior, migration +ordering, PRAGMAs, and single-writer coordination remain parity requirements. + ## Workspace gates ADR 0010 defines the backend integration truth: diff --git a/packages/soma-rmcp/README.md b/packages/soma-rmcp/README.md index cf9c4003..154a1775 100644 --- a/packages/soma-rmcp/README.md +++ b/packages/soma-rmcp/README.md @@ -666,12 +666,12 @@ just validate-plugin ### Workspace layout -41 cargo members: +42 cargo members: | Path | Contents | |---|---| | `crates/soma/*` | Product code for this server — domain, application, config, client, api, cli, mcp, runtime, integrations, palette, web, test-support | -| `crates/shared/*` | Reusable engine crates other servers consume — auth, mcp (client/server/proxy/gateway), provider-core, provider-adapters, http-api, http-server, observability, openapi, self-update, traces, codemode, cli-core, and namespaced reusable families such as `cortex/ingest-core` | +| `crates/shared/*` | Reusable engine crates other servers consume — auth, mcp (client/server/proxy/gateway), provider-core, provider-adapters, http-api, http-server, observability, openapi, self-update, traces, codemode, cli-core, and namespaced reusable families such as `cortex/domain` and `cortex/ingest-core` | | `crates/integrations/*` | Upstream service bridges — `gotify`, `unifi` | | `apps/soma` | The `soma` binary and its integration tests. **The only cargo member under `apps/`** — `apps/web` (Next.js) and `apps/palette` (assets) are not Rust crates. | | `packages/python` | pyo3 Python provider platform (`abi3-py311`) | diff --git a/xtask/src/patterns/util.rs b/xtask/src/patterns/util.rs index 599943ce..24b79d79 100644 --- a/xtask/src/patterns/util.rs +++ b/xtask/src/patterns/util.rs @@ -29,6 +29,21 @@ pub(super) fn size_limit(path: &Path) -> Option { // keep it visible as a warning without blocking unrelated CI gates. return Some(700); } + let cortex_sqlite_parity_limit = match path.to_string_lossy().as_ref() { + // Wave 2 preserves these donor SQLite modules as behavior-parity units + // before the Wave 8 decomposition. Keep each exact path warning-visible + // without granting the crate or directory a blanket exemption. + "crates/shared/cortex/storage-sqlite/src/analytics.rs" => Some(750), + "crates/shared/cortex/storage-sqlite/src/graph.rs" => Some(1_400), + "crates/shared/cortex/storage-sqlite/src/graph_inventory.rs" => Some(400), + "crates/shared/cortex/storage-sqlite/src/maintenance.rs" => Some(550), + "crates/shared/cortex/storage-sqlite/src/pool.rs" => Some(1_600), + "crates/shared/cortex/storage-sqlite/src/queries.rs" => Some(1_600), + _ => None, + }; + if cortex_sqlite_parity_limit.is_some() { + return cortex_sqlite_parity_limit; + } if path == Path::new("crates/soma/application/src/provider_registry.rs") { // Provider registration and dispatch is intentionally centralized while // the drop-in provider contract is settling. Keep it warning-visible. @@ -288,6 +303,33 @@ mod tests { ))); } + #[test] + fn cortex_sqlite_parity_modules_have_narrow_transitional_budgets() { + let cases = [ + ("crates/shared/cortex/storage-sqlite/src/analytics.rs", 750), + ("crates/shared/cortex/storage-sqlite/src/graph.rs", 1_400), + ( + "crates/shared/cortex/storage-sqlite/src/graph_inventory.rs", + 400, + ), + ( + "crates/shared/cortex/storage-sqlite/src/maintenance.rs", + 550, + ), + ("crates/shared/cortex/storage-sqlite/src/pool.rs", 1_600), + ("crates/shared/cortex/storage-sqlite/src/queries.rs", 1_600), + ]; + for (path, limit) in cases { + assert_eq!(size_limit(Path::new(path)), Some(limit), "{path}"); + } + assert_eq!( + size_limit(Path::new( + "crates/shared/cortex/storage-sqlite/src/heartbeat.rs" + )), + Some(350) + ); + } + #[test] fn transitional_xtask_modules_warn_before_hard_failing() { assert_eq!(size_limit(Path::new("xtask/src/scaffold.rs")), Some(600)); diff --git a/xtask/src/test_siblings.rs b/xtask/src/test_siblings.rs index b0f6992d..09f2ed1b 100644 --- a/xtask/src/test_siblings.rs +++ b/xtask/src/test_siblings.rs @@ -12,6 +12,10 @@ const ORPHAN_EXEMPT: &[&str] = &[ "mcp_tests.rs", "http_oauth_stubs_tests.rs", "live_servers_tests.rs", + // `queries.rs` intentionally has two split suites: `queries_tests.rs` is + // the canonical sibling and this graph-focused module carries the larger + // traversal/fan-out cases without creating a fake `queries_graph.rs`. + "queries_graph_tests.rs", ]; pub(crate) fn check() -> Result<()> { @@ -142,7 +146,10 @@ fn filename(path: &Path) -> String { const CHECKED_SRC_ROOTS: &[&str] = &[ "apps/soma/src", "crates/shared/codemode/src", + "crates/shared/cortex/domain/src", "crates/shared/cortex/ingest-core/src", + "crates/shared/cortex/inventory/src", + "crates/shared/cortex/storage-sqlite/src", "crates/shared/incus-client/src", "crates/shared/mcp/client/src", "crates/shared/mcp/gateway/src",