feat(copilot): OTel cache-token parsing (#477) + maintainer review fixes

docs/providers/copilot.md

@@ -8,18 +8,38 @@ GitHub Copilot Chat (CLI and VS Code extension transcripts).
 
 ## Where it reads from
 
-Two locations. Both are walked on every run; results merge.
+Two JSONL locations plus an optional OpenTelemetry SQLite source (see below). All
+discovered sources are walked on every run; results merge and dedupe.
 
 1. **Legacy CLI sessions:** `~/.copilot/session-state/`
 2. **VS Code transcripts:** `~/Library/Application Support/Code/User/workspaceStorage/<hash>/GitHub.copilot-chat/transcripts/` and equivalents on Windows / Linux
+3. **OTel SQLite store:** VS Code Copilot Chat's `agent-traces.db` (see the OTel section). Preferred when present because it carries full input / output / cache token counts; the JSONL sources only record output tokens.
 
 ## Storage format
 
-JSONL in both locations, but the schemas differ. The parser switches by detecting which schema the first event uses (`copilot.ts:83-159` for legacy, `copilot.ts:215-293` for transcripts).
+JSONL in the first two locations (schemas differ; the parser switches by detecting which schema the first event uses), and a SQLite DB for the OTel source.
+
+## OpenTelemetry (OTel) source
+
+When VS Code Copilot Chat's `agent-traces.db` exists, the parser reads per-LLM-call token
+breakdowns (input, output, cache-read, cache-creation) from it, which the JSONL sources do
+not record. Discovery is skipped with `CODEBURN_COPILOT_DISABLE_OTEL=1`, and the DB path
+can be overridden with `CODEBURN_COPILOT_OTEL_DB`.
+
+- **Requires Node 22+.** The OTel source uses the built-in `node:sqlite` module (the same
+  backend as Cursor / OpenCode). On Node 20, or if the DB is missing / locked / corrupt /
+  wrong-schema, OTel is skipped and the JSONL/transcript sources are used as a fallback.
+- **Durable cache (monotonic totals).** Copilot is marked `durableSources`: OTel-derived
+  cache entries are never evicted when VS Code prunes old spans from the DB, so
+  month-to-date totals do not drop as the DB rotates. Entries age out after 90 days.
+- **Upgrade note.** The first run after upgrading to the OTel version bumps the copilot
+  parse version, which discards the prior copilot cache. Spans already pruned from the DB
+  before the upgrade cannot be recovered, so monotonicity starts from the upgrade point,
+  not retroactively.
 
 ## Caching
 
-None at the provider level.
+None for the JSONL sources. The OTel source uses a durable cache (see above).
 
 ## Deduplication
 

src/parser.ts

@@ -16,12 +16,13 @@ import {
   type SessionCache,
   cleanupOrphanedTempFiles,
   computeEnvFingerprint,
+  DURABLE_PROVIDER_NAMES,
   fingerprintFile,
   loadCache,
   reconcileFile,
   saveCache,
 } from './session-cache.js'
-import type { ParsedProviderCall } from './providers/types.js'
+import type { ParsedProviderCall, SessionSource } from './providers/types.js'
 import type {
   AssistantMessageContent,
   ClassifiedTurn,
@@ -1946,7 +1947,7 @@ function warnProviderParseFailure(providerName: string, sourcePath: string, err:
 
 async function parseProviderSources(
   providerName: string,
-  sources: Array<{ path: string; project: string }>,
+  sources: SessionSource[],
   seenKeys: Set<string>,
   diskCache: SessionCache,
   dateRange?: DateRange,
@@ -1957,8 +1958,8 @@ async function parseProviderSources(
   const section = getOrCreateProviderSection(diskCache, providerName)
   const allDiscoveredFiles = new Set<string>()
 
-  type SourceInfo = { source: { path: string; project: string }; fp: NonNullable<Awaited<ReturnType<typeof fingerprintFile>>> }
-  const unchangedSources: Array<{ source: { path: string; project: string }; cached: CachedFile }> = []
+  type SourceInfo = { source: SessionSource; fp: NonNullable<Awaited<ReturnType<typeof fingerprintFile>>> }
+  const unchangedSources: Array<{ source: SessionSource; cached: CachedFile }> = []
   const changedSources: SourceInfo[] = []
 
   for (const source of sources) {
@@ -2001,20 +2002,27 @@ async function parseProviderSources(
 
   // Parse changed files, update cache
   let didParse = false
+  // Track which paths have already been cleared this pass so that subsequent
+  // sources sharing the same path (e.g. multiple OTel conversations from one
+  // agent-traces.db) can accumulate via the merge logic below rather than
+  // being wiped on every iteration.
+  const clearedPaths = new Set<string>()
   try {
     for (const { source, fp } of changedSources) {
       if (dateRange) {
         if (fp.mtimeMs < dateRange.start.getTime()) continue
       }
 
-      // Clear stale entry before parse — if parse fails, file is excluded
-      delete section.files[source.path]
+      // Clear stale entry before parse — but only once per path so that
+      // multiple sources mapping to the same file path can merge their turns.
+      // Durable providers (e.g. copilot OTel) never clear existing entries so
+      // that pruned-away data is preserved for monotonic monthly totals.
+      if (!provider.durableSources && !clearedPaths.has(source.path)) {
+        delete section.files[source.path]
+        clearedPaths.add(source.path)
+      }
 
-      const parser = provider.createSessionParser(
-        { path: source.path, project: source.project, provider: providerName },
-        parserDedup,
-        dateRange,
-      )
+      const parser = provider.createSessionParser(source, parserDedup, dateRange)
 
       try {
         const providerCalls: ParsedProviderCall[] = []
@@ -2023,7 +2031,38 @@ async function parseProviderSources(
         }
         const canonicalCalls = await Promise.all(providerCalls.map(canonicalizeProviderCallProject))
         const turns = providerCallsToCachedTurns(canonicalCalls)
-        section.files[source.path] = { fingerprint: fp, mcpInventory: [], turns }
+
+        // Store/merge parsed turns into the cache.
+        // Durable providers use a union-by-deduplicationKey merge: existing turns
+        // are NEVER deleted (preserves data for spans pruned from the DB), and
+        // only turns whose dedup keys are not already cached are appended.
+        // Non-durable providers keep the original overwrite-or-append behaviour.
+        if (provider.durableSources) {
+          const existingEntry = section.files[source.path]
+          if (existingEntry) {
+            const existingKeys = new Set(
+              existingEntry.turns.flatMap(t => t.calls.map(c => c.deduplicationKey))
+            )
+            const newTurns = turns.filter(t =>
+              t.calls.every(c => !existingKeys.has(c.deduplicationKey))
+            )
+            existingEntry.turns = [...existingEntry.turns, ...newTurns]
+            existingEntry.fingerprint = fp
+          } else {
+            section.files[source.path] = { fingerprint: fp, mcpInventory: [], turns }
+          }
+        } else {
+          // Non-durable: overwrite (clearedPaths already deleted stale entry above)
+          // or append when multiple sources map to the same path. NOTE: the append
+          // path assumes discoverSessions yields a unique path per source, which all
+          // current providers do; it only fires for same-path multi-source providers.
+          const existingCacheEntry = section.files[source.path]
+          if (existingCacheEntry) {
+            existingCacheEntry.turns = [...existingCacheEntry.turns, ...turns]
+          } else {
+            section.files[source.path] = { fingerprint: fp, mcpInventory: [], turns }
+          }
+        }
         didParse = true
         ;(diskCache as { _dirty?: boolean })._dirty = true
       } catch (err) {
@@ -2051,7 +2090,14 @@ async function parseProviderSources(
     }
   }
 
-  if (sources.length > 0) {
+  // Stamp the durable flag into the cache section so the orphan-bootstrap in
+  // parseAllSessions can fast-check without a getProvider() round-trip.
+  if (provider.durableSources && !section.durable) {
+    section.durable = true
+    ;(diskCache as { _dirty?: boolean })._dirty = true
+  }
+
+  if (sources.length > 0 && !provider.durableSources) {
     for (const cachedPath of Object.keys(section.files)) {
       if (!allDiscoveredFiles.has(cachedPath)) {
         delete section.files[cachedPath]
@@ -2060,6 +2106,23 @@ async function parseProviderSources(
     }
   }
 
+  // 90-day age-out for durable providers: remove entries whose newest call is
+  // older than 90 days so the cache doesn't grow unboundedly over time.
+  if (provider.durableSources) {
+    const cutoffMs = Date.now() - 90 * 24 * 60 * 60 * 1000
+    for (const [cachedPath, cachedFile] of Object.entries(section.files)) {
+      const newestTs = cachedFile.turns
+        .flatMap(t => t.calls)
+        .map(c => new Date(c.timestamp).getTime())
+        .filter(ts => !isNaN(ts))
+        .reduce((max, ts) => Math.max(max, ts), 0)
+      if (newestTs > 0 && newestTs < cutoffMs) {
+        delete section.files[cachedPath]
+        ;(diskCache as { _dirty?: boolean })._dirty = true
+      }
+    }
+  }
+
   // Query-time: derive SessionSummary from all cached turns.
   // Uses seenKeys (shared across providers) for cross-provider dedup.
   const sessionMap = new Map<string, { project: string; projectPath?: string; turns: ClassifiedTurn[] }>()
@@ -2097,6 +2160,43 @@ async function parseProviderSources(
     }
   }
 
+  // Second pass: durable orphans — cache entries for paths that are no longer
+  // discovered (e.g. OTel conversations pruned from the DB). Their turns are
+  // counted here so the monthly total never drops.
+  if (provider.durableSources) {
+    for (const [cachedPath, cachedFile] of Object.entries(section.files)) {
+      if (allDiscoveredFiles.has(cachedPath)) continue  // already counted above
+
+      for (const turn of cachedFile.turns) {
+        const hasDup = turn.calls.some(c => seenKeys.has(c.deduplicationKey))
+        if (hasDup) continue
+
+        for (const c of turn.calls) seenKeys.add(c.deduplicationKey)
+
+        if (dateRange) {
+          const callTs = turn.calls[0]?.timestamp
+          if (!callTs) continue
+          const ts = new Date(callTs)
+          if (ts < dateRange.start || ts > dateRange.end) continue
+        }
+
+        const classified = cachedTurnToClassified(turn)
+        const project = turn.calls[0]?.project ?? providerName
+        const key = `${providerName}:${turn.sessionId}:${project}`
+
+        const existingEntry = sessionMap.get(key)
+        if (existingEntry) {
+          existingEntry.turns.push(classified)
+          if (!existingEntry.projectPath && turn.calls[0]?.projectPath) {
+            existingEntry.projectPath = turn.calls[0]!.projectPath
+          }
+        } else {
+          sessionMap.set(key, { project, projectPath: turn.calls[0]?.projectPath, turns: [classified] })
+        }
+      }
+    }
+  }
+
   const projectMap = new Map<string, { projectPath?: string; sessions: SessionSummary[] }>()
   for (const [key, { project, projectPath, turns }] of sessionMap) {
     const sessionId = key.split(':')[1] ?? key
@@ -2246,10 +2346,10 @@ export async function parseAllSessions(dateRange?: DateRange, providerFilter?: s
   const claudeDirs = claudeSources.map(s => ({ path: s.path, name: s.project }))
   const claudeProjects = await scanProjectDirs(claudeDirs, seenMsgIds, diskCache, dateRange)
 
-  const providerGroups = new Map<string, Array<{ path: string; project: string }>>()
+  const providerGroups = new Map<string, SessionSource[]>()
   for (const source of nonClaudeSources) {
     const existing = providerGroups.get(source.provider) ?? []
-    existing.push({ path: source.path, project: source.project })
+    existing.push(source)
     providerGroups.set(source.provider, existing)
   }
 
@@ -2259,6 +2359,26 @@ export async function parseAllSessions(dateRange?: DateRange, providerFilter?: s
     otherProjects.push(...projects)
   }
 
+  // Durable providers with cached data but NO discovered sources (all files pruned
+  // by VS Code / the external tool) still need their orphan pass to run so the
+  // monthly total never drops. Call parseProviderSources with empty sources for
+  // any such provider found in the disk cache.
+  const processedProviders = new Set(providerGroups.keys())
+  for (const providerName of Object.keys(diskCache.providers)) {
+    if (processedProviders.has(providerName)) continue
+    // Skip if filtered to a different provider
+    if (providerFilter && providerFilter !== 'all' && providerFilter !== providerName) continue
+    const section = diskCache.providers[providerName]
+    if (!section || Object.keys(section.files).length === 0) continue
+    // Use the persisted durable flag (set by parseProviderSources when it first
+    // processes a durableSources provider) OR the static DURABLE_PROVIDER_NAMES
+    // constant — both checks are O(1) and avoid a getProvider() dynamic-import
+    // round-trip for every unprocessed provider in the disk cache.
+    if (!section.durable && !DURABLE_PROVIDER_NAMES.has(providerName)) continue
+    const projects = await parseProviderSources(providerName, [], seenKeys, diskCache, dateRange)
+    otherProjects.push(...projects)
+  }
+
   if ((diskCache as { _dirty?: boolean })._dirty) {
     try { await saveCache(diskCache) } catch {}
   }