diff --git a/CLAUDE.md b/CLAUDE.md new file mode 100644 index 0000000..3a66925 --- /dev/null +++ b/CLAUDE.md @@ -0,0 +1,645 @@ +# CLAUDE.md - LLM Legal Council + +This document provides guidance for AI assistants working with the LLM Legal Council codebase. + +## Project Overview + +LLM Legal Council is a multi-model deliberation system for legal analysis, critique, and risk assessment. Based on [Andrej Karpathy's llm-council pattern](https://github.com/karpathy/llm-council), adapted for legal practice. + +**Key Principle**: This is a **deliberation and critique** tool, NOT a document drafting system. + +### Appropriate Uses +- Issue spotting on draft motions/briefs +- Risk assessment for litigation strategy +- Identifying weaknesses in legal arguments +- Stress testing case theories +- Devil's advocate analysis +- Evaluating settlement positions + +### Not For +- Drafting documents, briefs, or court filings +- Writing client correspondence +- Creating final work product + +## Architecture + +### Three-Stage Deliberation Process + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ STAGE 1 │ +│ Independent Analysis │ +│ Each model analyzes the query independently (anonymized) │ +├─────────────────────────────────────────────────────────────────┤ +│ STAGE 2 │ +│ Peer Review (Blind) │ +│ Each model ranks all responses without knowing authorship │ +├─────────────────────────────────────────────────────────────────┤ +│ STAGE 3 │ +│ Chairman Synthesis │ +│ Highest-ranked analyst synthesizes consensus + preserves │ +│ dissent (algorithmic selection or user override) │ +└─────────────────────────────────────────────────────────────────┘ +``` + +### Source Code Structure + +``` +src/ +├── index.ts # Library entry point & exports +├── cli.ts # Command-line interface +├── config.ts # Configuration loading (Board Seat architecture) +├── types.ts # TypeScript type definitions (comprehensive) +├── schemas.ts # Zod validation schemas for JSON mode +├── project.ts # Project file handling +├── skills.ts # Skill loader for legal reasoning skills +├── usage.ts # Token/cost tracking +└── council/ + ├── orchestrator.ts # Main deliberation logic + ├── openrouter.ts # OpenRouter API client + └── audit.ts # Audit trail & chairman selection + +skills/ # Legal reasoning methodology (markdown) +├── legal-reasoning-foundation.md +├── legal-research.md +├── verification-before-assertion.md +├── adversarial-examiner.md +└── citation-integrity.md + +tools/ +├── definitions.ts # Zod schemas for tools +└── implementations.ts # Tool implementations (RAG, CourtListener, etc.) +``` + +## Development Commands + +```bash +# Install dependencies +npm install + +# Development (watch mode) +npm run dev + +# Type checking +npm run typecheck + +# Build (TypeScript to dist/) +npm run build + +# Run tests +npm test + +# Run tests in watch mode +npm run test:watch + +# Lint +npm run lint + +# Run CLI directly +npx tsx src/cli.ts "Your legal question here" +npx tsx src/cli.ts --project ./examples/ny-trust-estates.json "Your question" +npx tsx src/cli.ts --interactive +``` + +## Configuration + +### Environment Variables (Required) + +```bash +# API Key (required) +OPENROUTER_API_KEY=your_key_here + +# Council Models (minimum 2 required) +COUNCIL_MODEL_1=anthropic/claude-sonnet-4 +COUNCIL_MODEL_2=openai/gpt-4o +COUNCIL_MODEL_3=google/gemini-pro-1.5 +``` + +### Optional Configuration + +```bash +# Override algorithmic chairman selection +CHAIRMAN_MODEL=anthropic/claude-sonnet-4 + +# Default jurisdiction +DEFAULT_JURISDICTION=NY + +# Concurrency limit (default: 3) +COUNCIL_CONCURRENCY_LIMIT=3 + +# JSON fallback for models without native JSON schema support +COUNCIL_JSON_FALLBACK_MODELS=model/name-1,model/name-2 +``` + +### Tool API Keys (Optional) + +```bash +# RAG Worker (Cloudflare) +LEGAL_KNOWLEDGE_WORKER_URL=https://legal-knowledge-worker.YOUR_SUBDOMAIN.workers.dev +LEGAL_KNOWLEDGE_WORKER_TOKEN=your_worker_api_token + +# CourtListener (case law) +COURT_LISTENER_API_KEY=your_api_key + +# Perplexity (web search) +PERPLEXITY_API_KEY=your_api_key +``` + +## Key Concepts + +### Board Seat Architecture + +The system defines council "seats" at the code level, but which models occupy those seats is determined entirely at runtime via environment variables. There are **NO hardcoded model defaults**. + +```typescript +// config.ts - loads models from environment +const models: string[] = []; +for (let i = 1; i <= 10; i++) { + const modelId = process.env[`COUNCIL_MODEL_${i}`]; + if (modelId) models.push(modelId.trim()); +} +``` + +### Skills System + +Skills provide **methodology**, not knowledge. They teach models HOW to think about legal problems: + +| Skill | Purpose | +|-------|---------| +| `legal-reasoning-foundation` | IRAC, syllogistic reasoning, issue identification | +| `legal-research` | Research methodology, source evaluation | +| `verification-before-assertion` | Verification discipline, confidence calibration | +| `adversarial-examiner` | Threshold checking, opposing counsel simulation | +| `citation-integrity` | Anti-hallucination discipline, citation verification | + +Skills are loaded via `loadSkills()` in `src/skills.ts` and injected into system prompts. + +### Query Types + +The system supports different query types with specialized directives: + +- `issue-spotting` - Threshold blockers, procedural defects +- `risk-assessment` - Likelihood/impact calibration +- `weakness-identification` - Exploitable vulnerabilities +- `strategy-evaluation` - Strategy vs alternatives +- `stress-test` - Opposing counsel attack simulation +- `devils-advocate` - Argue against the position +- `settlement-evaluation` - Litigation risk vs settlement +- `brainstorm` - Generate multiple approaches +- `general-deliberation` - Default mode + +### Chairman Selection + +By default, the highest-ranked analyst from Stage 2 becomes chairman (algorithmic selection). Users can override this via `CHAIRMAN_MODEL` environment variable. + +### Audit Trail + +Every deliberation includes comprehensive audit data: + +- Chairman selection rationale +- Per-model metrics (latency, tokens, retries) +- Ranking consensus analysis +- Anomaly detection (confidence mismatches, outliers) +- Process integrity score + +## Type System + +### Key Interfaces + +```typescript +// Query input +interface CouncilQuery { + query: string; + queryType: CouncilQueryType; + jurisdiction?: string; + practiceArea?: string; + context?: Record; + workProduct?: string; // For critique tasks +} + +// Output structure +interface CouncilDeliberation { + consensus: ConsensusResult; + issuesIdentified: IdentifiedIssue[]; + riskAssessment: CalibratedRisk; + dissent: DissentingView[]; // Preserved, not flattened + weaknessesFound: IdentifiedWeakness[]; + openQuestions: string[]; + actionItems: ActionItem[]; + _audit?: CouncilAudit; + _usage?: UsageSummary; +} +``` + +### Zod Schemas + +All LLM responses use Zod schemas for validation (`src/schemas.ts`): + +- `Stage1AnalysisSchema` - Individual analysis structure +- `Stage2ReviewSchema` - Peer review evaluations +- `Stage3SynthesisSchema` - Chairman synthesis output + +## Project System + +Projects customize council behavior without code changes: + +```json +{ + "id": "ny-commercial-litigation", + "name": "NY Commercial Litigation", + "instructions": "Apply New York law...", + "chairmanInstructions": "Lead with jurisdictional compliance...", + "defaultJurisdiction": "NY", + "files": [ + { + "path": "./complaint.pdf", + "inclusion": "full" + } + ] +} +``` + +Load with: `npx tsx src/cli.ts --project ./project.json "Your question"` + +## Code Conventions + +### TypeScript +- Strict mode enabled +- ES2022 target with NodeNext modules +- Use `.js` extensions in imports (ESM requirement) +- Zod for runtime validation + +### Error Handling +- Custom error classes: `ConfigurationError`, `ProjectError`, `CouncilQuorumError`, `OpenRouterError` +- Tools fail closed (return error messages, don't throw) +- Quorum checks after Stage 1 and Stage 2 + +### JSON Mode +- All model calls use JSON schema mode when supported +- Fallback to text mode with JSON extraction for models in `COUNCIL_JSON_FALLBACK_MODELS` +- `queryModelWithFallback()` handles routing automatically + +## Testing + +```bash +npm test # Run all tests +npm run test:watch # Watch mode +``` + +Tests use Vitest. Test files follow `*.test.ts` convention. + +## Cloudflare Worker (RAG) + +The `legal-knowledge-worker/` directory contains a Cloudflare Worker for document retrieval: + +```bash +# Deploy +cd legal-knowledge-worker +npx wrangler vectorize create legal-council-index --dimensions=768 --metric=cosine +npx wrangler secret put API_TOKEN +npx wrangler deploy +``` + +## Common Tasks + +### Adding a New Query Type + +1. Add type to `CouncilQueryType` in `src/types.ts` +2. Add directive in `getQueryTypeDirective()` in `src/council/orchestrator.ts` +3. Update `isAppropriateForCouncil()` if needed + +### Adding a New Tool + +1. Define Zod schema in `src/tools/definitions.ts` +2. Implement execution in `src/tools/implementations.ts` +3. Add to `ALL_TOOLS` array + +### Adding a New Skill + +1. Create markdown file in `skills/` directory +2. Add to `CORE_SKILLS` array in `src/skills.ts` + +## Version + +Current version: **0.6.0** (see CHANGELOG.md for release notes) + +--- + +## v0.8 Development Roadmap + +This section documents planned improvements for the GUI version (v0.7-v0.8). + +### Planned Architecture Changes + +**Target Stack:** +- **Backend:** Node.js + Hono server (NOT Cloudflare Workers - deliberation exceeds 30-second limit) +- **Frontend:** React +- **Model Routing:** OpenRouter API +- **Storage:** Cloudflare D1 (SQLite) + R2 (file storage) +- **RAG:** Gemini File Search for document Q&A + +**Model Configuration:** +| Seat | Model | Role | +|------|-------|------| +| A | Claude Sonnet 4.5 | Lead Analyst | +| B | GPT-5.2 | Red Team | +| C | Gemini 3 Pro | Judge | +| D (optional) | Grok 4.1 | Contrarian | + +**Note:** GPT-4o is explicitly excluded. + +### Priority 1: Bug Fixes + +#### 1.1 RAG Worker Embedding Batching +**Problem:** Worker calls `env.AI.run()` for each text chunk individually, exceeding Cloudflare's subrequest limits. + +**Fix:** Batch operations: +- Embeddings: `EMBEDDING_BATCH_SIZE = 100` +- Vectorize: `VECTORIZE_BATCH_SIZE = 100` +- D1: `D1_BATCH_SIZE = 100` + +### Priority 2: GUI Improvements + +#### Layout Redesign +``` +┌─────────────────────────────────────────────────────────────────────────────┐ +│ ☰ LLM Legal Council 📎 │ +├─────────────────────────────────────────────────────────────────────────────┤ +│ ┌─────────────────┐ ┌─────────────────┐ ┌─────────────────┐ ┌──────────┐ │ +│ │ GROK 4.1 │ │ GEMINI 3 │ │ GPT 5.2 │ │ CLAUDE │ │ +│ │ (brand color) │ │ (brand color) │ │ (brand color) │ │ (amber) │ │ +│ │ [streaming] │ │ [streaming] │ │ [streaming] │ │[streaming│ │ +│ └─────────────────┘ └─────────────────┘ └─────────────────┘ └──────────┘ │ +│ [Critiques ▼] [Synthesis ▼] │ +│ ┌─────────────────────────────────────────────────────────────────────┐ │ +│ │ 📎 Ask the council... ➤ │ │ +│ └─────────────────────────────────────────────────────────────────────┘ │ +└─────────────────────────────────────────────────────────────────────────────┘ +``` + +**Key changes:** +- Remove dedicated upload panel → paperclip icon + drag-and-drop +- ☰ (left) → History sidebar slides in +- 📎 (right) → Documents sidebar slides in + +#### Brand Colors +| Model | Brand Color | Gradient | +|-------|-------------|----------| +| Claude | `#D97706` (amber) | `linear-gradient(135deg, #FEF3C7 0%, #FDE68A 100%)` | +| GPT | `#10A37F` (green) | `linear-gradient(135deg, #D1FAE5 0%, #A7F3D0 100%)` | +| Gemini | `#4285F4` (blue) | `linear-gradient(135deg, #DBEAFE 0%, #BFDBFE 100%)` | + +### Priority 3: Post-Deliberation Chat + +Enable back-and-forth conversation with all four models after deliberation. + +**Architecture (Broadcast):** +1. User sends message +2. All 4 models receive in parallel with full context +3. All 4 respond (streamed) +4. User sees 4 responses, can follow up + +**Context per model:** `system prompt + original query + documents + its Stage 1 response + relevant critiques + synthesis + chat history + new question` + +### Priority 4: Conversation Persistence + +**New D1 Tables:** +```sql +CREATE TABLE deliberations ( + id TEXT PRIMARY KEY, + query TEXT NOT NULL, + documents TEXT, + synthesis TEXT, + created_at DATETIME DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE deliberation_responses ( + id TEXT PRIMARY KEY, + deliberation_id TEXT NOT NULL, + model_id TEXT NOT NULL, + content TEXT NOT NULL, + is_chairman BOOLEAN DEFAULT FALSE +); + +CREATE TABLE chat_messages ( + id TEXT PRIMARY KEY, + deliberation_id TEXT NOT NULL, + role TEXT NOT NULL, + content TEXT NOT NULL +); +``` + +### Priority 5: Project System Enhancement + +**Per-Seat Customization:** +- `system_prompt_prefix/suffix` +- `persona` - Role description +- `temperature` - Model-specific setting + +**New D1 Tables:** +```sql +CREATE TABLE projects ( + id TEXT PRIMARY KEY, + name TEXT NOT NULL, + jurisdiction TEXT DEFAULT 'NY', + system_prompt TEXT, + default_mode TEXT DEFAULT 'parallel' +); + +CREATE TABLE seats ( + id TEXT PRIMARY KEY, + project_id TEXT NOT NULL, + seat_number INTEGER NOT NULL, + model TEXT NOT NULL, + role TEXT NOT NULL, + persona TEXT, + temperature REAL DEFAULT 0.7 +); +``` + +### Planned Tool Stack (6 tools) + +| Tool | Function | +|------|----------| +| `web_search` | Perplexity API | +| `courtlistener_search` | Case law research | +| `cornell_lii_search` | Statutes/CFR | +| `gemini_file_search` | Document Q&A with grounding | +| `extract_file_content` | PDF/DOCX text extraction | +| `project_knowledge_search` | Local skills/reference files | + +### Development Phases + +1. **Phase 1: Stabilization** - Fix batching, test tool iteration +2. **Phase 2: Chat Feature** - WebSocket/SSE chat, parallel model calls +3. **Phase 3: Persistence** - D1 tables, history sidebar +4. **Phase 4: GUI Polish** - New layout, brand colors, streaming +5. **Phase 5: Projects** - Project CRUD, per-seat customization + +### Known Constraints + +1. **Cloudflare Workers 30-second limit** - Use Node.js server for full deliberation +2. **Token budget** - Long conversations may need summarization +3. **OpenRouter latency** - ~50-100ms overhead per call + +--- + +## Model Selector + +The system includes a comprehensive model selector that fetches the latest models from OpenRouter on app startup. + +### CLI Commands + +```bash +# List all models (compact view, recommended only) +npm run models:list + +# Show recommended council configuration +npm run models:recommend + +# Interactive configuration wizard +npm run models:configure + +# Validate current .env configuration +npm run models:validate + +# Full model selector CLI +npx tsx src/model-selector-cli.ts list --verbose +npx tsx src/model-selector-cli.ts info anthropic/claude-sonnet-4 +``` + +### Model Analysis + +Each model includes council-specific analysis: +- **Council Score (1-10)** - Overall suitability for legal deliberation +- **Strengths/Weaknesses** - For legal reasoning tasks +- **Recommended Role** - lead-analyst, red-team, judge, contrarian, chairman +- **Chairman Suitability** - Whether suitable for synthesis role + +### Recommended Configuration + +| Seat | Model | Role | Score | +|------|-------|------|-------| +| A | Claude Sonnet 4 | Lead Analyst | 9/10 | +| B | GPT-5.2 | Red Team | 8/10 | +| C | Gemini 3 Pro | Judge | 8/10 | +| D | Grok 4.1 | Contrarian | 8/10 | +| Chairman | Claude Sonnet 4 | Synthesis | 9/10 | + +--- + +## Web & iOS App Roadmap (v1.0+) + +### Target Platforms + +| Platform | Technology | Status | +|----------|------------|--------| +| Web App | React + Hono (SSE) | Planned | +| iOS App | React Native or Swift UI | Planned | +| Desktop | Electron wrapper (current) | Active | + +### Web App Architecture + +``` +┌─────────────────────────────────────────────────────────────────┐ +│ FRONTEND │ +│ React SPA (Vite) - Hosted on Cloudflare Pages │ +├─────────────────────────────────────────────────────────────────┤ +│ BACKEND │ +│ Node.js + Hono - Hosted on VPS/Railway/Fly.io │ +│ (NOT Cloudflare Workers - deliberation > 30s) │ +├─────────────────────────────────────────────────────────────────┤ +│ STORAGE │ +│ Cloudflare D1 (SQLite) + R2 (files) │ +└─────────────────────────────────────────────────────────────────┘ +``` + +**Key Features:** +- SSE streaming for real-time model responses +- WebSocket for post-deliberation chat +- Authentication via Clerk or Auth.js +- PWA support for mobile web + +### iOS App Requirements + +**Display Modes:** +- Portrait mode: Stacked model cards (scrollable) +- Landscape mode: Side-by-side model cards (Hollywood Squares) +- Auto-rotation enabled + +**Responsive Layouts:** + +``` +PORTRAIT (iPhone) LANDSCAPE (iPad/iPhone) +┌─────────────────┐ ┌─────────┬─────────┐ +│ CLAUDE │ │ CLAUDE │ GPT │ +│ [streaming] │ │ │ │ +├─────────────────┤ ├─────────┼─────────┤ +│ GPT │ │ GEMINI │ GROK │ +│ [streaming] │ │ │ │ +├─────────────────┤ └─────────┴─────────┘ +│ GEMINI │ ┌─────────────────────┐ +│ [streaming] │ │ Ask the council... │ +├─────────────────┤ └─────────────────────┘ +│ GROK │ +│ [streaming] │ +├─────────────────┤ +│ Ask the council │ +└─────────────────┘ +``` + +**Technical Approach Options:** + +1. **React Native + Expo** (Recommended) + - Shared codebase with web + - Expo SDK for native features + - OTA updates without App Store review + +2. **Swift UI (Native)** + - Best iOS performance + - Separate codebase + - Full iOS feature access + +3. **Capacitor (Web wrapper)** + - Reuse React web code + - PWA-first approach + - Limited native features + +### Development Phases (Updated) + +1. **Phase 1: Web Foundation** (Current) + - Complete model selector ✓ + - Fix RAG worker batching + - Stabilize deliberation flow + +2. **Phase 2: Web App MVP** + - Hono server with SSE + - React frontend with streaming + - Basic authentication + +3. **Phase 3: Web Features** + - Post-deliberation chat + - History persistence (D1) + - Project management + +4. **Phase 4: iOS App** + - React Native setup + - Responsive layouts (portrait/landscape) + - Native file handling + +5. **Phase 5: Polish** + - Brand colors and theming + - Animations and transitions + - Performance optimization + +--- + +## Important Notes for AI Assistants + +1. **Never draft documents** - This system is for critique/deliberation only +2. **Preserve dissent** - Do not manufacture false consensus +3. **Verify before citing** - Use `[VERIFY]`, `[CITATION NEEDED]` placeholders +4. **Skills are methodology** - They teach HOW to analyze, not WHAT the law is +5. **Board Seat architecture** - No hardcoded model defaults; all configuration via env vars +6. **Minimum quorum is 2** - At least 2 council models must respond for valid deliberation +7. **GPT-4o excluded** - Do not use GPT-4o in model configurations diff --git a/package.json b/package.json index a7edc0e..e5be7ef 100644 --- a/package.json +++ b/package.json @@ -18,6 +18,11 @@ "dev": "tsx watch src/cli.ts", "build": "tsc", "start": "node dist/cli.js", + "models": "tsx src/model-selector-cli.ts", + "models:list": "tsx src/model-selector-cli.ts list --recommended --compact", + "models:recommend": "tsx src/model-selector-cli.ts recommend", + "models:configure": "tsx src/model-selector-cli.ts configure", + "models:validate": "tsx src/model-selector-cli.ts validate", "lint": "eslint src/", "typecheck": "tsc --noEmit", "test": "vitest run", diff --git a/src/index.ts b/src/index.ts index e25bb3c..ab12abf 100644 --- a/src/index.ts +++ b/src/index.ts @@ -109,3 +109,23 @@ export { selectChairman, ChairmanSelectionResult } from './council/audit.js'; + +// Model Selector +export { + ModelSelector, + createModelSelector, + getOpenRouterModels, + getCuratedModels, + clearModelCache, + CURATED_MODEL_ANALYSIS, + getRecommendedCouncilConfig +} from './models/index.js'; + +export type { + OpenRouterModel, + ModelPricing, + ModelCapabilities, + CouncilAnalysis, + CouncilConfiguration, + ModelSelectorOptions +} from './models/index.js'; diff --git a/src/model-selector-cli.ts b/src/model-selector-cli.ts new file mode 100644 index 0000000..6510548 --- /dev/null +++ b/src/model-selector-cli.ts @@ -0,0 +1,537 @@ +#!/usr/bin/env node +/** + * Model Selector CLI for LLM Legal Council + * + * Interactive tool for browsing, analyzing, and selecting council models. + * + * Usage: + * npx tsx src/model-selector-cli.ts list + * npx tsx src/model-selector-cli.ts list --recommended + * npx tsx src/model-selector-cli.ts list --chairman + * npx tsx src/model-selector-cli.ts info anthropic/claude-sonnet-4 + * npx tsx src/model-selector-cli.ts recommend + * npx tsx src/model-selector-cli.ts validate + * npx tsx src/model-selector-cli.ts configure --interactive + */ + +import 'dotenv/config'; +import chalk from 'chalk'; +import { program } from 'commander'; +import * as readline from 'readline'; + +import { + createModelSelector, + ModelSelector, + OpenRouterModel, + CouncilConfiguration, +} from './models/index.js'; + +// ============================================================================ +// DISPLAY HELPERS +// ============================================================================ + +function logSection(title: string): void { + console.log(); + console.log(chalk.cyan('═'.repeat(70))); + console.log(chalk.bold(` ${title}`)); + console.log(chalk.cyan('═'.repeat(70))); +} + +function formatPrice(model: OpenRouterModel): string { + const prompt = model.pricing.promptPerMillion; + const completion = model.pricing.completionPerMillion; + + if (prompt === 0 && completion === 0) { + return chalk.green('FREE'); + } + + return `$${prompt.toFixed(2)}/${completion.toFixed(2)} per M`; +} + +function formatTier(tier: OpenRouterModel['tier']): string { + const colors = { + frontier: chalk.magenta, + flagship: chalk.blue, + standard: chalk.white, + budget: chalk.yellow, + free: chalk.green, + }; + return colors[tier](tier.toUpperCase()); +} + +function formatScore(score: number | undefined): string { + if (!score) return chalk.dim('N/A'); + if (score >= 8) return chalk.green(`${score}/10`); + if (score >= 6) return chalk.yellow(`${score}/10`); + return chalk.red(`${score}/10`); +} + +function formatRole(role: string | undefined): string { + if (!role) return ''; + + const roleColors: Record string> = { + 'lead-analyst': chalk.blue, + 'red-team': chalk.red, + 'judge': chalk.magenta, + 'contrarian': chalk.yellow, + 'chairman': chalk.cyan, + 'generalist': chalk.white, + }; + + return (roleColors[role] || chalk.white)(role); +} + +function displayModel(model: OpenRouterModel, verbose: boolean = false): void { + const analysis = model.councilAnalysis; + + // Header line + const recommended = model.legalCouncilRecommended ? chalk.green(' ✓ RECOMMENDED') : ''; + const chairmanBadge = analysis?.chairmanSuitable ? chalk.cyan(' 👑 Chairman') : ''; + + console.log(); + console.log(chalk.bold(model.id) + recommended + chairmanBadge); + console.log(chalk.dim(` ${model.name} by ${model.provider}`)); + + // Specs line + const tier = formatTier(model.tier); + const price = formatPrice(model); + const context = `${(model.contextLength / 1000).toFixed(0)}K ctx`; + const maxOut = model.maxOutputTokens ? `${(model.maxOutputTokens / 1000).toFixed(0)}K max` : ''; + + console.log(` ${tier} | ${price} | ${context}${maxOut ? ' | ' + maxOut : ''}`); + + // Council analysis + if (analysis) { + const score = formatScore(analysis.councilScore); + const role = formatRole(analysis.recommendedRole); + console.log(` Council Score: ${score} | Role: ${role}`); + } + + // Capabilities + const caps = model.capabilities; + const capList = []; + if (caps.functionCalling) capList.push('tools'); + if (caps.jsonMode) capList.push('json'); + if (caps.vision) capList.push('vision'); + if (caps.streaming) capList.push('stream'); + console.log(chalk.dim(` Capabilities: ${capList.join(', ')}`)); + + // Verbose output + if (verbose && analysis) { + console.log(); + console.log(chalk.green(' Strengths:')); + for (const s of analysis.strengths.slice(0, 5)) { + console.log(chalk.green(` + ${s}`)); + } + + console.log(chalk.red(' Weaknesses:')); + for (const w of analysis.weaknesses.slice(0, 3)) { + console.log(chalk.red(` - ${w}`)); + } + + console.log(); + console.log(chalk.dim(` ${analysis.summary}`)); + } +} + +function displayModelCompact(model: OpenRouterModel): void { + const analysis = model.councilAnalysis; + const score = analysis?.councilScore ? `[${analysis.councilScore}]` : ' '; + const rec = model.legalCouncilRecommended ? chalk.green('✓') : ' '; + const chr = analysis?.chairmanSuitable ? chalk.cyan('👑') : ' '; + const role = analysis?.recommendedRole ? analysis.recommendedRole.substring(0, 8).padEnd(8) : ' '; + const price = formatPrice(model).padEnd(20); + + console.log( + `${rec} ${chr} ${score} ${model.id.padEnd(35)} ${formatTier(model.tier).padEnd(15)} ${price} ${chalk.dim(role)}` + ); +} + +// ============================================================================ +// CLI COMMANDS +// ============================================================================ + +async function listModels(options: { + recommended?: boolean; + chairman?: boolean; + role?: string; + tier?: string; + verbose?: boolean; + compact?: boolean; +}): Promise { + console.log(chalk.dim('Fetching models from OpenRouter...')); + + const apiKey = process.env.OPENROUTER_API_KEY; + const selector = await createModelSelector(apiKey); + let models = selector.getAllModels(); + + // Apply filters + if (options.recommended) { + models = models.filter(m => m.legalCouncilRecommended); + } + + if (options.chairman) { + models = models.filter(m => m.councilAnalysis?.chairmanSuitable); + } + + if (options.role) { + models = models.filter(m => m.councilAnalysis?.recommendedRole === options.role); + } + + if (options.tier) { + const tiers = options.tier.split(','); + models = models.filter(m => tiers.includes(m.tier)); + } + + // Sort by council score (descending) + models.sort((a, b) => { + const scoreA = a.councilAnalysis?.councilScore || 0; + const scoreB = b.councilAnalysis?.councilScore || 0; + return scoreB - scoreA; + }); + + logSection(`Available Models (${models.length} total)`); + + if (options.compact) { + console.log(); + console.log(chalk.dim(' ✓ 👑 [Score] Model ID Tier Price Role')); + console.log(chalk.dim(' ─'.repeat(50))); + + for (const model of models) { + displayModelCompact(model); + } + } else { + for (const model of models) { + displayModel(model, options.verbose); + } + } + + console.log(); + console.log(chalk.dim(`Showing ${models.length} models. Use --recommended to filter to council-suitable models.`)); +} + +async function showModelInfo(modelId: string): Promise { + console.log(chalk.dim('Fetching model information...')); + + const apiKey = process.env.OPENROUTER_API_KEY; + const selector = await createModelSelector(apiKey); + const model = selector.getModel(modelId); + + if (!model) { + console.log(chalk.red(`Model not found: ${modelId}`)); + console.log(chalk.dim('Use "list" command to see available models.')); + process.exit(1); + } + + logSection('Model Details'); + displayModel(model, true); + + // Full pricing breakdown + console.log(); + console.log(chalk.bold('Pricing:')); + console.log(` Input: $${model.pricing.promptPerMillion.toFixed(4)} per million tokens`); + console.log(` Output: $${model.pricing.completionPerMillion.toFixed(4)} per million tokens`); + if (model.pricing.requestFee) { + console.log(` Per Request: $${model.pricing.requestFee.toFixed(4)}`); + } + + // Estimate cost per deliberation (if this model used for all seats) + console.log(); + console.log(chalk.bold('Estimated Cost (if used for entire council):')); + const config: CouncilConfiguration = { + seatA: modelId, + seatB: modelId, + seatC: modelId, + seatD: modelId, + chairman: modelId, + }; + const pricing = selector.getConfigPricingSummary(config); + console.log(` ~$${pricing.estimatedCostPerDeliberation.toFixed(4)} per deliberation`); +} + +async function showRecommendation(): Promise { + console.log(chalk.dim('Loading recommended configuration...')); + + const apiKey = process.env.OPENROUTER_API_KEY; + const selector = await createModelSelector(apiKey); + const rec = selector.getRecommendedConfig(); + + logSection('Recommended Council Configuration'); + + console.log(); + console.log(chalk.bold('Seat A (Lead Analyst):')); + const seatA = selector.getModel(rec.seatA); + if (seatA) displayModel(seatA, false); + + console.log(); + console.log(chalk.bold('Seat B (Red Team):')); + const seatB = selector.getModel(rec.seatB); + if (seatB) displayModel(seatB, false); + + console.log(); + console.log(chalk.bold('Seat C (Judge):')); + const seatC = selector.getModel(rec.seatC); + if (seatC) displayModel(seatC, false); + + console.log(); + console.log(chalk.bold('Seat D (Contrarian):')); + const seatD = selector.getModel(rec.seatD); + if (seatD) displayModel(seatD, false); + + console.log(); + console.log(chalk.cyan('═'.repeat(70))); + console.log(chalk.bold(' Chairman (Synthesis):')); + const chairman = selector.getModel(rec.chairman); + if (chairman) displayModel(chairman, false); + + // Pricing summary + console.log(); + logSection('Pricing Summary'); + const pricing = selector.getConfigPricingSummary({ + seatA: rec.seatA, + seatB: rec.seatB, + seatC: rec.seatC, + seatD: rec.seatD, + chairman: rec.chairman, + }); + + console.log(); + console.log(chalk.bold(`Estimated cost per deliberation: $${pricing.estimatedCostPerDeliberation.toFixed(4)}`)); + console.log(); + console.log('Breakdown:'); + for (const [model, costs] of Object.entries(pricing.breakdown)) { + const total = costs.promptCost + costs.completionCost; + console.log(` ${model.padEnd(40)} $${total.toFixed(4)}`); + } + + // Rationale + logSection('Rationale'); + console.log(); + console.log(rec.rationale); + + // Environment variable output + logSection('Environment Variables'); + console.log(); + console.log(chalk.dim('Add to your .env file:')); + console.log(); + console.log(`COUNCIL_MODEL_1=${rec.seatA}`); + console.log(`COUNCIL_MODEL_2=${rec.seatB}`); + console.log(`COUNCIL_MODEL_3=${rec.seatC}`); + console.log(`COUNCIL_MODEL_4=${rec.seatD}`); + console.log(`CHAIRMAN_MODEL=${rec.chairman}`); +} + +async function validateConfig(): Promise { + console.log(chalk.dim('Validating current configuration...')); + + const apiKey = process.env.OPENROUTER_API_KEY; + const selector = await createModelSelector(apiKey); + + // Read current config from environment + const config: CouncilConfiguration = { + seatA: process.env.COUNCIL_MODEL_1 || '', + seatB: process.env.COUNCIL_MODEL_2 || '', + seatC: process.env.COUNCIL_MODEL_3 || '', + seatD: process.env.COUNCIL_MODEL_4, + chairman: process.env.CHAIRMAN_MODEL || process.env.COUNCIL_MODEL_1 || '', + }; + + logSection('Current Configuration'); + + console.log(); + console.log(`Seat A: ${config.seatA || chalk.red('(not set)')}`); + console.log(`Seat B: ${config.seatB || chalk.red('(not set)')}`); + console.log(`Seat C: ${config.seatC || chalk.red('(not set)')}`); + console.log(`Seat D: ${config.seatD || chalk.dim('(optional, not set)')}`); + console.log(`Chairman: ${config.chairman || chalk.dim('(will use algorithmic selection)')}`); + + if (!config.seatA || !config.seatB) { + console.log(); + console.log(chalk.red('ERROR: Minimum 2 council models required.')); + console.log(chalk.dim('Set COUNCIL_MODEL_1 and COUNCIL_MODEL_2 in your .env file.')); + process.exit(1); + } + + const validation = selector.validateConfig(config); + + logSection('Validation Results'); + + if (validation.errors.length > 0) { + console.log(); + console.log(chalk.red('ERRORS:')); + for (const error of validation.errors) { + console.log(chalk.red(` ✗ ${error}`)); + } + } + + if (validation.warnings.length > 0) { + console.log(); + console.log(chalk.yellow('WARNINGS:')); + for (const warning of validation.warnings) { + console.log(chalk.yellow(` ⚠ ${warning}`)); + } + } + + if (validation.valid && validation.warnings.length === 0) { + console.log(); + console.log(chalk.green('✓ Configuration is valid and recommended.')); + } else if (validation.valid) { + console.log(); + console.log(chalk.yellow('⚠ Configuration is valid but has warnings.')); + } else { + console.log(); + console.log(chalk.red('✗ Configuration has errors. Please fix before running deliberations.')); + process.exit(1); + } + + // Show pricing + const pricing = selector.getConfigPricingSummary(config); + console.log(); + console.log(chalk.bold(`Estimated cost per deliberation: $${pricing.estimatedCostPerDeliberation.toFixed(4)}`)); +} + +async function interactiveConfigure(): Promise { + const apiKey = process.env.OPENROUTER_API_KEY; + const selector = await createModelSelector(apiKey); + + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + + const question = (prompt: string): Promise => { + return new Promise((resolve) => { + rl.question(prompt, resolve); + }); + }; + + logSection('Interactive Council Configuration'); + + console.log(); + console.log(chalk.dim('This wizard will help you configure your Legal Council.')); + console.log(chalk.dim('You can skip any seat by pressing Enter.')); + console.log(); + + // Get recommended models for each role + const leadModels = selector.getModelsForRole('lead-analyst').slice(0, 5); + const redTeamModels = selector.getModelsForRole('red-team').slice(0, 5); + const judgeModels = selector.getModelsForRole('judge').slice(0, 5); + const contrarianModels = selector.getModelsForRole('contrarian').slice(0, 5); + const chairmanModels = selector.getChairmanModels().slice(0, 5); + + console.log(chalk.bold('Seat A - Lead Analyst')); + console.log(chalk.dim('Recommended models:')); + leadModels.forEach((m, i) => console.log(` ${i + 1}. ${m.id}`)); + const seatA = await question(chalk.cyan('Enter model ID (or number): ')); + + console.log(); + console.log(chalk.bold('Seat B - Red Team')); + console.log(chalk.dim('Recommended models:')); + redTeamModels.forEach((m, i) => console.log(` ${i + 1}. ${m.id}`)); + const seatB = await question(chalk.cyan('Enter model ID (or number): ')); + + console.log(); + console.log(chalk.bold('Seat C - Judge')); + console.log(chalk.dim('Recommended models:')); + judgeModels.forEach((m, i) => console.log(` ${i + 1}. ${m.id}`)); + const seatC = await question(chalk.cyan('Enter model ID (or number): ')); + + console.log(); + console.log(chalk.bold('Seat D - Contrarian (Optional)')); + console.log(chalk.dim('Recommended models:')); + contrarianModels.forEach((m, i) => console.log(` ${i + 1}. ${m.id}`)); + const seatD = await question(chalk.cyan('Enter model ID (or press Enter to skip): ')); + + console.log(); + console.log(chalk.bold('Chairman - Synthesis')); + console.log(chalk.dim('Recommended models:')); + chairmanModels.forEach((m, i) => console.log(` ${i + 1}. ${m.id}`)); + const chairman = await question(chalk.cyan('Enter model ID (or press Enter for algorithmic selection): ')); + + rl.close(); + + // Resolve numbers to model IDs + const resolveSelection = (input: string, models: OpenRouterModel[]): string => { + const num = parseInt(input); + if (!isNaN(num) && num >= 1 && num <= models.length) { + return models[num - 1].id; + } + return input; + }; + + const config: CouncilConfiguration = { + seatA: resolveSelection(seatA, leadModels) || leadModels[0]?.id || '', + seatB: resolveSelection(seatB, redTeamModels) || redTeamModels[0]?.id || '', + seatC: resolveSelection(seatC, judgeModels) || judgeModels[0]?.id || '', + seatD: seatD ? resolveSelection(seatD, contrarianModels) : undefined, + chairman: chairman ? resolveSelection(chairman, chairmanModels) : config.seatA || '', + }; + + logSection('Your Configuration'); + console.log(); + console.log(`COUNCIL_MODEL_1=${config.seatA}`); + console.log(`COUNCIL_MODEL_2=${config.seatB}`); + console.log(`COUNCIL_MODEL_3=${config.seatC}`); + if (config.seatD) { + console.log(`COUNCIL_MODEL_4=${config.seatD}`); + } + if (config.chairman && config.chairman !== config.seatA) { + console.log(`CHAIRMAN_MODEL=${config.chairman}`); + } + + // Validate + const validation = selector.validateConfig(config); + if (!validation.valid) { + console.log(); + console.log(chalk.red('Configuration has errors:')); + validation.errors.forEach(e => console.log(chalk.red(` ✗ ${e}`))); + } + + // Pricing + const pricing = selector.getConfigPricingSummary(config); + console.log(); + console.log(chalk.bold(`Estimated cost per deliberation: $${pricing.estimatedCostPerDeliberation.toFixed(4)}`)); +} + +// ============================================================================ +// CLI SETUP +// ============================================================================ + +program + .name('model-selector') + .description('Model selection and configuration for LLM Legal Council') + .version('0.1.0'); + +program + .command('list') + .description('List available models') + .option('-r, --recommended', 'Show only council-recommended models') + .option('-c, --chairman', 'Show only chairman-suitable models') + .option('--role ', 'Filter by recommended role (lead-analyst, red-team, judge, contrarian)') + .option('--tier ', 'Filter by tier (frontier, flagship, standard, budget, free)') + .option('-v, --verbose', 'Show detailed analysis for each model') + .option('--compact', 'Show compact table view') + .action(listModels); + +program + .command('info ') + .description('Show detailed information for a specific model') + .action(showModelInfo); + +program + .command('recommend') + .description('Show recommended council configuration') + .action(showRecommendation); + +program + .command('validate') + .description('Validate current configuration from environment') + .action(validateConfig); + +program + .command('configure') + .description('Interactive configuration wizard') + .option('-i, --interactive', 'Run interactive configuration') + .action(interactiveConfigure); + +program.parse(); diff --git a/src/models/curated-analysis.ts b/src/models/curated-analysis.ts new file mode 100644 index 0000000..fd244cf --- /dev/null +++ b/src/models/curated-analysis.ts @@ -0,0 +1,905 @@ +/** + * Curated Model Analysis for LLM Legal Council + * + * Expert analysis of each model's strengths and weaknesses for legal deliberation. + * This serves as both: + * 1. Fallback data when OpenRouter API is unavailable + * 2. Council-specific analysis overlaid on API-fetched models + * + * Analysis criteria: + * - Legal reasoning capability + * - Citation accuracy and hallucination resistance + * - Adversarial thinking ability + * - Synthesis and summarization quality + * - Instruction following (for structured JSON output) + * - Context utilization + */ + +import { OpenRouterModel, CouncilAnalysis } from './types.js'; + +/** + * Curated analysis keyed by model ID + */ +export const CURATED_MODEL_ANALYSIS: Record = { + // ============================================================================ + // ANTHROPIC MODELS + // ============================================================================ + + 'anthropic/claude-sonnet-4': { + id: 'anthropic/claude-sonnet-4', + name: 'Claude Sonnet 4', + provider: 'anthropic', + description: 'Anthropic\'s latest Sonnet model with enhanced reasoning and instruction following.', + contextLength: 200000, + maxOutputTokens: 8192, + pricing: { + promptPerMillion: 3.00, + completionPerMillion: 15.00, + }, + capabilities: { + functionCalling: true, + jsonMode: true, + vision: true, + streaming: true, + systemPrompt: true, + }, + tier: 'frontier', + legalCouncilRecommended: true, + councilAnalysis: { + councilScore: 9, + strengths: [ + 'Exceptional instruction following for structured legal analysis', + 'Strong adversarial reasoning - excels at identifying counterarguments', + 'Reliable JSON mode output with consistent schema adherence', + 'Nuanced understanding of legal precedent and authority hierarchy', + 'Excellent at preserving dissent without false consensus', + 'Low hallucination rate for legal citations', + '200K context handles large document sets effectively', + ], + weaknesses: [ + 'Can be overly cautious, may hedge more than necessary', + 'Sometimes verbose in explanations', + 'Higher cost than budget alternatives', + ], + recommendedRole: 'lead-analyst', + chairmanSuitable: true, + summary: 'Best overall choice for lead analyst or chairman. Excels at structured legal reasoning, adversarial thinking, and synthesis. High instruction compliance makes it ideal for JSON-structured deliberations.', + }, + }, + + 'anthropic/claude-3.5-sonnet': { + id: 'anthropic/claude-3.5-sonnet', + name: 'Claude 3.5 Sonnet', + provider: 'anthropic', + description: 'Previous generation Sonnet with strong reasoning capabilities.', + contextLength: 200000, + maxOutputTokens: 8192, + pricing: { + promptPerMillion: 3.00, + completionPerMillion: 15.00, + }, + capabilities: { + functionCalling: true, + jsonMode: true, + vision: true, + streaming: true, + systemPrompt: true, + }, + tier: 'flagship', + legalCouncilRecommended: true, + councilAnalysis: { + councilScore: 8, + strengths: [ + 'Proven track record for legal analysis', + 'Strong structured output compliance', + 'Good adversarial reasoning', + 'Handles long documents well', + ], + weaknesses: [ + 'Slightly older model, may lack latest improvements', + 'Same pricing as Claude Sonnet 4 with less capability', + ], + recommendedRole: 'generalist', + chairmanSuitable: true, + summary: 'Reliable workhorse for legal council. Consider upgrading to Claude Sonnet 4 for new deployments.', + }, + }, + + 'anthropic/claude-3-opus': { + id: 'anthropic/claude-3-opus', + name: 'Claude 3 Opus', + provider: 'anthropic', + description: 'Anthropic\'s most powerful model with exceptional reasoning.', + contextLength: 200000, + maxOutputTokens: 4096, + pricing: { + promptPerMillion: 15.00, + completionPerMillion: 75.00, + }, + capabilities: { + functionCalling: true, + jsonMode: true, + vision: true, + streaming: true, + systemPrompt: true, + }, + tier: 'frontier', + legalCouncilRecommended: true, + councilAnalysis: { + councilScore: 9, + strengths: [ + 'Deepest reasoning capability in the Claude family', + 'Excellent for complex multi-factor legal analysis', + 'Superior at synthesizing conflicting viewpoints', + 'Best-in-class for chairman synthesis tasks', + 'Exceptional at preserving nuance in dissent', + ], + weaknesses: [ + 'Significantly more expensive (5x Sonnet)', + 'Slower response times', + 'May be overkill for straightforward analyses', + ], + recommendedRole: 'chairman', + chairmanSuitable: true, + summary: 'Premium choice for chairman role. Best reasoning depth justifies cost for final synthesis. Consider for high-stakes matters.', + }, + }, + + 'anthropic/claude-3-haiku': { + id: 'anthropic/claude-3-haiku', + name: 'Claude 3 Haiku', + provider: 'anthropic', + description: 'Fast, cost-effective Claude model for simpler tasks.', + contextLength: 200000, + maxOutputTokens: 4096, + pricing: { + promptPerMillion: 0.25, + completionPerMillion: 1.25, + }, + capabilities: { + functionCalling: true, + jsonMode: true, + vision: true, + streaming: true, + systemPrompt: true, + }, + tier: 'budget', + legalCouncilRecommended: false, + councilAnalysis: { + councilScore: 5, + strengths: [ + 'Very fast responses', + 'Cost-effective for high-volume use', + 'Good for simple extraction tasks', + ], + weaknesses: [ + 'Insufficient reasoning depth for legal analysis', + 'Higher hallucination rate than larger models', + 'May miss subtle legal issues', + 'Not suitable for adversarial examination', + ], + recommendedRole: 'generalist', + chairmanSuitable: false, + summary: 'Not recommended for council deliberation. Insufficient reasoning depth for legal analysis tasks.', + }, + }, + + // ============================================================================ + // OPENAI MODELS + // ============================================================================ + + 'openai/gpt-4o': { + id: 'openai/gpt-4o', + name: 'GPT-4o', + provider: 'openai', + description: 'OpenAI\'s flagship multimodal model.', + contextLength: 128000, + maxOutputTokens: 16384, + pricing: { + promptPerMillion: 2.50, + completionPerMillion: 10.00, + }, + capabilities: { + functionCalling: true, + jsonMode: true, + vision: true, + streaming: true, + systemPrompt: true, + }, + tier: 'frontier', + legalCouncilRecommended: false, // Explicitly excluded per handoff + councilAnalysis: { + councilScore: 4, // Downgraded due to known issues + strengths: [ + 'Fast inference speed', + 'Good multimodal capabilities', + 'Wide tool ecosystem', + ], + weaknesses: [ + 'EXPLICITLY EXCLUDED from Legal Council - do not use', + 'Higher hallucination rate than Claude for legal citations', + 'Inconsistent JSON mode behavior', + 'Tends toward confident but incorrect legal assertions', + 'Less reliable adversarial reasoning', + ], + recommendedRole: 'generalist', + chairmanSuitable: false, + summary: 'NOT RECOMMENDED for Legal Council. Explicitly excluded due to reliability issues with legal analysis. Use GPT-5.2 instead.', + }, + }, + + 'openai/gpt-4-turbo': { + id: 'openai/gpt-4-turbo', + name: 'GPT-4 Turbo', + provider: 'openai', + description: 'Previous generation GPT-4 with turbo optimizations.', + contextLength: 128000, + maxOutputTokens: 4096, + pricing: { + promptPerMillion: 10.00, + completionPerMillion: 30.00, + }, + capabilities: { + functionCalling: true, + jsonMode: true, + vision: true, + streaming: true, + systemPrompt: true, + }, + tier: 'flagship', + legalCouncilRecommended: false, + councilAnalysis: { + councilScore: 5, + strengths: [ + 'Improved over base GPT-4', + 'Good general reasoning', + ], + weaknesses: [ + 'Superseded by newer models', + 'Same hallucination issues as GPT-4o', + 'Higher cost than alternatives', + ], + recommendedRole: 'generalist', + chairmanSuitable: false, + summary: 'Not recommended. Consider GPT-5.2 for OpenAI models in legal council.', + }, + }, + + 'openai/o1': { + id: 'openai/o1', + name: 'OpenAI o1', + provider: 'openai', + description: 'OpenAI\'s reasoning-focused model with extended thinking.', + contextLength: 200000, + maxOutputTokens: 100000, + pricing: { + promptPerMillion: 15.00, + completionPerMillion: 60.00, + }, + capabilities: { + functionCalling: false, // o1 has limited function calling + jsonMode: true, + vision: true, + streaming: false, // o1 doesn't stream + systemPrompt: false, // o1 has limited system prompt support + }, + tier: 'frontier', + legalCouncilRecommended: true, + councilAnalysis: { + councilScore: 8, + strengths: [ + 'Extended reasoning chains for complex legal problems', + 'Excellent at multi-step logical analysis', + 'Strong at identifying subtle issues', + 'Good at adversarial thinking', + ], + weaknesses: [ + 'No streaming - poor UX for real-time display', + 'Limited system prompt support', + 'No function calling - cannot use council tools', + 'Very slow response times (minutes)', + 'Expensive', + ], + recommendedRole: 'judge', + chairmanSuitable: false, // No streaming, limited system prompt + summary: 'Excellent reasoning but operational limitations. Best as judge role for deep analysis where streaming not needed.', + }, + }, + + 'openai/o1-mini': { + id: 'openai/o1-mini', + name: 'OpenAI o1-mini', + provider: 'openai', + description: 'Smaller, faster version of o1 with reasoning capabilities.', + contextLength: 128000, + maxOutputTokens: 65536, + pricing: { + promptPerMillion: 3.00, + completionPerMillion: 12.00, + }, + capabilities: { + functionCalling: false, + jsonMode: true, + vision: false, + streaming: false, + systemPrompt: false, + }, + tier: 'standard', + legalCouncilRecommended: false, + councilAnalysis: { + councilScore: 6, + strengths: [ + 'Reasoning focus at lower cost than o1', + 'Good for specific analytical tasks', + ], + weaknesses: [ + 'Same operational limitations as o1', + 'Less capable than full o1', + 'No streaming or function calling', + ], + recommendedRole: 'generalist', + chairmanSuitable: false, + summary: 'Consider for specialized reasoning tasks only. Operational limitations reduce council utility.', + }, + }, + + // Hypothetical GPT-5.2 (from handoff document) + 'openai/gpt-5.2': { + id: 'openai/gpt-5.2', + name: 'GPT-5.2', + provider: 'openai', + description: 'OpenAI\'s latest flagship model with improved reasoning and reliability.', + contextLength: 256000, + maxOutputTokens: 32768, + pricing: { + promptPerMillion: 5.00, + completionPerMillion: 15.00, + }, + capabilities: { + functionCalling: true, + jsonMode: true, + vision: true, + streaming: true, + systemPrompt: true, + }, + tier: 'frontier', + legalCouncilRecommended: true, + councilAnalysis: { + councilScore: 8, + strengths: [ + 'Significantly improved over GPT-4o', + 'Better citation accuracy', + 'Strong adversarial reasoning', + 'Reliable tool iteration', + 'Good for red team role - finds weaknesses', + ], + weaknesses: [ + 'Tool iteration requires proper handling (see handoff)', + 'May still be more confident than warranted', + 'Less nuanced than Claude for synthesis', + ], + recommendedRole: 'red-team', + chairmanSuitable: true, + summary: 'Recommended OpenAI model for council. Assign to red team role for aggressive weakness identification.', + }, + }, + + // ============================================================================ + // GOOGLE MODELS + // ============================================================================ + + 'google/gemini-2.0-flash-exp': { + id: 'google/gemini-2.0-flash-exp', + name: 'Gemini 2.0 Flash (Experimental)', + provider: 'google', + description: 'Google\'s latest experimental Gemini model.', + contextLength: 1000000, + maxOutputTokens: 8192, + pricing: { + promptPerMillion: 0.00, // Often free during experimental + completionPerMillion: 0.00, + }, + capabilities: { + functionCalling: true, + jsonMode: true, + vision: true, + streaming: true, + systemPrompt: true, + }, + tier: 'frontier', + legalCouncilRecommended: false, // Experimental + councilAnalysis: { + councilScore: 6, + strengths: [ + 'Massive 1M context window', + 'Free during experimental period', + 'Good for document-heavy analysis', + ], + weaknesses: [ + 'Experimental - may have unpredictable behavior', + 'Not production-ready', + 'JSON mode can be inconsistent', + ], + recommendedRole: 'generalist', + chairmanSuitable: false, + summary: 'Not recommended for production legal council. Consider Gemini Pro for stable deployment.', + }, + }, + + 'google/gemini-1.5-pro': { + id: 'google/gemini-1.5-pro', + name: 'Gemini 1.5 Pro', + provider: 'google', + description: 'Google\'s production Gemini model with excellent context handling.', + contextLength: 2000000, + maxOutputTokens: 8192, + pricing: { + promptPerMillion: 1.25, + completionPerMillion: 5.00, + }, + capabilities: { + functionCalling: true, + jsonMode: true, + vision: true, + streaming: true, + systemPrompt: true, + }, + tier: 'flagship', + legalCouncilRecommended: true, + councilAnalysis: { + councilScore: 8, + strengths: [ + 'Exceptional 2M context window - best for document analysis', + 'Good at synthesizing across multiple documents', + 'Cost-effective for document-heavy deliberations', + 'Strong factual grounding', + 'Natural judge/arbiter role', + ], + weaknesses: [ + 'JSON mode sometimes requires fallback handling', + 'Less consistent than Claude for structured output', + 'May need COUNCIL_JSON_FALLBACK_MODELS config', + ], + recommendedRole: 'judge', + chairmanSuitable: true, + summary: 'Excellent for judge role. Massive context enables full document ingestion. Good synthesis capabilities for chairman backup.', + }, + }, + + // Hypothetical Gemini 3 Pro (from handoff) + 'google/gemini-3-pro': { + id: 'google/gemini-3-pro', + name: 'Gemini 3 Pro', + provider: 'google', + description: 'Google\'s latest production Gemini model.', + contextLength: 2000000, + maxOutputTokens: 16384, + pricing: { + promptPerMillion: 2.50, + completionPerMillion: 10.00, + }, + capabilities: { + functionCalling: true, + jsonMode: true, + vision: true, + streaming: true, + systemPrompt: true, + }, + tier: 'frontier', + legalCouncilRecommended: true, + councilAnalysis: { + councilScore: 8, + strengths: [ + 'Industry-leading context window', + 'Improved JSON mode reliability', + 'Excellent document grounding', + 'Strong at cross-referencing authorities', + 'Natural arbiter/judge capabilities', + ], + weaknesses: [ + 'May defer to consensus rather than push back', + 'Less aggressive in adversarial mode', + ], + recommendedRole: 'judge', + chairmanSuitable: true, + summary: 'Recommended for judge role. Excellent at weighing evidence and synthesizing multiple viewpoints objectively.', + }, + }, + + 'google/gemini-1.5-flash': { + id: 'google/gemini-1.5-flash', + name: 'Gemini 1.5 Flash', + provider: 'google', + description: 'Fast, cost-effective Gemini model.', + contextLength: 1000000, + maxOutputTokens: 8192, + pricing: { + promptPerMillion: 0.075, + completionPerMillion: 0.30, + }, + capabilities: { + functionCalling: true, + jsonMode: true, + vision: true, + streaming: true, + systemPrompt: true, + }, + tier: 'budget', + legalCouncilRecommended: false, + councilAnalysis: { + councilScore: 5, + strengths: [ + 'Very low cost', + 'Large context window', + 'Fast responses', + ], + weaknesses: [ + 'Reduced reasoning depth', + 'Higher error rate on complex legal questions', + 'Not suitable for adversarial analysis', + ], + recommendedRole: 'generalist', + chairmanSuitable: false, + summary: 'Not recommended for legal council. Insufficient reasoning depth for deliberation tasks.', + }, + }, + + // ============================================================================ + // XAI MODELS + // ============================================================================ + + 'x-ai/grok-2': { + id: 'x-ai/grok-2', + name: 'Grok 2', + provider: 'x-ai', + description: 'xAI\'s Grok 2 model with strong reasoning.', + contextLength: 131072, + maxOutputTokens: 8192, + pricing: { + promptPerMillion: 2.00, + completionPerMillion: 10.00, + }, + capabilities: { + functionCalling: true, + jsonMode: true, + vision: false, + streaming: true, + systemPrompt: true, + }, + tier: 'flagship', + legalCouncilRecommended: true, + councilAnalysis: { + councilScore: 7, + strengths: [ + 'Strong contrarian thinking', + 'Less prone to consensus-seeking', + 'Good at identifying unconventional arguments', + 'Willing to take unpopular positions', + ], + weaknesses: [ + 'Can be overly aggressive/contrarian', + 'May push back unnecessarily', + 'Less refined than Claude for synthesis', + 'Newer model with less track record', + ], + recommendedRole: 'contrarian', + chairmanSuitable: false, + summary: 'Excellent contrarian. Assign to Seat D for devil\'s advocate role. Will challenge consensus and find weaknesses others miss.', + }, + }, + + // Hypothetical Grok 4.1 (from handoff) + 'x-ai/grok-4.1': { + id: 'x-ai/grok-4.1', + name: 'Grok 4.1', + provider: 'x-ai', + description: 'xAI\'s latest Grok model with enhanced reasoning.', + contextLength: 200000, + maxOutputTokens: 16384, + pricing: { + promptPerMillion: 3.00, + completionPerMillion: 12.00, + }, + capabilities: { + functionCalling: true, + jsonMode: true, + vision: true, + streaming: true, + systemPrompt: true, + }, + tier: 'frontier', + legalCouncilRecommended: true, + councilAnalysis: { + councilScore: 8, + strengths: [ + 'Best-in-class contrarian analysis', + 'Excellent at stress testing arguments', + 'Unafraid to identify fatal flaws', + 'Strong adversarial examination', + 'Good at opposing counsel simulation', + ], + weaknesses: [ + 'May be too aggressive for some analyses', + 'Less suited for neutral synthesis', + ], + recommendedRole: 'contrarian', + chairmanSuitable: false, + summary: 'Top choice for contrarian/Seat D. Will ruthlessly stress-test your position and identify weaknesses.', + }, + }, + + // ============================================================================ + // MISTRAL MODELS + // ============================================================================ + + 'mistralai/mistral-large': { + id: 'mistralai/mistral-large', + name: 'Mistral Large', + provider: 'mistralai', + description: 'Mistral\'s flagship model with strong reasoning.', + contextLength: 128000, + maxOutputTokens: 8192, + pricing: { + promptPerMillion: 2.00, + completionPerMillion: 6.00, + }, + capabilities: { + functionCalling: true, + jsonMode: true, + vision: false, + streaming: true, + systemPrompt: true, + }, + tier: 'flagship', + legalCouncilRecommended: true, + councilAnalysis: { + councilScore: 7, + strengths: [ + 'Cost-effective alternative to frontier models', + 'Good structured output', + 'Solid reasoning capabilities', + 'Efficient token usage', + ], + weaknesses: [ + 'Less depth than Claude or GPT-5 on complex issues', + 'May miss subtle legal distinctions', + 'Less established for legal tasks', + ], + recommendedRole: 'generalist', + chairmanSuitable: true, + summary: 'Good budget-conscious option. Can serve any role but excels as cost-effective generalist.', + }, + }, + + 'mistralai/codestral-latest': { + id: 'mistralai/codestral-latest', + name: 'Codestral', + provider: 'mistralai', + description: 'Mistral\'s code-focused model.', + contextLength: 32000, + maxOutputTokens: 8192, + pricing: { + promptPerMillion: 0.30, + completionPerMillion: 0.90, + }, + capabilities: { + functionCalling: true, + jsonMode: true, + vision: false, + streaming: true, + systemPrompt: true, + }, + tier: 'standard', + legalCouncilRecommended: false, + councilAnalysis: { + councilScore: 3, + strengths: [ + 'Good for code-related legal issues', + 'Low cost', + ], + weaknesses: [ + 'Optimized for code, not legal reasoning', + 'Limited context window', + 'Not designed for deliberation tasks', + ], + recommendedRole: 'generalist', + chairmanSuitable: false, + summary: 'Not recommended. Code-focused model unsuitable for legal council.', + }, + }, + + // ============================================================================ + // COHERE MODELS + // ============================================================================ + + 'cohere/command-r-plus': { + id: 'cohere/command-r-plus', + name: 'Command R+', + provider: 'cohere', + description: 'Cohere\'s flagship retrieval-augmented model.', + contextLength: 128000, + maxOutputTokens: 4096, + pricing: { + promptPerMillion: 2.50, + completionPerMillion: 10.00, + }, + capabilities: { + functionCalling: true, + jsonMode: true, + vision: false, + streaming: true, + systemPrompt: true, + }, + tier: 'flagship', + legalCouncilRecommended: false, + councilAnalysis: { + councilScore: 6, + strengths: [ + 'Strong RAG/retrieval capabilities', + 'Good citation handling', + 'Cost-effective', + ], + weaknesses: [ + 'Less capable reasoning than Claude/GPT', + 'Limited adversarial thinking', + 'Not ideal for complex deliberation', + ], + recommendedRole: 'generalist', + chairmanSuitable: false, + summary: 'Consider for RAG-heavy tasks only. Not recommended as primary council member.', + }, + }, + + // ============================================================================ + // META MODELS + // ============================================================================ + + 'meta-llama/llama-3.1-405b-instruct': { + id: 'meta-llama/llama-3.1-405b-instruct', + name: 'Llama 3.1 405B', + provider: 'meta-llama', + description: 'Meta\'s largest open model.', + contextLength: 131072, + maxOutputTokens: 4096, + pricing: { + promptPerMillion: 2.70, + completionPerMillion: 2.70, + }, + capabilities: { + functionCalling: true, + jsonMode: true, + vision: false, + streaming: true, + systemPrompt: true, + }, + tier: 'flagship', + legalCouncilRecommended: false, + councilAnalysis: { + councilScore: 6, + strengths: [ + 'Large parameter count', + 'Open source (can self-host)', + 'Good general reasoning', + ], + weaknesses: [ + 'Less refined than Claude/GPT for legal tasks', + 'JSON mode less reliable', + 'Higher hallucination rate', + ], + recommendedRole: 'generalist', + chairmanSuitable: false, + summary: 'Not recommended for legal council. Consider if self-hosting is required.', + }, + }, + + 'meta-llama/llama-3.1-70b-instruct': { + id: 'meta-llama/llama-3.1-70b-instruct', + name: 'Llama 3.1 70B', + provider: 'meta-llama', + description: 'Mid-size Llama model.', + contextLength: 131072, + maxOutputTokens: 4096, + pricing: { + promptPerMillion: 0.52, + completionPerMillion: 0.75, + }, + capabilities: { + functionCalling: true, + jsonMode: true, + vision: false, + streaming: true, + systemPrompt: true, + }, + tier: 'standard', + legalCouncilRecommended: false, + councilAnalysis: { + councilScore: 5, + strengths: [ + 'Good cost/performance ratio', + 'Open source', + ], + weaknesses: [ + 'Insufficient for legal deliberation', + 'Higher error rates', + ], + recommendedRole: 'generalist', + chairmanSuitable: false, + summary: 'Not recommended for legal council.', + }, + }, + + // ============================================================================ + // DEEPSEEK MODELS + // ============================================================================ + + 'deepseek/deepseek-r1': { + id: 'deepseek/deepseek-r1', + name: 'DeepSeek R1', + provider: 'deepseek', + description: 'DeepSeek\'s reasoning-focused model.', + contextLength: 64000, + maxOutputTokens: 8192, + pricing: { + promptPerMillion: 0.55, + completionPerMillion: 2.19, + }, + capabilities: { + functionCalling: true, + jsonMode: true, + vision: false, + streaming: true, + systemPrompt: true, + }, + tier: 'standard', + legalCouncilRecommended: false, + councilAnalysis: { + councilScore: 6, + strengths: [ + 'Very cost-effective', + 'Good reasoning for price', + 'Improving rapidly', + ], + weaknesses: [ + 'Limited context window', + 'Less tested for legal applications', + 'May have data residency concerns', + ], + recommendedRole: 'generalist', + chairmanSuitable: false, + summary: 'Promising budget option but not yet recommended for legal council.', + }, + }, +}; + +/** + * Get recommended council configuration + */ +export function getRecommendedCouncilConfig(): { + seatA: string; + seatB: string; + seatC: string; + seatD: string; + chairman: string; + rationale: string; +} { + return { + seatA: 'anthropic/claude-sonnet-4', + seatB: 'openai/gpt-5.2', + seatC: 'google/gemini-3-pro', + seatD: 'x-ai/grok-4.1', + chairman: 'anthropic/claude-sonnet-4', + rationale: ` +Recommended configuration balances capabilities: +- Seat A (Lead Analyst): Claude Sonnet 4 - Best overall reasoning and instruction compliance +- Seat B (Red Team): GPT-5.2 - Aggressive weakness identification +- Seat C (Judge): Gemini 3 Pro - Objective weighing with massive context +- Seat D (Contrarian): Grok 4.1 - Devil's advocate, stress testing +- Chairman: Claude Sonnet 4 - Excellent synthesis and dissent preservation + +Alternative configurations: +- Budget: Replace with Mistral Large, Gemini Flash (reduced capability) +- Maximum: Use Claude 3 Opus as chairman for deepest synthesis +- Speed: Use Claude Haiku for Seat D if latency critical (not recommended) +`.trim(), + }; +} diff --git a/src/models/index.ts b/src/models/index.ts new file mode 100644 index 0000000..1caedd6 --- /dev/null +++ b/src/models/index.ts @@ -0,0 +1,10 @@ +/** + * Model Selector Module - Entry Point + * + * Exports all model selection utilities for LLM Legal Council. + */ + +export * from './types.js'; +export * from './selector.js'; +export * from './openrouter-client.js'; +export { CURATED_MODEL_ANALYSIS, getRecommendedCouncilConfig } from './curated-analysis.js'; diff --git a/src/models/openrouter-client.ts b/src/models/openrouter-client.ts new file mode 100644 index 0000000..0119970 --- /dev/null +++ b/src/models/openrouter-client.ts @@ -0,0 +1,250 @@ +/** + * OpenRouter Model Fetcher + * + * Fetches the latest model list from OpenRouter API on app startup. + * Falls back to curated list if API is unavailable. + */ + +import { OpenRouterModel, ModelPricing, ModelCapabilities } from './types.js'; +import { CURATED_MODEL_ANALYSIS } from './curated-analysis.js'; + +const OPENROUTER_MODELS_URL = 'https://openrouter.ai/api/v1/models'; + +/** + * Raw model response from OpenRouter API + */ +interface OpenRouterApiModel { + id: string; + name: string; + description?: string; + context_length: number; + pricing: { + prompt: string; // Cost per token as string (e.g., "0.000003") + completion: string; // Cost per token as string + request?: string; // Per-request fee if applicable + image?: string; // Per-image fee if applicable + }; + top_provider?: { + context_length?: number; + max_completion_tokens?: number; + is_moderated?: boolean; + }; + architecture?: { + modality?: string; + tokenizer?: string; + instruct_type?: string; + }; + per_request_limits?: { + prompt_tokens?: string; + completion_tokens?: string; + }; +} + +interface OpenRouterApiResponse { + data: OpenRouterApiModel[]; +} + +/** + * Fetch all models from OpenRouter API + */ +export async function fetchOpenRouterModels(apiKey?: string): Promise { + try { + const headers: Record = { + 'Content-Type': 'application/json', + }; + + // API key optional for models endpoint but may improve rate limits + if (apiKey) { + headers['Authorization'] = `Bearer ${apiKey}`; + } + + const response = await fetch(OPENROUTER_MODELS_URL, { headers }); + + if (!response.ok) { + throw new Error(`OpenRouter API returned ${response.status}`); + } + + const data = await response.json() as OpenRouterApiResponse; + + if (!data.data || !Array.isArray(data.data)) { + throw new Error('Invalid response format from OpenRouter API'); + } + + return data.data.map(transformApiModel); + } catch (error) { + console.warn('Failed to fetch models from OpenRouter:', error instanceof Error ? error.message : error); + throw error; + } +} + +/** + * Transform OpenRouter API model to our internal format + */ +function transformApiModel(apiModel: OpenRouterApiModel): OpenRouterModel { + const provider = apiModel.id.split('/')[0] || 'unknown'; + + // Parse pricing (API returns per-token, we want per-million) + const promptPerToken = parseFloat(apiModel.pricing.prompt) || 0; + const completionPerToken = parseFloat(apiModel.pricing.completion) || 0; + + const pricing: ModelPricing = { + promptPerMillion: promptPerToken * 1_000_000, + completionPerMillion: completionPerToken * 1_000_000, + requestFee: apiModel.pricing.request ? parseFloat(apiModel.pricing.request) : undefined, + }; + + // Infer capabilities from model characteristics + const capabilities = inferCapabilities(apiModel); + + // Determine tier based on pricing and provider + const tier = inferTier(apiModel.id, pricing); + + // Get curated analysis if available + const curatedAnalysis = CURATED_MODEL_ANALYSIS[apiModel.id]; + + return { + id: apiModel.id, + name: apiModel.name, + provider, + description: apiModel.description || '', + contextLength: apiModel.context_length, + maxOutputTokens: apiModel.top_provider?.max_completion_tokens, + pricing, + capabilities, + architecture: apiModel.architecture?.modality, + tier, + legalCouncilRecommended: curatedAnalysis?.councilAnalysis?.councilScore >= 7 || false, + councilAnalysis: curatedAnalysis?.councilAnalysis, + updatedAt: new Date().toISOString(), + }; +} + +/** + * Infer model capabilities from API data + */ +function inferCapabilities(apiModel: OpenRouterApiModel): ModelCapabilities { + const id = apiModel.id.toLowerCase(); + const modality = apiModel.architecture?.modality?.toLowerCase() || ''; + + // Most modern models support these features + const isChatModel = apiModel.architecture?.instruct_type !== undefined; + const isVisionModel = modality.includes('vision') || modality.includes('multimodal') || + id.includes('vision') || id.includes('4o') || id.includes('gemini'); + + // Function calling support by known models + const supportsFunctions = + id.includes('claude') || + id.includes('gpt-4') || + id.includes('gpt-3.5') || + id.includes('gemini') || + id.includes('mistral') && !id.includes('tiny') || + id.includes('command'); + + // JSON mode support + const supportsJson = supportsFunctions || id.includes('json'); + + return { + functionCalling: supportsFunctions, + jsonMode: supportsJson, + vision: isVisionModel, + streaming: true, // Most models support streaming + systemPrompt: isChatModel, + }; +} + +/** + * Infer model tier from id and pricing + */ +function inferTier(id: string, pricing: ModelPricing): OpenRouterModel['tier'] { + const lowerid = id.toLowerCase(); + const avgPrice = (pricing.promptPerMillion + pricing.completionPerMillion) / 2; + + // Free models + if (avgPrice === 0) return 'free'; + + // Frontier models (latest, most capable) + if ( + lowerid.includes('claude-3-opus') || + lowerid.includes('claude-sonnet-4') || + lowerid.includes('gpt-4o') && !lowerid.includes('mini') || + lowerid.includes('gpt-4-turbo') || + lowerid.includes('gpt-5') || + lowerid.includes('gemini-1.5-pro') || + lowerid.includes('gemini-2') || + lowerid.includes('gemini-3') || + lowerid.includes('opus') + ) { + return 'frontier'; + } + + // Flagship models (previous generation top models) + if ( + lowerid.includes('claude-3-sonnet') || + lowerid.includes('claude-3.5-sonnet') || + lowerid.includes('gpt-4') && !lowerid.includes('mini') || + lowerid.includes('gemini-pro') || + lowerid.includes('command-r-plus') || + avgPrice > 10 + ) { + return 'flagship'; + } + + // Budget models + if (avgPrice < 1 || lowerid.includes('mini') || lowerid.includes('flash') || lowerid.includes('haiku')) { + return 'budget'; + } + + return 'standard'; +} + +/** + * Cache for fetched models + */ +let modelsCache: OpenRouterModel[] | null = null; +let cacheTimestamp: number = 0; +const CACHE_TTL_MS = 5 * 60 * 1000; // 5 minutes + +/** + * Get models with caching + */ +export async function getOpenRouterModels( + apiKey?: string, + forceRefresh: boolean = false +): Promise { + const now = Date.now(); + + if (!forceRefresh && modelsCache && (now - cacheTimestamp) < CACHE_TTL_MS) { + return modelsCache; + } + + try { + modelsCache = await fetchOpenRouterModels(apiKey); + cacheTimestamp = now; + return modelsCache; + } catch { + // Return cached data if available, even if stale + if (modelsCache) { + console.warn('Using cached model data'); + return modelsCache; + } + + // Fall back to curated list + console.warn('Using curated fallback model list'); + return getCuratedModels(); + } +} + +/** + * Get curated fallback models (when API unavailable) + */ +export function getCuratedModels(): OpenRouterModel[] { + return Object.values(CURATED_MODEL_ANALYSIS); +} + +/** + * Clear the model cache + */ +export function clearModelCache(): void { + modelsCache = null; + cacheTimestamp = 0; +} diff --git a/src/models/selector.ts b/src/models/selector.ts new file mode 100644 index 0000000..28b1339 --- /dev/null +++ b/src/models/selector.ts @@ -0,0 +1,305 @@ +/** + * Model Selector for LLM Legal Council + * + * Provides utilities for selecting and configuring council models. + * Fetches latest models from OpenRouter on app startup. + */ + +import { + OpenRouterModel, + CouncilConfiguration, + ModelSelectorOptions, + CouncilAnalysis, +} from './types.js'; +import { getOpenRouterModels, getCuratedModels } from './openrouter-client.js'; +import { CURATED_MODEL_ANALYSIS, getRecommendedCouncilConfig } from './curated-analysis.js'; + +/** + * Model Selector class - main entry point for model selection + */ +export class ModelSelector { + private models: OpenRouterModel[] = []; + private loaded: boolean = false; + private apiKey?: string; + + constructor(apiKey?: string) { + this.apiKey = apiKey; + } + + /** + * Initialize the selector by fetching models from OpenRouter + * Call this when the app starts + */ + async initialize(forceRefresh: boolean = false): Promise { + try { + this.models = await getOpenRouterModels(this.apiKey, forceRefresh); + + // Overlay curated analysis on fetched models + this.models = this.models.map(model => { + const curated = CURATED_MODEL_ANALYSIS[model.id]; + if (curated?.councilAnalysis) { + return { + ...model, + councilAnalysis: curated.councilAnalysis, + legalCouncilRecommended: curated.legalCouncilRecommended, + }; + } + return model; + }); + + this.loaded = true; + } catch (error) { + console.warn('Failed to fetch models, using curated list:', error); + this.models = getCuratedModels(); + this.loaded = true; + } + } + + /** + * Get all available models + */ + getAllModels(): OpenRouterModel[] { + if (!this.loaded) { + console.warn('ModelSelector not initialized, using curated fallback'); + return getCuratedModels(); + } + return this.models; + } + + /** + * Filter models based on options + */ + filterModels(options: ModelSelectorOptions = {}): OpenRouterModel[] { + let filtered = this.getAllModels(); + + // Filter by tier + if (options.tiers && options.tiers.length > 0) { + filtered = filtered.filter(m => options.tiers!.includes(m.tier)); + } + + // Filter by minimum context length + if (options.minContextLength) { + filtered = filtered.filter(m => m.contextLength >= options.minContextLength!); + } + + // Filter by max price + if (options.maxPricePerMillion) { + filtered = filtered.filter(m => { + const avgPrice = (m.pricing.promptPerMillion + m.pricing.completionPerMillion) / 2; + return avgPrice <= options.maxPricePerMillion!; + }); + } + + // Filter by council recommendation + if (options.councilRecommendedOnly) { + filtered = filtered.filter(m => m.legalCouncilRecommended); + } + + // Filter by chairman suitability + if (options.chairmanSuitableOnly) { + filtered = filtered.filter(m => m.councilAnalysis?.chairmanSuitable); + } + + // Filter by required capabilities + if (options.requiredCapabilities) { + filtered = filtered.filter(m => { + const caps = options.requiredCapabilities!; + if (caps.functionCalling && !m.capabilities.functionCalling) return false; + if (caps.jsonMode && !m.capabilities.jsonMode) return false; + if (caps.vision && !m.capabilities.vision) return false; + if (caps.streaming && !m.capabilities.streaming) return false; + if (caps.systemPrompt && !m.capabilities.systemPrompt) return false; + return true; + }); + } + + return filtered; + } + + /** + * Get models recommended for a specific role + */ + getModelsForRole(role: CouncilAnalysis['recommendedRole']): OpenRouterModel[] { + return this.getAllModels() + .filter(m => m.councilAnalysis?.recommendedRole === role) + .sort((a, b) => (b.councilAnalysis?.councilScore || 0) - (a.councilAnalysis?.councilScore || 0)); + } + + /** + * Get models suitable for chairman + */ + getChairmanModels(): OpenRouterModel[] { + return this.getAllModels() + .filter(m => m.councilAnalysis?.chairmanSuitable) + .sort((a, b) => (b.councilAnalysis?.councilScore || 0) - (a.councilAnalysis?.councilScore || 0)); + } + + /** + * Get a specific model by ID + */ + getModel(id: string): OpenRouterModel | undefined { + return this.getAllModels().find(m => m.id === id); + } + + /** + * Get recommended council configuration + */ + getRecommendedConfig(): CouncilConfiguration & { rationale: string } { + return getRecommendedCouncilConfig(); + } + + /** + * Validate a council configuration + */ + validateConfig(config: CouncilConfiguration): { + valid: boolean; + errors: string[]; + warnings: string[]; + } { + const errors: string[] = []; + const warnings: string[] = []; + + // Check if all models exist + const seats = [ + { name: 'Seat A', id: config.seatA }, + { name: 'Seat B', id: config.seatB }, + { name: 'Seat C', id: config.seatC }, + { name: 'Chairman', id: config.chairman }, + ]; + + if (config.seatD) { + seats.push({ name: 'Seat D', id: config.seatD }); + } + + for (const seat of seats) { + const model = this.getModel(seat.id); + if (!model) { + errors.push(`${seat.name}: Model "${seat.id}" not found in available models`); + continue; + } + + // Check council suitability + if (!model.legalCouncilRecommended) { + warnings.push(`${seat.name}: "${model.name}" is not recommended for legal council`); + } + + // Check capabilities + if (!model.capabilities.jsonMode) { + warnings.push(`${seat.name}: "${model.name}" may not support JSON mode - add to COUNCIL_JSON_FALLBACK_MODELS`); + } + + if (!model.capabilities.functionCalling) { + warnings.push(`${seat.name}: "${model.name}" does not support function calling - cannot use council tools`); + } + + // Check chairman suitability + if (seat.name === 'Chairman' && !model.councilAnalysis?.chairmanSuitable) { + warnings.push(`Chairman: "${model.name}" is not recommended for chairman role`); + } + + // Check for GPT-4o (explicitly excluded) + if (seat.id === 'openai/gpt-4o') { + errors.push(`${seat.name}: GPT-4o is explicitly excluded from Legal Council - use GPT-5.2 instead`); + } + } + + // Check for duplicate models (allowed but warn) + const modelIds = seats.map(s => s.id); + const duplicates = modelIds.filter((id, i) => modelIds.indexOf(id) !== i); + if (duplicates.length > 0) { + warnings.push(`Duplicate models in council: ${[...new Set(duplicates)].join(', ')}`); + } + + return { + valid: errors.length === 0, + errors, + warnings, + }; + } + + /** + * Format model for display + */ + formatModelDisplay(model: OpenRouterModel): string { + const analysis = model.councilAnalysis; + const score = analysis?.councilScore ? `[${analysis.councilScore}/10]` : ''; + const role = analysis?.recommendedRole ? `(${analysis.recommendedRole})` : ''; + const chairman = analysis?.chairmanSuitable ? ' 👑' : ''; + const recommended = model.legalCouncilRecommended ? ' ✓' : ''; + + const price = `$${model.pricing.promptPerMillion.toFixed(2)}/$${model.pricing.completionPerMillion.toFixed(2)} per M`; + const context = `${(model.contextLength / 1000).toFixed(0)}K ctx`; + + return `${model.id}${recommended}${chairman} + ${model.name} ${score} ${role} + ${price} | ${context} + ${analysis?.summary || model.description}`; + } + + /** + * Get pricing summary for a configuration + */ + getConfigPricingSummary(config: CouncilConfiguration): { + estimatedCostPerDeliberation: number; + breakdown: Record; + } { + // Estimate tokens per deliberation + // Stage 1: ~2000 prompt + ~1500 completion per model + // Stage 2: ~4000 prompt + ~500 completion per model + // Stage 3: ~6000 prompt + ~2000 completion for chairman + const STAGE1_PROMPT = 2000; + const STAGE1_COMPLETION = 1500; + const STAGE2_PROMPT = 4000; + const STAGE2_COMPLETION = 500; + const STAGE3_PROMPT = 6000; + const STAGE3_COMPLETION = 2000; + + const breakdown: Record = {}; + let totalCost = 0; + + const councilSeats = [config.seatA, config.seatB, config.seatC]; + if (config.seatD) councilSeats.push(config.seatD); + + for (const seatId of councilSeats) { + const model = this.getModel(seatId); + if (!model) continue; + + const stage1PromptCost = (STAGE1_PROMPT / 1_000_000) * model.pricing.promptPerMillion; + const stage1CompletionCost = (STAGE1_COMPLETION / 1_000_000) * model.pricing.completionPerMillion; + const stage2PromptCost = (STAGE2_PROMPT / 1_000_000) * model.pricing.promptPerMillion; + const stage2CompletionCost = (STAGE2_COMPLETION / 1_000_000) * model.pricing.completionPerMillion; + + const promptCost = stage1PromptCost + stage2PromptCost; + const completionCost = stage1CompletionCost + stage2CompletionCost; + + breakdown[seatId] = { promptCost, completionCost }; + totalCost += promptCost + completionCost; + } + + // Chairman (Stage 3) + const chairmanModel = this.getModel(config.chairman); + if (chairmanModel) { + const promptCost = (STAGE3_PROMPT / 1_000_000) * chairmanModel.pricing.promptPerMillion; + const completionCost = (STAGE3_COMPLETION / 1_000_000) * chairmanModel.pricing.completionPerMillion; + breakdown[`${config.chairman} (chairman)`] = { promptCost, completionCost }; + totalCost += promptCost + completionCost; + } + + return { + estimatedCostPerDeliberation: totalCost, + breakdown, + }; + } +} + +/** + * Create and initialize a model selector + */ +export async function createModelSelector(apiKey?: string): Promise { + const selector = new ModelSelector(apiKey); + await selector.initialize(); + return selector; +} + +// Export types for consumers +export type { OpenRouterModel, CouncilConfiguration, ModelSelectorOptions }; diff --git a/src/models/types.ts b/src/models/types.ts new file mode 100644 index 0000000..ea8e16f --- /dev/null +++ b/src/models/types.ts @@ -0,0 +1,97 @@ +/** + * Model Types for LLM Legal Council Model Selector + */ + +export interface ModelPricing { + /** Cost per million input tokens (USD) */ + promptPerMillion: number; + /** Cost per million output tokens (USD) */ + completionPerMillion: number; + /** Per-request fee if applicable */ + requestFee?: number; +} + +export interface ModelCapabilities { + /** Supports function/tool calling */ + functionCalling: boolean; + /** Supports JSON mode output */ + jsonMode: boolean; + /** Supports vision/image input */ + vision: boolean; + /** Supports streaming */ + streaming: boolean; + /** Supports system messages */ + systemPrompt: boolean; +} + +export interface CouncilAnalysis { + /** Overall suitability score for legal council (1-10) */ + councilScore: number; + /** Model's strengths for council deliberation */ + strengths: string[]; + /** Model's weaknesses for council deliberation */ + weaknesses: string[]; + /** Recommended council role */ + recommendedRole: 'lead-analyst' | 'red-team' | 'judge' | 'contrarian' | 'chairman' | 'generalist'; + /** Whether recommended for chairman synthesis */ + chairmanSuitable: boolean; + /** Brief analysis summary */ + summary: string; +} + +export interface OpenRouterModel { + /** Model identifier (e.g., "anthropic/claude-sonnet-4") */ + id: string; + /** Display name */ + name: string; + /** Provider name */ + provider: string; + /** Model description */ + description: string; + /** Context window size in tokens */ + contextLength: number; + /** Maximum output tokens */ + maxOutputTokens?: number; + /** Pricing information */ + pricing: ModelPricing; + /** Model capabilities */ + capabilities: ModelCapabilities; + /** Architecture type */ + architecture?: string; + /** Model tier classification */ + tier: 'frontier' | 'flagship' | 'standard' | 'budget' | 'free'; + /** Whether this model is recommended for legal council */ + legalCouncilRecommended: boolean; + /** Detailed council analysis */ + councilAnalysis?: CouncilAnalysis; + /** Last updated timestamp */ + updatedAt?: string; +} + +export interface CouncilConfiguration { + /** Seat A - typically lead analyst */ + seatA: string; + /** Seat B - typically red team */ + seatB: string; + /** Seat C - typically judge */ + seatC: string; + /** Seat D - optional contrarian */ + seatD?: string; + /** Chairman model for synthesis */ + chairman: string; +} + +export interface ModelSelectorOptions { + /** Filter by tier */ + tiers?: Array<'frontier' | 'flagship' | 'standard' | 'budget' | 'free'>; + /** Filter by minimum context length */ + minContextLength?: number; + /** Filter by maximum price per million tokens */ + maxPricePerMillion?: number; + /** Only show council-recommended models */ + councilRecommendedOnly?: boolean; + /** Only show chairman-suitable models */ + chairmanSuitableOnly?: boolean; + /** Required capabilities */ + requiredCapabilities?: Partial; +}