diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 038b2d3fd..4dfc122ea 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -253,6 +253,8 @@ jobs: tags: | ghcr.io/${{ github.repository }}/api-proxy:${{ needs.bump-version.outputs.version_number }} ghcr.io/${{ github.repository }}/api-proxy:latest + build-args: | + AWF_VERSION=${{ needs.bump-version.outputs.version_number }} cache-from: type=gha,scope=api-proxy cache-to: type=gha,mode=max,scope=api-proxy @@ -523,10 +525,16 @@ jobs: - name: Generate versioned JSON Schema run: | mkdir -p release - node scripts/generate-schema.mjs --version ${{ needs.bump-version.outputs.version }} --print > release/awf-config.v1.schema.json - cp release/awf-config.v1.schema.json release/awf-config.schema.json + # Generate all schemas with versioned $id URLs + node scripts/generate-schema.mjs --version ${{ needs.bump-version.outputs.version }} + # Copy generated files to release/ for upload + cp docs/awf-config.schema.json release/awf-config.schema.json + # Compatibility alias for consumers that previously pinned to awf-config.v1.schema.json + cp docs/awf-config.schema.json release/awf-config.v1.schema.json + cp schemas/audit.schema.json release/audit.schema.json + cp schemas/token-usage.schema.json release/token-usage.schema.json echo "=== Schema preview (first 10 lines) ===" - head -10 release/awf-config.v1.schema.json + head -10 release/awf-config.schema.json - name: Generate checksums run: | @@ -655,6 +663,8 @@ jobs: release/containers.txt release/awf-config.schema.json release/awf-config.v1.schema.json + release/audit.schema.json + release/token-usage.schema.json release/checksums.txt env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/containers/api-proxy/Dockerfile b/containers/api-proxy/Dockerfile index f29142160..6fcd7c7ea 100644 --- a/containers/api-proxy/Dockerfile +++ b/containers/api-proxy/Dockerfile @@ -23,6 +23,13 @@ RUN addgroup -S apiproxy && adduser -S apiproxy -G apiproxy # Switch to non-root user USER apiproxy +# AWF version baked into the image at release build time. +# The release workflow passes --build-arg AWF_VERSION= so that +# token-usage.jsonl records carry the exact api-proxy image version. +# Falls back to "0.0.0-dev" for local/un-versioned builds. +ARG AWF_VERSION=0.0.0-dev +ENV AWF_VERSION=${AWF_VERSION} + # Expose ports # 10000 - OpenAI API proxy (also serves as health check endpoint) # 10001 - Anthropic API proxy diff --git a/containers/api-proxy/token-tracker.js b/containers/api-proxy/token-tracker.js index 16e58a217..f52a4721e 100644 --- a/containers/api-proxy/token-tracker.js +++ b/containers/api-proxy/token-tracker.js @@ -35,6 +35,18 @@ const TOKEN_LOG_FILE = path.join(TOKEN_LOG_DIR, 'token-usage.jsonl'); const DIAG_LOG_FILE = path.join(TOKEN_LOG_DIR, 'token-diag.log'); const DIAG_ENABLED = process.env.AWF_DEBUG_TOKENS === '1'; +// AWF version used to identify schema version in JSONL records. +// Set to the container image version at build time via ARG AWF_VERSION in the Dockerfile +// (baked in by the release workflow with --build-arg AWF_VERSION=). +// Falls back to "0.0.0-dev" for local/un-versioned builds. +const AWF_VERSION = process.env.AWF_VERSION; +if (!AWF_VERSION) { + // Log a warning (to stderr to avoid polluting stdout) when running without the env var. + // This can happen during local development or tests outside the container. + process.stderr.write('{"level":"warn","event":"awf_version_missing","message":"AWF_VERSION env var not set; _schema will use 0.0.0-dev"}\n'); +} +const TOKEN_USAGE_SCHEMA = `token-usage/v${AWF_VERSION || '0.0.0-dev'}`; + let logStream = null; let diagStream = null; @@ -85,7 +97,7 @@ function getLogStream() { } /** - * Validate a token usage record against the token-usage/v1 schema contract. + * Validate a token usage record against the token-usage schema contract. * * Checks that all required fields are present and have the expected types. * Logs a warning and returns false if the record is non-conformant; does @@ -133,11 +145,11 @@ function validateTokenUsageRecord(record) { } } - if (record._schema !== 'token-usage/v1') { + if (!/^token-usage\/v\d+\.\d+\.\d+(-\w+)?$/.test(record._schema)) { logRequest('warn', 'token_record_schema_violation', { request_id: record.request_id, field: '_schema', - expected: 'token-usage/v1', + expected: 'token-usage/v', actual: record._schema, }); return false; @@ -148,7 +160,7 @@ function validateTokenUsageRecord(record) { /** * Write a token usage record to the JSONL log file. - * Validates the record against the token-usage/v1 schema before writing. + * Validates the record against the token-usage schema before writing. * Handles backpressure by dropping writes when the stream buffer is full. */ function writeTokenUsage(record) { @@ -543,7 +555,7 @@ function trackTokenUsage(proxyRes, opts) { // Build log record const record = { - _schema: 'token-usage/v1', + _schema: TOKEN_USAGE_SCHEMA, timestamp: new Date().toISOString(), request_id: requestId, provider, @@ -759,7 +771,7 @@ function trackWebSocketTokenUsage(upstreamSocket, opts) { } const record = { - _schema: 'token-usage/v1', + _schema: TOKEN_USAGE_SCHEMA, timestamp: new Date().toISOString(), request_id: requestId, provider, diff --git a/containers/api-proxy/token-tracker.test.js b/containers/api-proxy/token-tracker.test.js index 2bf042513..63ffbdda2 100644 --- a/containers/api-proxy/token-tracker.test.js +++ b/containers/api-proxy/token-tracker.test.js @@ -1052,7 +1052,7 @@ describe('trackWebSocketTokenUsage', () => { describe('validateTokenUsageRecord', () => { const validRecord = { - _schema: 'token-usage/v1', + _schema: 'token-usage/v0.0.0-dev', timestamp: '2025-01-01T00:00:00.000Z', request_id: 'req-123', provider: 'anthropic', @@ -1075,10 +1075,20 @@ describe('validateTokenUsageRecord', () => { expect(validateTokenUsageRecord({ ...validRecord, response_bytes: 512 })).toBe(true); }); + test('accepts any semver version in _schema', () => { + expect(validateTokenUsageRecord({ ...validRecord, _schema: 'token-usage/v1.2.3' })).toBe(true); + expect(validateTokenUsageRecord({ ...validRecord, _schema: 'token-usage/v0.26.0' })).toBe(true); + expect(validateTokenUsageRecord({ ...validRecord, _schema: 'token-usage/v0.0.0-dev' })).toBe(true); + }); + test('rejects a record with wrong _schema', () => { expect(validateTokenUsageRecord({ ...validRecord, _schema: 'wrong/v99' })).toBe(false); }); + test('rejects a record with non-semver _schema', () => { + expect(validateTokenUsageRecord({ ...validRecord, _schema: 'token-usage/v1' })).toBe(false); + }); + test('rejects a record missing _schema', () => { const { _schema, ...noSchema } = validRecord; expect(validateTokenUsageRecord(noSchema)).toBe(false); @@ -1158,9 +1168,9 @@ describe('token-usage JSONL record schema field', () => { await closeLogStream(); }); - test('writeTokenUsage serializes _schema:"token-usage/v1" into the JSONL stream', () => { + test('writeTokenUsage serializes _schema with semver version into the JSONL stream', () => { const record = { - _schema: 'token-usage/v1', + _schema: 'token-usage/v0.0.0', timestamp: new Date().toISOString(), request_id: 'direct-write-test', provider: 'openai', @@ -1179,11 +1189,11 @@ describe('token-usage JSONL record schema field', () => { expect(mockStream.write).toHaveBeenCalledTimes(1); const parsed = mockStream.writtenRecords[0]; - expect(parsed._schema).toBe('token-usage/v1'); + expect(parsed._schema).toMatch(/^token-usage\/v\d+\.\d+\.\d+(-\w+)?$/); expect(parsed.request_id).toBe('direct-write-test'); }); - test('trackTokenUsage HTTP path writes _schema:"token-usage/v1" to the stream', (done) => { + test('trackTokenUsage HTTP path writes versioned _schema to the stream', (done) => { const proxyRes = new EventEmitter(); proxyRes.headers = { 'content-type': 'application/json' }; proxyRes.statusCode = 200; @@ -1205,13 +1215,13 @@ describe('token-usage JSONL record schema field', () => { setTimeout(() => { expect(mockStream.write).toHaveBeenCalledTimes(1); const parsed = mockStream.writtenRecords[0]; - expect(parsed._schema).toBe('token-usage/v1'); + expect(parsed._schema).toMatch(/^token-usage\/v\d+\.\d+\.\d+(-\w+)?$/); expect(parsed.request_id).toBe('schema-field-http'); done(); }, 20); }); - test('trackWebSocketTokenUsage path writes _schema:"token-usage/v1" to the stream', (done) => { + test('trackWebSocketTokenUsage path writes versioned _schema to the stream', (done) => { const socket = new EventEmitter(); function buildFrame(text) { @@ -1246,9 +1256,65 @@ describe('token-usage JSONL record schema field', () => { setTimeout(() => { expect(mockStream.write).toHaveBeenCalledTimes(1); const parsed = mockStream.writtenRecords[0]; - expect(parsed._schema).toBe('token-usage/v1'); + expect(parsed._schema).toMatch(/^token-usage\/v\d+\.\d+\.\d+(-\w+)?$/); expect(parsed.request_id).toBe('schema-field-ws'); done(); }, 20); }); }); + +// ── AWF_VERSION env var propagated as exact _schema value ───────────── +// +// Uses jest.isolateModules() to load a fresh token-tracker instance with a +// controlled AWF_VERSION env var so the test verifies the exact _schema value +// emitted, not just the semver pattern. + +describe('token-usage _schema exact version from AWF_VERSION', () => { + test('emits exact AWF_VERSION in _schema field', (done) => { + const origVersion = process.env.AWF_VERSION; + process.env.AWF_VERSION = '9.8.7'; + + // Load an isolated copy of token-tracker with AWF_VERSION=9.8.7 already set + let isolated; + jest.isolateModules(() => { + isolated = require('./token-tracker'); + }); + + // Restore env var right away — the isolated module already captured it + process.env.AWF_VERSION = origVersion; + + const mockStream = makeMockStream(); + const mkdirSpy = jest.spyOn(fs, 'mkdirSync').mockReturnValue(undefined); + const writeStreamSpy = jest.spyOn(fs, 'createWriteStream').mockReturnValue(mockStream); + + const proxyRes = new EventEmitter(); + proxyRes.headers = { 'content-type': 'application/json' }; + proxyRes.statusCode = 200; + + isolated.trackTokenUsage(proxyRes, { + requestId: 'exact-version-test', + provider: 'openai', + path: '/v1/chat/completions', + startTime: Date.now(), + metrics: null, + }); + + proxyRes.emit('data', Buffer.from(JSON.stringify({ + model: 'gpt-4o', + usage: { prompt_tokens: 10, completion_tokens: 5, total_tokens: 15 }, + }))); + proxyRes.emit('end'); + + setTimeout(async () => { + mkdirSpy.mockRestore(); + writeStreamSpy.mockRestore(); + await isolated.closeLogStream(); + + expect(mockStream.write).toHaveBeenCalledTimes(1); + const parsed = mockStream.writtenRecords[0]; + expect(parsed._schema).toBe('token-usage/v9.8.7'); + expect(parsed.request_id).toBe('exact-version-test'); + done(); + }, 20); + }); +}); diff --git a/docs/awf-config.schema.json b/docs/awf-config.schema.json index 5cd6a0926..b21cb5021 100644 --- a/docs/awf-config.schema.json +++ b/docs/awf-config.schema.json @@ -2,7 +2,6 @@ "$schema": "https://json-schema.org/draft/2020-12/schema", "$id": "https://raw.githubusercontent.com/github/gh-aw-firewall/main/docs/awf-config.schema.json", "title": "AWF Configuration", - "version": "1", "description": "JSON/YAML configuration for awf CLI. CLI flags override config file values. See https://github.com/github/gh-aw-firewall for documentation.", "type": "object", "additionalProperties": false, diff --git a/docs/awf-config.v1.schema.json b/docs/awf-config.v1.schema.json deleted file mode 100644 index ed187c7fc..000000000 --- a/docs/awf-config.v1.schema.json +++ /dev/null @@ -1,334 +0,0 @@ -{ - "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/github/gh-aw-firewall/main/docs/awf-config.v1.schema.json", - "title": "AWF Configuration", - "version": "1", - "description": "JSON/YAML configuration for awf CLI. CLI flags override config file values. See https://github.com/github/gh-aw-firewall for documentation.", - "type": "object", - "additionalProperties": false, - "properties": { - "$schema": { - "type": "string", - "description": "JSON Schema URL for IDE validation and autocomplete." - }, - "network": { - "type": "object", - "description": "Network egress configuration.", - "additionalProperties": false, - "properties": { - "allowDomains": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Domains that the agent is allowed to reach. Both the bare domain and all subdomains are permitted (e.g. \"github.com\" also allows \"api.github.com\")." - }, - "blockDomains": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Domains that are explicitly blocked, overriding allowDomains." - }, - "dnsServers": { - "type": "array", - "items": { - "type": "string" - }, - "description": "DNS servers to use inside the container. Defaults to Google DNS (8.8.8.8, 8.8.4.4). Accepts IPv4 and IPv6 addresses." - }, - "upstreamProxy": { - "type": "string", - "description": "Upstream HTTP proxy URL (e.g. \"http://proxy.corp.example.com:8080\"). When set, the AWF Squid proxy forwards traffic through this proxy." - } - } - }, - "apiProxy": { - "type": "object", - "description": "API proxy sidecar configuration. The sidecar injects real API credentials so the agent never has direct access to them.", - "additionalProperties": false, - "properties": { - "enabled": { - "type": "boolean", - "description": "Enable the API proxy sidecar container." - }, - "enableOpenCode": { - "type": "boolean", - "description": "Enable the OpenCode API proxy endpoint (port 10004)." - }, - "anthropicAutoCache": { - "type": "boolean", - "description": "Automatically apply Anthropic prompt-cache optimizations on /v1/messages requests." - }, - "anthropicCacheTailTtl": { - "type": "string", - "enum": [ - "5m", - "1h" - ], - "description": "TTL for Anthropic cache tail optimization. Only applies when anthropicAutoCache is enabled. Allowed values: \"5m\" or \"1h\"." - }, - "targets": { - "type": "object", - "description": "Override upstream API endpoints for each provider.", - "additionalProperties": false, - "properties": { - "openai": { - "$ref": "#/$defs/providerTarget", - "description": "OpenAI API target override." - }, - "anthropic": { - "$ref": "#/$defs/providerTarget", - "description": "Anthropic API target override." - }, - "copilot": { - "$ref": "#/$defs/providerHostOnlyTarget", - "description": "GitHub Copilot API target override (basePath not supported)." - }, - "gemini": { - "$ref": "#/$defs/providerTarget", - "description": "Google Gemini API target override." - } - } - }, - "models": { - "type": "object", - "description": "Model alias mapping. Keys are canonical model names; values are arrays of alternative names or patterns that should be rewritten to the canonical name.", - "additionalProperties": { - "type": "array", - "items": { - "type": "string" - } - } - } - } - }, - "security": { - "type": "object", - "description": "Security and isolation configuration.", - "additionalProperties": false, - "properties": { - "sslBump": { - "type": "boolean", - "description": "Enable SSL bumping (TLS interception) in the Squid proxy. Requires a custom CA certificate." - }, - "enableDlp": { - "type": "boolean", - "description": "Enable Data Loss Prevention (DLP) inspection of outbound traffic." - }, - "enableHostAccess": { - "type": "boolean", - "description": "Mount the host filesystem (read-only for system paths, read-write for the workspace). Enabled by default; set to false to run without host filesystem access." - }, - "allowHostPorts": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ], - "description": "Host TCP ports the agent may connect to (e.g. local dev services). Accepts a single port string or an array of port strings." - }, - "allowHostServicePorts": { - "oneOf": [ - { - "type": "string" - }, - { - "type": "array", - "items": { - "type": "string" - } - } - ], - "description": "Named service ports on the host that the agent may connect to. Accepts a single port string or an array of port strings." - }, - "difcProxy": { - "type": "object", - "description": "DIFC (Data-in-Flight Control) proxy configuration.", - "additionalProperties": false, - "properties": { - "host": { - "type": "string", - "description": "DIFC proxy host." - }, - "caCert": { - "type": "string", - "description": "Path to the CA certificate for DIFC proxy TLS verification." - } - } - } - } - }, - "container": { - "type": "object", - "description": "Container and Docker configuration.", - "additionalProperties": false, - "properties": { - "memoryLimit": { - "type": "string", - "description": "Docker memory limit for the agent container (e.g. \"4g\", \"512m\"). Uses Docker memory limit syntax." - }, - "agentTimeout": { - "type": "integer", - "minimum": 1, - "description": "Maximum time (in minutes) the agent command is allowed to run." - }, - "enableDind": { - "type": "boolean", - "description": "Enable Docker-in-Docker support inside the agent container." - }, - "workDir": { - "type": "string", - "description": "Host path used as the AWF working directory for generated configs and logs. Defaults to a temporary directory." - }, - "containerWorkDir": { - "type": "string", - "description": "Working directory inside the agent container." - }, - "imageRegistry": { - "type": "string", - "description": "Container image registry to pull from. Defaults to \"ghcr.io/github/gh-aw-firewall\"." - }, - "imageTag": { - "type": "string", - "description": "Container image tag to use. Defaults to \"latest\"." - }, - "skipPull": { - "type": "boolean", - "description": "Skip pulling container images (use locally cached images)." - }, - "buildLocal": { - "type": "boolean", - "description": "Build container images from source instead of pulling from the registry." - }, - "agentImage": { - "type": "string", - "description": "Override the agent container image (e.g. for a GitHub Actions parity image)." - }, - "tty": { - "type": "boolean", - "description": "Allocate a pseudo-TTY for the agent container." - }, - "dockerHost": { - "type": "string", - "description": "Docker daemon socket or host to connect to (e.g. \"unix:///var/run/docker.sock\")." - } - } - }, - "environment": { - "type": "object", - "description": "Environment variable propagation into the agent container.", - "additionalProperties": false, - "properties": { - "envFile": { - "type": "string", - "description": "Path to a .env file whose variables are injected into the agent container." - }, - "envAll": { - "type": "boolean", - "description": "Forward all host environment variables into the agent container. Use with caution — may expose secrets." - }, - "excludeEnv": { - "type": "array", - "items": { - "type": "string" - }, - "description": "Environment variable names to exclude when envAll is true." - } - } - }, - "logging": { - "type": "object", - "description": "Logging and diagnostics configuration.", - "additionalProperties": false, - "properties": { - "logLevel": { - "type": "string", - "enum": [ - "debug", - "info", - "warn", - "error" - ], - "description": "Log verbosity level. Defaults to \"info\"." - }, - "diagnosticLogs": { - "type": "boolean", - "description": "Enable diagnostic logging (Squid access logs, iptables logs). Logs are written to the work directory." - }, - "auditDir": { - "type": "string", - "description": "Directory path for audit logs." - }, - "proxyLogsDir": { - "type": "string", - "description": "Directory path for Squid proxy access logs." - }, - "sessionStateDir": { - "type": "string", - "description": "Directory path for agent session state (e.g. conversation history). Set to \"/tmp/gh-aw/sandbox/agent/session-state\" for Copilot agent runs." - } - } - }, - "rateLimiting": { - "type": "object", - "description": "Egress rate limiting configuration.", - "additionalProperties": false, - "properties": { - "enabled": { - "type": "boolean", - "description": "Enable egress rate limiting." - }, - "requestsPerMinute": { - "type": "integer", - "minimum": 1, - "description": "Maximum number of HTTP requests per minute." - }, - "requestsPerHour": { - "type": "integer", - "minimum": 1, - "description": "Maximum number of HTTP requests per hour." - }, - "bytesPerMinute": { - "type": "integer", - "minimum": 1, - "description": "Maximum number of bytes transferred per minute." - } - } - } - }, - "$defs": { - "providerTarget": { - "type": "object", - "description": "API provider target override.", - "additionalProperties": false, - "properties": { - "host": { - "type": "string", - "description": "Override the provider API host." - }, - "basePath": { - "type": "string", - "description": "Override the provider API base path." - } - } - }, - "providerHostOnlyTarget": { - "type": "object", - "description": "API provider target override (host only; basePath not supported).", - "additionalProperties": false, - "properties": { - "host": { - "type": "string", - "description": "Override the provider API host." - } - } - } - } -} diff --git a/docs/releasing.md b/docs/releasing.md index fee3fb8f6..b7f36bf68 100644 --- a/docs/releasing.md +++ b/docs/releasing.md @@ -44,7 +44,7 @@ Once the workflow completes: - Linux arm64 binary (`awf-linux-arm64`) - NPM tarball (`awf.tgz`) - Checksums file (`checksums.txt`) - - JSON Schema files (`awf-config.schema.json`, `awf-config.v1.schema.json`) + - JSON Schema files (`awf-config.schema.json`, `audit.schema.json`, `token-usage.schema.json`) - Installation instructions with GHCR image references 3. Go to **Packages** page (in repository) 4. Verify Docker images are published: @@ -63,25 +63,34 @@ Each release includes: - `awf-linux-arm64` - Linux arm64 standalone executable - `awf.tgz` - NPM package tarball (alternative installation method) - `checksums.txt` - SHA256 checksums for all files -- `awf-config.schema.json` - AWF config JSON Schema (latest alias, same content as `awf-config.v1.schema.json`) -- `awf-config.v1.schema.json` - AWF config JSON Schema, version 1 (stable versioned copy) +- `awf-config.schema.json` - AWF config JSON Schema +- `awf-config.v1.schema.json` - **Deprecated alias** of `awf-config.schema.json` (kept for backward compatibility) +- `audit.schema.json` - AWF audit JSONL record JSON Schema +- `token-usage.schema.json` - AWF token-usage JSONL record JSON Schema ### JSON Schema versioning -Each release generates the schema with a `$id` URL that includes the release tag, creating a stable, pinnable reference: +Each release generates all three schemas with a `$id` URL that includes the release tag, creating stable, pinnable references: ``` -https://github.com/github/gh-aw-firewall/releases/download/v0.23.1/awf-config.v1.schema.json +https://github.com/github/gh-aw-firewall/releases/download/v0.26.0/awf-config.schema.json +https://github.com/github/gh-aw-firewall/releases/download/v0.26.0/audit.schema.json +https://github.com/github/gh-aw-firewall/releases/download/v0.26.0/token-usage.schema.json ``` -The unversioned `awf-config.schema.json` asset is a copy of the v1 schema for convenience. External consumers (e.g. the gh-aw compiler) should pin to the versioned URL or the stable raw URL: +External consumers (e.g. the gh-aw compiler) should pin to the release URL or the stable raw URL: -| Reference | URL | -|-----------|-----| -| Pinned to a specific release tag | `https://github.com/github/gh-aw-firewall/releases/download//awf-config.v1.schema.json` | -| Always-latest from `main` branch | `https://raw.githubusercontent.com/github/gh-aw-firewall/main/docs/awf-config.v1.schema.json` | +| Schema | Pinned to release tag | Always-latest from `main` | +|--------|----------------------|--------------------------| +| Config | `https://github.com/github/gh-aw-firewall/releases/download//awf-config.schema.json` | `https://raw.githubusercontent.com/github/gh-aw-firewall/main/docs/awf-config.schema.json` | +| Audit | `https://github.com/github/gh-aw-firewall/releases/download//audit.schema.json` | `https://raw.githubusercontent.com/github/gh-aw-firewall/main/schemas/audit.schema.json` | +| Token usage | `https://github.com/github/gh-aw-firewall/releases/download//token-usage.schema.json` | `https://raw.githubusercontent.com/github/gh-aw-firewall/main/schemas/token-usage.schema.json` | -**Schema version bumping:** The schema version (`"version": "1"` in the schema body) must be incremented whenever breaking changes are made to the config surface (removed fields, changed types, stricter constraints). Non-breaking additions do not require a version bump. When the version is bumped (e.g. from `1` → `2`), a new file `awf-config.v2.schema.json` should be introduced in `docs/` and `scripts/generate-schema.mjs` updated accordingly. +> **Migration note:** The previously documented URL `…/releases/download//awf-config.v1.schema.json` and the raw URL `…/main/docs/awf-config.v1.schema.json` are now **deprecated**. `awf-config.v1.schema.json` continues to be published as a compatibility alias (identical content to `awf-config.schema.json`) so existing consumers are not broken, but new consumers should use `awf-config.schema.json` instead. The `docs/awf-config.v1.schema.json` file in the repository has been removed; use `docs/awf-config.schema.json` for the always-latest copy. + +**JSONL `_schema` field:** Each JSONL record embeds the AWF version in its `_schema` field, e.g. `"_schema": "audit/v0.26.0"`. For audit logs, the version reflects the AWF CLI version that configured the Squid proxy. For token-usage logs, the version is baked into the api-proxy container image at release build time (via `--build-arg AWF_VERSION=…`), so it correctly reflects the api-proxy image version even when `--image-tag` pins the proxy to a different release. Dev/local builds use `audit/v` and `token-usage/v0.0.0-dev` respectively. Consumers should use a prefix match (`_schema.startsWith("audit/")`) rather than an exact match to handle any version gracefully. + +**Schema evolution:** Breaking changes (field removal, rename, type change, new required field) should be documented in the changelog. Since the repo version is used as the schema version, consumers can pin to a specific release tag to ensure compatibility. ### GitHub Container Registry (GHCR) Docker images are published to `ghcr.io/github/gh-aw-firewall`: diff --git a/samples/audit/audit.jsonl b/samples/audit/audit.jsonl index 821b1382a..15937eddf 100644 --- a/samples/audit/audit.jsonl +++ b/samples/audit/audit.jsonl @@ -1,3 +1,3 @@ -{"_schema":"audit/v1","ts":1774290908.910,"client":"172.30.0.20","host":"api.github.com:443","dest":"140.82.116.5:443","method":"CONNECT","status":200,"decision":"TCP_TUNNEL","url":"api.github.com:443"} -{"_schema":"audit/v1","ts":1774290909.180,"client":"172.30.0.20","host":"api.github.com:443","dest":"140.82.116.5:443","method":"CONNECT","status":200,"decision":"TCP_TUNNEL","url":"api.github.com:443"} -{"_schema":"audit/v1","ts":1774290909.186,"client":"172.30.0.20","host":"evil.example.com:443","dest":"-:-","method":"CONNECT","status":403,"decision":"TCP_DENIED","url":"evil.example.com:443"} +{"_schema":"audit/v0.23.1","ts":1774290908.910,"client":"172.30.0.20","host":"api.github.com:443","dest":"140.82.116.5:443","method":"CONNECT","status":200,"decision":"TCP_TUNNEL","url":"api.github.com:443"} +{"_schema":"audit/v0.23.1","ts":1774290909.180,"client":"172.30.0.20","host":"api.github.com:443","dest":"140.82.116.5:443","method":"CONNECT","status":200,"decision":"TCP_TUNNEL","url":"api.github.com:443"} +{"_schema":"audit/v0.23.1","ts":1774290909.186,"client":"172.30.0.20","host":"evil.example.com:443","dest":"-:-","method":"CONNECT","status":403,"decision":"TCP_DENIED","url":"evil.example.com:443"} diff --git a/schemas/README.md b/schemas/README.md index d9ccd303e..309e0e605 100644 --- a/schemas/README.md +++ b/schemas/README.md @@ -1,29 +1,51 @@ # AWF JSONL Schemas -This directory contains versioned [JSON Schema](https://json-schema.org/) files for the JSONL artifact files emitted by AWF at runtime. +This directory contains [JSON Schema](https://json-schema.org/) files for the JSONL artifact files emitted by AWF at runtime. ## Files | Schema file | JSONL file | Writer | |---|---|---| -| [`token-usage.v1.schema.json`](token-usage.v1.schema.json) | `token-usage.jsonl` | `containers/api-proxy/token-tracker.js` | -| [`audit.v1.schema.json`](audit.v1.schema.json) | `audit.jsonl` | Squid proxy (`src/squid-config.ts`) | +| [`token-usage.schema.json`](token-usage.schema.json) | `token-usage.jsonl` | `containers/api-proxy/token-tracker.js` | +| [`audit.schema.json`](audit.schema.json) | `audit.jsonl` | Squid proxy (`src/squid-config.ts`) | ## Schema versioning policy -- **Additive changes** (new optional fields) → update the existing `v1` schema, no version bump required. -- **Breaking changes** (field removal, rename, type change, new required field) → create a new `v2` schema file and bump the `_schema` value in the writer. +Schema files do not carry an independent version suffix. Instead, the repo release tag is used as the version: + +- The `$id` field in each schema is updated at release time to a stable release download URL (e.g. `https://github.com/github/gh-aw-firewall/releases/download/v0.26.0/audit.schema.json`). +- The `_schema` wire-format field in each JSONL record embeds the repo version (e.g. `"_schema": "audit/v0.26.0"`). +- **Additive changes** (new optional fields) → update the schema file directly; no special action required. +- **Breaking changes** (field removal, rename, type change, new required field) → document in the changelog; consumers should pin to a specific release tag. ## Record identification -Every JSONL record includes a `_schema` field that identifies the schema name and version: +Every JSONL record includes a `_schema` field that identifies the record type and the AWF version that produced it: ```json -{ "_schema": "token-usage/v1", "timestamp": "2025-01-01T00:00:00.000Z", ... } -{ "_schema": "audit/v1", "ts": 1761074374.646, ... } +{ "_schema": "token-usage/v0.26.0", "timestamp": "2025-01-01T00:00:00.000Z", ... } +{ "_schema": "audit/v0.26.0", "ts": 1761074374.646, ... } +``` + +The `_schema` field uses the pattern `/v`. Consumers should use a prefix match (`_schema.startsWith("audit/")`) rather than an exact match to handle future versions gracefully. + +## Release download URLs + +Each release publishes all schemas as release assets: + +``` +https://github.com/github/gh-aw-firewall/releases/download//audit.schema.json +https://github.com/github/gh-aw-firewall/releases/download//token-usage.schema.json +https://github.com/github/gh-aw-firewall/releases/download//awf-config.schema.json ``` -Consumers should check `_schema` before parsing fields so they can handle future versions gracefully. +For always-latest (main branch) references: + +``` +https://raw.githubusercontent.com/github/gh-aw-firewall/main/schemas/audit.schema.json +https://raw.githubusercontent.com/github/gh-aw-firewall/main/schemas/token-usage.schema.json +https://raw.githubusercontent.com/github/gh-aw-firewall/main/docs/awf-config.schema.json +``` ## Validation @@ -35,6 +57,6 @@ npm install -g ajv-cli # Validate all records in audit.jsonl while IFS= read -r line; do - echo "$line" | ajv validate -s schemas/audit.v1.schema.json -d /dev/stdin + echo "$line" | ajv validate -s schemas/audit.schema.json -d /dev/stdin done < /path/to/audit.jsonl ``` diff --git a/schemas/audit.v1.schema.json b/schemas/audit.schema.json similarity index 88% rename from schemas/audit.v1.schema.json rename to schemas/audit.schema.json index a803848ba..72a5da650 100644 --- a/schemas/audit.v1.schema.json +++ b/schemas/audit.schema.json @@ -1,9 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/github/gh-aw-firewall/schemas/audit.v1.schema.json", + "$id": "https://raw.githubusercontent.com/github/gh-aw-firewall/main/schemas/audit.schema.json", "title": "AWF Audit Log Record", "description": "A single L7 HTTP/HTTPS traffic decision record emitted to audit.jsonl by the AWF Squid proxy.", - "version": "1", "type": "object", "required": [ "_schema", @@ -20,8 +19,8 @@ "properties": { "_schema": { "type": "string", - "const": "audit/v1", - "description": "Schema identifier and version for this record." + "pattern": "^audit/v\\d+\\.\\d+\\.\\d+(-\\w+)?$", + "description": "Schema identifier and version for this record (e.g. \"audit/v0.26.0\"). Dev builds use \"audit/v0.0.0-dev\"." }, "ts": { "type": "number", diff --git a/schemas/token-usage.v1.schema.json b/schemas/token-usage.schema.json similarity index 87% rename from schemas/token-usage.v1.schema.json rename to schemas/token-usage.schema.json index ebfc02211..190a58304 100644 --- a/schemas/token-usage.v1.schema.json +++ b/schemas/token-usage.schema.json @@ -1,9 +1,8 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://github.com/github/gh-aw-firewall/schemas/token-usage.v1.schema.json", + "$id": "https://raw.githubusercontent.com/github/gh-aw-firewall/main/schemas/token-usage.schema.json", "title": "AWF Token Usage Record", "description": "A single per-API-call token usage record emitted to token-usage.jsonl by the AWF api-proxy sidecar.", - "version": "1", "type": "object", "required": [ "_schema", @@ -24,8 +23,8 @@ "properties": { "_schema": { "type": "string", - "const": "token-usage/v1", - "description": "Schema identifier and version for this record." + "pattern": "^token-usage/v\\d+\\.\\d+\\.\\d+(-\\w+)?$", + "description": "Schema identifier and version for this record (e.g. \"token-usage/v0.26.0\"). Dev builds use \"token-usage/v0.0.0-dev\"." }, "timestamp": { "type": "string", @@ -38,7 +37,14 @@ }, "provider": { "type": "string", - "enum": ["anthropic", "openai", "copilot", "gemini", "opencode", "unknown"], + "enum": [ + "anthropic", + "openai", + "copilot", + "gemini", + "opencode", + "unknown" + ], "description": "LLM provider that handled this request." }, "model": { diff --git a/scripts/generate-schema.mjs b/scripts/generate-schema.mjs index 36f68ed98..aa5f5ce09 100644 --- a/scripts/generate-schema.mjs +++ b/scripts/generate-schema.mjs @@ -1,24 +1,26 @@ #!/usr/bin/env node /** - * Generates the JSON Schema for the AWF config file. + * Generates the JSON Schema files for AWF. * * Usage: - * node scripts/generate-schema.mjs # writes docs/awf-config.schema.json and docs/awf-config.v1.schema.json - * node scripts/generate-schema.mjs --version v0.23.1 # embeds a versioned $id in release output - * node scripts/generate-schema.mjs --print # prints to stdout + * node scripts/generate-schema.mjs # writes docs/awf-config.schema.json and src/awf-config-schema.json + * node scripts/generate-schema.mjs --version v0.26.0 # embeds a versioned $id in release output + * node scripts/generate-schema.mjs --print # prints awf-config schema to stdout * - * Output files: - * docs/awf-config.v1.schema.json — stable versioned file (canonical source) - * docs/awf-config.schema.json — latest alias (always points to current version content) + * Output files (non-print mode): + * docs/awf-config.schema.json — config schema (canonical source) * src/awf-config-schema.json — bundleable copy for runtime validation + * schemas/audit.schema.json — audit JSONL schema with versioned $id + * schemas/token-usage.schema.json — token-usage JSONL schema with versioned $id * + * The schema version is embedded in the $id URL using the repo release tag. * The schema reflects the validated config surface defined in src/config-file.ts * (validateAwfFileConfig), not just the AwfFileConfig TypeScript interface. * When validation rules change (e.g. new fields, enum constraints), update this script to match. */ -import { writeFileSync, mkdirSync } from 'fs'; +import { writeFileSync, mkdirSync, readFileSync } from 'fs'; import { join, dirname } from 'path'; import { fileURLToPath } from 'url'; @@ -51,20 +53,24 @@ const version = versionIdx !== -1 ? args[versionIdx + 1] : null; const printOnly = args.includes('--print'); // --- Build the schema --- -// Versioned $id (stable reference for v1 of the config schema) -const schemaV1Id = version - ? `https://github.com/github/gh-aw-firewall/releases/download/${version}/awf-config.v1.schema.json` - : 'https://raw.githubusercontent.com/github/gh-aw-firewall/main/docs/awf-config.v1.schema.json'; - -// Unversioned "latest" $id (always points to the current schema) -const schemaLatestId = version +// Config schema $id (release URL when version provided, raw URL otherwise) +const schemaConfigId = version ? `https://github.com/github/gh-aw-firewall/releases/download/${version}/awf-config.schema.json` : 'https://raw.githubusercontent.com/github/gh-aw-firewall/main/docs/awf-config.schema.json'; +// JSONL schema $id for audit records +const schemaAuditId = version + ? `https://github.com/github/gh-aw-firewall/releases/download/${version}/audit.schema.json` + : 'https://raw.githubusercontent.com/github/gh-aw-firewall/main/schemas/audit.schema.json'; + +// JSONL schema $id for token-usage records +const schemaTokenUsageId = version + ? `https://github.com/github/gh-aw-firewall/releases/download/${version}/token-usage.schema.json` + : 'https://raw.githubusercontent.com/github/gh-aw-firewall/main/schemas/token-usage.schema.json'; + /** @type {object} */ const schemaBody = { title: 'AWF Configuration', - version: '1', description: 'JSON/YAML configuration for awf CLI. CLI flags override config file values. ' + 'See https://github.com/github/gh-aw-firewall for documentation.', @@ -402,41 +408,65 @@ const schemaBody = { }, }; -// Compose the versioned schema (stable, canonical) and the latest alias -const schemaV1 = { - $schema: 'https://json-schema.org/draft/2020-12/schema', - $id: schemaV1Id, - ...schemaBody, -}; +// Read JSONL schemas from the schemas/ directory and update their $id fields +function buildJsonlSchema(schemaFile, newId) { + const schemaDir = join(projectRoot, 'schemas'); + const schemaPath = join(schemaDir, schemaFile); + let raw; + try { + raw = readFileSync(schemaPath, 'utf8'); + } catch (err) { + console.error(`Error: could not read schema file '${schemaPath}': ${err.message}`); + process.exit(1); + } + let schema; + try { + schema = JSON.parse(raw); + } catch (err) { + console.error(`Error: could not parse JSON in '${schemaPath}': ${err.message}`); + process.exit(1); + } + schema.$id = newId; + return JSON.stringify(schema, null, 2) + '\n'; +} -const schemaLatest = { +// Compose the config schema +const schemaConfig = { $schema: 'https://json-schema.org/draft/2020-12/schema', - $id: schemaLatestId, + $id: schemaConfigId, ...schemaBody, }; -const outputV1 = JSON.stringify(schemaV1, null, 2) + '\n'; -const outputLatest = JSON.stringify(schemaLatest, null, 2) + '\n'; +const outputConfig = JSON.stringify(schemaConfig, null, 2) + '\n'; if (printOnly) { - // --print emits the versioned (v1) schema to stdout - process.stdout.write(outputV1); + // --print emits the config schema to stdout + process.stdout.write(outputConfig); } else { const docsDir = join(projectRoot, 'docs'); mkdirSync(docsDir, { recursive: true }); - // Stable versioned file (canonical) - const v1Path = join(docsDir, 'awf-config.v1.schema.json'); - writeFileSync(v1Path, outputV1); - console.log(`Schema written to ${v1Path}`); - - // Unversioned "latest" alias - const latestPath = join(docsDir, 'awf-config.schema.json'); - writeFileSync(latestPath, outputLatest); - console.log(`Schema written to ${latestPath}`); + // Config schema + const configPath = join(docsDir, 'awf-config.schema.json'); + writeFileSync(configPath, outputConfig); + console.log(`Schema written to ${configPath}`); // Also write to src/ for runtime loading (loaded dynamically by schema-validator.ts at startup) const srcPath = join(projectRoot, 'src', 'awf-config-schema.json'); - writeFileSync(srcPath, outputV1); + writeFileSync(srcPath, outputConfig); console.log(`Schema written to ${srcPath}`); + + // JSONL schemas — update $id with release/raw URL + const schemasDir = join(projectRoot, 'schemas'); + mkdirSync(schemasDir, { recursive: true }); + + const auditOutput = buildJsonlSchema('audit.schema.json', schemaAuditId); + const auditPath = join(schemasDir, 'audit.schema.json'); + writeFileSync(auditPath, auditOutput); + console.log(`Schema written to ${auditPath}`); + + const tokenUsageOutput = buildJsonlSchema('token-usage.schema.json', schemaTokenUsageId); + const tokenUsagePath = join(schemasDir, 'token-usage.schema.json'); + writeFileSync(tokenUsagePath, tokenUsageOutput); + console.log(`Schema written to ${tokenUsagePath}`); } diff --git a/src/awf-config-schema.json b/src/awf-config-schema.json index ed187c7fc..b21cb5021 100644 --- a/src/awf-config-schema.json +++ b/src/awf-config-schema.json @@ -1,8 +1,7 @@ { "$schema": "https://json-schema.org/draft/2020-12/schema", - "$id": "https://raw.githubusercontent.com/github/gh-aw-firewall/main/docs/awf-config.v1.schema.json", + "$id": "https://raw.githubusercontent.com/github/gh-aw-firewall/main/docs/awf-config.schema.json", "title": "AWF Configuration", - "version": "1", "description": "JSON/YAML configuration for awf CLI. CLI flags override config file values. See https://github.com/github/gh-aw-firewall for documentation.", "type": "object", "additionalProperties": false, diff --git a/src/docker-manager.ts b/src/docker-manager.ts index 5f3b3cc6a..b2b70a593 100644 --- a/src/docker-manager.ts +++ b/src/docker-manager.ts @@ -1790,6 +1790,11 @@ export function generateDockerCompose( // Forward GITHUB_API_URL so api-proxy can route /models to the correct GitHub REST API // target on GHES/GHEC (e.g. api.mycompany.ghe.com instead of api.github.com) ...(process.env.GITHUB_API_URL && { GITHUB_API_URL: process.env.GITHUB_API_URL }), + // Note: AWF_VERSION is intentionally NOT forwarded here. It is baked into the api-proxy + // container image at release build time (via --build-arg AWF_VERSION=...), so the + // token-usage.jsonl _schema field reflects the api-proxy image version rather than + // the CLI version. This ensures correct versioning when --image-tag pins the proxy + // to a different release. // Route through Squid to respect domain whitelisting HTTP_PROXY: `http://${networkConfig.squidIp}:${SQUID_PORT}`, HTTPS_PROXY: `http://${networkConfig.squidIp}:${SQUID_PORT}`, diff --git a/src/logs/log-parser.test.ts b/src/logs/log-parser.test.ts index 3f5e126d7..0dc97d7c2 100644 --- a/src/logs/log-parser.test.ts +++ b/src/logs/log-parser.test.ts @@ -291,7 +291,7 @@ describe('log-parser', () => { }); it('should parse records that include the _schema field', () => { - const line = '{"_schema":"audit/v1","ts":1774290908.910,"client":"172.30.0.20","host":"api.github.com:443","dest":"140.82.116.5:443","method":"CONNECT","status":200,"decision":"TCP_TUNNEL","url":"api.github.com:443"}'; + const line = '{"_schema":"audit/v0.23.1","ts":1774290908.910,"client":"172.30.0.20","host":"api.github.com:443","dest":"140.82.116.5:443","method":"CONNECT","status":200,"decision":"TCP_TUNNEL","url":"api.github.com:443"}'; const entry = parseAuditJsonlLine(line); expect(entry).not.toBeNull(); diff --git a/src/schema.test.ts b/src/schema.test.ts index 92d574eb1..465f1ba9e 100644 --- a/src/schema.test.ts +++ b/src/schema.test.ts @@ -3,7 +3,6 @@ import * as path from 'path'; import Ajv2020 from 'ajv/dist/2020'; const schemaPath = path.join(__dirname, '..', 'docs', 'awf-config.schema.json'); -const schemaV1Path = path.join(__dirname, '..', 'docs', 'awf-config.v1.schema.json'); describe('awf-config.schema.json', () => { let schema: Record; @@ -13,8 +12,6 @@ describe('awf-config.schema.json', () => { const raw = fs.readFileSync(schemaPath, 'utf8'); schema = JSON.parse(raw) as Record; const ajv = new Ajv2020(); - // 'version' is a metadata keyword; register it so strict mode doesn't reject the schema. - ajv.addKeyword({ keyword: 'version' }); validate = ajv.compile(schema); }); @@ -29,10 +26,6 @@ describe('awf-config.schema.json', () => { expect(schema.additionalProperties).toBe(false); }); - it('has a version field', () => { - expect(schema.version).toBe('1'); - }); - it('covers all AwfFileConfig top-level fields', () => { const properties = schema.properties as Record; expect(Object.keys(properties)).toEqual( @@ -191,17 +184,4 @@ describe('awf-config.schema.json', () => { delete srcRest.$id; expect(srcRest).toEqual(docsRest); }); - - it('docs/awf-config.v1.schema.json stays in sync with docs/awf-config.schema.json', () => { - const v1Schema = JSON.parse(fs.readFileSync(schemaV1Path, 'utf8')) as Record; - // v1 schema must have a versioned $id and match the latest schema (ignoring $id) - expect(v1Schema.version).toBe('1'); - expect(typeof v1Schema.$id).toBe('string'); - expect(v1Schema.$id as string).toContain('v1'); - const latestRest = { ...schema }; - delete latestRest.$id; - const v1Rest = { ...v1Schema }; - delete v1Rest.$id; - expect(v1Rest).toEqual(latestRest); - }); }); diff --git a/src/squid-config.test.ts b/src/squid-config.test.ts index b800274f8..b5797f0e0 100644 --- a/src/squid-config.test.ts +++ b/src/squid-config.test.ts @@ -1,5 +1,7 @@ import { generateSquidConfig, generatePolicyManifest } from './squid-config'; import { SquidConfig } from './types'; +// eslint-disable-next-line @typescript-eslint/no-require-imports +const { version: AWF_VERSION } = require('../package.json') as { version: string }; describe('defense-in-depth: rejects injected values', () => { const defaultPort = 3128; @@ -580,15 +582,15 @@ describe('generateSquidConfig', () => { expect(result).toContain('access_log /var/log/squid/audit.jsonl audit_jsonl'); }); - it('audit_jsonl logformat should include _schema:"audit/v1" field', () => { + it('audit_jsonl logformat should include versioned _schema field matching the package.json version', () => { const config: SquidConfig = { domains: ['example.com'], port: defaultPort, }; const result = generateSquidConfig(config); - // The audit_jsonl logformat line must embed the schema identifier so that - // every emitted record carries the version tag. - expect(result).toMatch(/logformat audit_jsonl \{.*"_schema":"audit\/v1"/); + // The audit_jsonl logformat line must embed the exact CLI version so that + // every emitted record carries the correct schema identifier. + expect(result).toContain(`"_schema":"audit/v${AWF_VERSION}"`); }); it('audit_jsonl logformat should include all required fields', () => { @@ -597,7 +599,7 @@ describe('generateSquidConfig', () => { port: defaultPort, }; const result = generateSquidConfig(config); - // Required fields per audit.v1.schema.json + // Required fields per audit.schema.json const auditLine = result.split('\n').find(l => l.startsWith('logformat audit_jsonl')); expect(auditLine).toBeDefined(); expect(auditLine).toContain('"ts":'); diff --git a/src/squid-config.ts b/src/squid-config.ts index f9ed3adf2..916733578 100644 --- a/src/squid-config.ts +++ b/src/squid-config.ts @@ -8,6 +8,8 @@ import { } from './domain-patterns'; import { generateDlpSquidConfig } from './dlp'; import { DEFAULT_DNS_SERVERS } from './dns-resolver'; +// eslint-disable-next-line @typescript-eslint/no-require-imports +const { version: AWF_VERSION } = require('../package.json') as { version: string }; /** * Generates Squid cache_peer / always_direct / never_direct directives for @@ -600,7 +602,7 @@ logformat firewall_detailed %ts.%03tu %>a:%>p %{Host}>h %Hs %Ss # Structured JSONL audit log for machine-readable analysis # Note: Squid logformat does not JSON-escape strings, so fields like User-Agent # could break JSON parsing. We omit User-Agent to reduce breakage risk. -logformat audit_jsonl {"_schema":"audit/v1","ts":%ts.%03tu,"client":"%>a","host":"%{Host}>h","dest":"%Hs,"decision":"%Ss","url":"%ru"} +logformat audit_jsonl {"_schema":"audit/v${AWF_VERSION}","ts":%ts.%03tu,"client":"%>a","host":"%{Host}>h","dest":"%Hs,"decision":"%Ss","url":"%ru"} # Access log and cache configuration # Don't log healthcheck probes from localhost (using ACL filter on access_log)