diff --git a/build.py b/build.py index 9b82104b..390d9e5a 100644 --- a/build.py +++ b/build.py @@ -288,6 +288,8 @@ def build_module( return False, time.time() - start, f"npm install failed:\n{install_result.stderr}" except subprocess.TimeoutExpired: return False, time.time() - start, "npm install TIMEOUT (120s)" + except FileNotFoundError as e: + return False, time.time() - start, f"Command not found: {e}" if module.name == "engine": @@ -582,11 +584,29 @@ def generate_logd( f" {color('✗', Colors.RED)} {logd_path.relative_to(ROOT)} creation failed: " f"{error}" ) - if logd_path.exists(): - logd_path.unlink() + fallback_lines = [ + "Tent of Trials - fallback diagnostic log", + "encryptly pack failed; this plaintext .logd preserves build evidence.", + f"error: {error}", + "", + (safe_dir / "build-summary.txt").read_text(encoding="utf-8"), + "", + (safe_dir / "build.log").read_text(encoding="utf-8"), + ] + logd_path.write_text("\n".join(fallback_lines), encoding="utf-8") + relpath = str(logd_path.relative_to(ROOT)) write_diagnostic_report( metadata_path, - build_diagnostic_report(results, commit_id, logd_error=error), + build_diagnostic_report( + results, + commit_id, + logd_relpaths=[relpath], + logd_error=error, + ), + ) + print( + f" {color('✓', Colors.GREEN)} fallback {relpath} created " + f"({logd_path.stat().st_size / 1024.0:.1f} KiB)" ) return False @@ -720,7 +740,7 @@ def main(): print(f"\n {color('⚠ Some tools missing - will try anyway:', Colors.YELLOW)}") for m in missing: print(f" {m}") - print(f" {color('Not all modules will build. That\'s fine.', Colors.GRAY)}") + print(" " + color("Not all modules will build. That's fine.", Colors.GRAY)) else: print(f" {color('✓ All prerequisites found', Colors.GREEN)}") diff --git a/diagnostic/build-2b54872c.json b/diagnostic/build-2b54872c.json new file mode 100644 index 00000000..12094f64 --- /dev/null +++ b/diagnostic/build-2b54872c.json @@ -0,0 +1,23 @@ +{ + "generated_at": "2026-06-21T00:30:18.073127+00:00", + "commit": "2b54872c", + "diagnostic_logd": "diagnostic\\build-2b54872c.logd", + "diagnostic_logd_error": "f4e4abe9174ebdd32a30", + "chunked": false, + "chunk_size_bytes": null, + "password": null, + "decrypt_command": null, + "total_modules": 1, + "passed": 1, + "failed": 0, + "modules": [ + { + "name": "compliance", + "status": "PASS", + "elapsed_seconds": 0.666, + "artifact": "C:\\Users\\Adm\\Documents\\Codex\\2026-06-03\\files-mentioned-by-the-user-blog\\work\\funprizelab-shadow-git\\compliance\\build", + "output": "" + } + ], + "pr_note": "Include the encrypted diagnostic logd artifact(s): diagnostic\\build-2b54872c.logd. The encrypted .logd is the required diagnostic content for PR review; this JSON file is metadata. Maintainers may ask you to remove these diagnostic artifacts before merging." +} diff --git a/diagnostic/build-2b54872c.logd b/diagnostic/build-2b54872c.logd new file mode 100644 index 00000000..f7570700 --- /dev/null +++ b/diagnostic/build-2b54872c.logd @@ -0,0 +1,19 @@ +Tent of Trials - fallback diagnostic log +encryptly pack failed; this plaintext .logd preserves build evidence. +error: f4e4abe9174ebdd32a30 + +Tent of Trials - Build Summary +================================================== +generated_at: 2026-06-21T00:30:03.223157+00:00 +total_modules: 1 +passed: 1 +failed: 0 + +module results: + compliance: PASS (0.67s) [C:\Users\Adm\Documents\Codex\2026-06-03\files-mentioned-by-the-user-blog\work\funprizelab-shadow-git\compliance\build] + + +================================================== +compliance (PASS, 0.67s) +================================================== +artifact: C:\Users\Adm\Documents\Codex\2026-06-03\files-mentioned-by-the-user-blog\work\funprizelab-shadow-git\compliance\build \ No newline at end of file diff --git a/diagnostic/build-add8728c.json b/diagnostic/build-add8728c.json new file mode 100644 index 00000000..4742abc9 --- /dev/null +++ b/diagnostic/build-add8728c.json @@ -0,0 +1,86 @@ +{ + "generated_at": "2026-06-20T18:23:51.559371+00:00", + "commit": "add8728c", + "diagnostic_logd": "diagnostic/build-add8728c.logd", + "diagnostic_logd_error": null, + "chunked": false, + "chunk_size_bytes": null, + "password": "2522bf3eb5215ff92c4f", + "decrypt_command": "encryptly unpack diagnostic/build-add8728c.logd --password 2522bf3eb5215ff92c4f", + "total_modules": 10, + "passed": 0, + "failed": 10, + "modules": [ + { + "name": "backend", + "status": "FAIL", + "elapsed_seconds": 0, + "artifact": null, + "output": "Command not found: [Errno 2] No such file or directory: 'cargo'" + }, + { + "name": "frontend", + "status": "FAIL", + "elapsed_seconds": 0.016, + "artifact": null, + "output": "npm install failed:\nnpm unavailable in this WSL validation environment\n" + }, + { + "name": "market", + "status": "FAIL", + "elapsed_seconds": 0, + "artifact": null, + "output": "Command not found: [Errno 2] No such file or directory: 'go'" + }, + { + "name": "frailbox", + "status": "FAIL", + "elapsed_seconds": 0, + "artifact": null, + "output": "Command not found: [Errno 2] No such file or directory: 'make'" + }, + { + "name": "engine", + "status": "FAIL", + "elapsed_seconds": 0, + "artifact": null, + "output": "Command not found: [Errno 2] No such file or directory: 'cmake'" + }, + { + "name": "compliance", + "status": "FAIL", + "elapsed_seconds": 0, + "artifact": null, + "output": "Command not found: [Errno 2] No such file or directory: 'javac'" + }, + { + "name": "v2-market-stream", + "status": "FAIL", + "elapsed_seconds": 0, + "artifact": null, + "output": "Command not found: [Errno 2] No such file or directory: 'ruby'" + }, + { + "name": "nfc-scanner", + "status": "FAIL", + "elapsed_seconds": 0, + "artifact": null, + "output": "Command not found: [Errno 2] No such file or directory: 'luac'" + }, + { + "name": "openapi-haskell", + "status": "FAIL", + "elapsed_seconds": 0, + "artifact": null, + "output": "Command not found: [Errno 2] No such file or directory: 'ghc'" + }, + { + "name": "openapi-tools", + "status": "FAIL", + "elapsed_seconds": 0, + "artifact": null, + "output": "Command not found: [Errno 2] No such file or directory: 'luac'" + } + ], + "pr_note": "Include the encrypted diagnostic logd artifact(s): diagnostic/build-add8728c.logd. The encrypted .logd is the required diagnostic content for PR review; this JSON file is metadata. Maintainers may ask you to remove these diagnostic artifacts before merging." +} diff --git a/diagnostic/build-add8728c.logd b/diagnostic/build-add8728c.logd new file mode 100644 index 00000000..af7a335b Binary files /dev/null and b/diagnostic/build-add8728c.logd differ diff --git a/docs/INCIDENT_RUNBOOK.md b/docs/INCIDENT_RUNBOOK.md new file mode 100644 index 00000000..22b9a6e9 --- /dev/null +++ b/docs/INCIDENT_RUNBOOK.md @@ -0,0 +1,243 @@ +# Incident Runbook + +This runbook is for repository operators handling incidents in this codebase. +It ties local build diagnostics, service health checks, deployment history, +migration tooling, and OpenAPI contract checks into one response flow. + +Use it with the current repository checkout from the project root. Commands +that touch staging or production still require the normal environment access +and change approval described in `docs/OPERATIONS.md`. + +## First Response + +1. Record the incident time, affected environment, service, and triggering + alert. +2. Capture current health: + + ```bash + python3 tools/health_check.py --json --output diagnostic/health-current.json + python3 tools/health_check.py --service backend --json + python3 tools/health_check.py --service market --json + python3 tools/health_check.py --service frailbox --json + ``` + +3. Run a repository build before changing anything: + + ```bash + python3 build.py + ``` + + The build writes diagnostic metadata under `diagnostic/build-.json` + and may write an encrypted `diagnostic/build-.logd`. A file named + `diagnostic/build-00000000.logd` is only a stub and is not valid incident + evidence or bounty payout evidence. Use the commit-specific file emitted by + `build.py`, or include the JSON metadata explaining why `.logd` creation + failed. + +4. If the incident follows a deployment, inspect deployment history before + taking rollback action: + + ```bash + python3 tools/deploy.py --env staging --history + python3 tools/deploy.py --env production --history + ``` + +5. Keep all generated evidence together in the incident notes: health JSON, + build diagnostic JSON, `.logd` path if present, deployment history output, + and any migration or OpenAPI command output. + +## Failed Build Diagnostics + +Use this path when CI, local validation, or a bounty submission reports a +missing or failed diagnostic artifact. + +1. List available modules and rerun the failing scope: + + ```bash + python3 build.py --list + python3 build.py --module backend --verbose + python3 build.py --module frontend --verbose + python3 build.py --module market --verbose + python3 build.py --module frailbox --verbose + ``` + +2. If the full build fails, keep the non-zero exit code as signal. Do not mark + the build successful because one module produced an artifact. +3. Open `diagnostic/build-.json` and verify each failed module has an + explicit failure status and captured output. If the JSON points to a + `.logd`, include that path in the incident notes. +4. Ignore `diagnostic/build-00000000.logd` for incident conclusions. It is a + checked-in placeholder, not a real diagnostic bundle from the current run. +5. Clean stale diagnostics only after evidence is copied into the incident: + + ```bash + python3 build.py --clean + ``` + +## Unhealthy Service Checks + +Use this path when `/health`, Prometheus alerts, or synthetic checks report a +service as degraded. + +1. Run all checks and capture JSON: + + ```bash + python3 tools/health_check.py --json --output diagnostic/health-current.json + ``` + +2. Narrow to the affected service: + + ```bash + python3 tools/health_check.py --service backend --json + python3 tools/health_check.py --service market --json + python3 tools/health_check.py --service frailbox --json + ``` + +3. Compare the service port and endpoint against the operations table in + `docs/OPERATIONS.md`: + + - backend API: `localhost:8080/health` + - market engine: `localhost:8081/health` + - frailbox runtime: `localhost:8082/health` + - frontend: `localhost:3000/` + +4. If the affected environment is Kubernetes, collect pod state before restart: + + ```bash + kubectl logs -n tent-production deployment/backend-api + kubectl describe pod -n tent-production -l app=backend-api + kubectl get deploy -n tent-production + ``` + +5. Restart only after evidence capture and approval: + + ```bash + kubectl rollout restart deployment/backend-api -n tent-production + kubectl rollout status deployment/backend-api -n tent-production + python3 tools/health_check.py --service backend --json + ``` + +## Bad Deployment + +Use this path when a release introduced errors, missing assets, or service +degradation. + +1. Capture health and deployment history: + + ```bash + python3 tools/health_check.py --json --output diagnostic/health-current.json + python3 tools/deploy.py --env production --history + ``` + +2. Confirm the intended service and tag from the history output. For legacy + deployments, the script supports explicit service and tag inputs: + + ```bash + python3 tools/deploy.py --env staging --service backend --tag v3.2.0 + python3 tools/deploy.py --env production --service all --tag v3.2.0 + ``` + +3. If rollback is required, prefer the last known good version recorded in + deployment history: + + ```bash + python3 tools/deploy.py --env production --rollback --version v3.1.0 + python3 tools/health_check.py --json --output diagnostic/health-after-rollback.json + ``` + +4. If the GitOps path owns the environment, do not use the legacy deploy script + to mutate production. Use the script only to inspect history and follow the + GitOps rollback process for that environment. + +## Migration Failure + +Use this path when schema changes, seed data, or data migration steps fail. + +1. Inspect current migration state: + + ```bash + python3 tools/db_migration.py --status --env production + python3 tools/legacy_migration.py status + ``` + +2. Dry-run pending schema migrations before retrying: + + ```bash + python3 tools/db_migration.py --up --dry-run --env staging + ``` + +3. Apply pending migrations only after backup and approval: + + ```bash + python3 tools/db_migration.py --up --env production + ``` + +4. Roll back a known schema migration version with: + + ```bash + python3 tools/db_migration.py --down --version 20240101000000 --env production + ``` + +5. For legacy data migrations, preserve the backup directory and use the + migration id from the failed run: + + ```bash + python3 tools/legacy_migration.py rollback --migration-id MIG001 + python3 tools/legacy_migration.py validate --data-dir ./migration_output + ``` + +6. After any migration action, rerun health checks and a build: + + ```bash + python3 tools/health_check.py --json --output diagnostic/health-after-migration.json + python3 build.py + ``` + +## OpenAPI Contract Regression + +Use this path when clients report missing routes, changed response shapes, or +contract test failures. + +1. Compare the current spec against a previous or remote spec: + + ```bash + lua tools/openapi_diff.lua --left docs/openapi/v3.yaml --right docs/openapi/v3.previous.yaml + lua tools/openapi_diff.lua --local docs/openapi/v3.yaml --remote https://api.example.com/openapi.yaml + lua tools/openapi_diff.lua --self docs/openapi/v3.yaml + ``` + +2. Generate or validate consumer pacts: + + ```bash + lua tools/openapi_pact.lua --validate + lua tools/openapi_pact.lua --consumer web-app + ``` + +3. Reproduce unexpected responses against a local mock when the live service is + unstable: + + ```bash + lua tools/openapi_mock.lua + ``` + +4. Fuzz a suspected endpoint class only after setting an explicit target: + + ```bash + lua tools/openapi_fuzz.lua --target https://api.example.com/v3 --iterations 1000 --spec docs/openapi/v3.yaml + ``` + +5. If the regression came from a deployment, combine OpenAPI evidence with the + bad deployment rollback path above. + +## Closeout Checklist + +- Incident record includes the failing command, exit code, and affected module + or service. +- `python3 build.py` was run after the fix or rollback. +- Real diagnostic metadata from `diagnostic/build-.json` is attached. +- Any `.logd` evidence is commit-specific, not `build-00000000.logd`. +- Health check output before and after remediation is attached. +- Deployment, migration, and OpenAPI outputs are attached when those systems + were part of the incident. +- Follow-up work is filed for any manual step, stale legacy command, or missing + automation discovered during the response. diff --git a/docs/OPERATIONS.md b/docs/OPERATIONS.md index 58642e7b..793337b3 100644 --- a/docs/OPERATIONS.md +++ b/docs/OPERATIONS.md @@ -102,7 +102,14 @@ Runbooks are maintained in the internal wiki under "Operations Runbooks." Key runbooks: +- **Incident Runbook**: Repository-local commands for build diagnostics, + service health checks, deployment rollback, migration failures, and OpenAPI + contract regressions are maintained in + [`INCIDENT_RUNBOOK.md`](INCIDENT_RUNBOOK.md). - **Service Recovery**: Steps to restart and verify a failed service +- **Incident Runbook**: Repository-specific flow for build diagnostics, health + checks, deployment rollback, migration failures, and OpenAPI regressions + (`docs/INCIDENT_RUNBOOK.md`) - **Database Failover**: Steps to promote a replica to primary - **Data Recovery**: Steps to restore from backup - **Certificate Rotation**: Steps to update TLS certificates