diff --git a/docs/evidence-redaction-fixtures.md b/docs/evidence-redaction-fixtures.md new file mode 100644 index 0000000..67f5ae4 --- /dev/null +++ b/docs/evidence-redaction-fixtures.md @@ -0,0 +1,59 @@ +# Evidence Redaction Fixtures Report + +This report covers the fixture pack at +`verify/fixtures/evidence-redaction/`. It is intended for Frantic workers, +reviewers, and future verifier tests that need a common evidence-redaction +corpus. + +## Redaction Rule + +Evidence should keep the reviewable fact while removing the private value: + +- `fr_test_token_*` becomes `[REDACTED:TOKEN]`. +- `C:\Users\ExampleOperator\...` becomes `[REDACTED:LOCAL_PATH]`. +- `https://internal.example.invalid/...` becomes `[REDACTED:PRIVATE_URL]`. +- `worker@example.invalid` becomes `[REDACTED:EMAIL]`. + +Reviewer action for every unsafe case is `revise`, with the fixture file and +category named in the review note. + +## Validation Results + +The fixture pack includes six fixtures: + +- `safe-health-check.json`: valid JSON with `summary` and `observations`; + reviewer action `accept`. +- `safe-delivery-linter.json`: valid JSON with `summary` and `observations`; + reviewer action `accept`. +- `safe-board-health.json`: valid JSON with `summary` and `observations`; + reviewer action `accept`. +- `unsafe-token.json`: synthetic token sentinel; reviewer action `revise` and + redact token. +- `unsafe-local-path.txt`: synthetic local path sentinel; reviewer action + `revise` and redact local path. +- `unsafe-private-url-and-email.json`: synthetic private URL and email + sentinels; reviewer action `revise` and redact both values. + +No fixture contains a live credential, real private customer record, or usable +secret. + +## Proposed Verifier Usage + +Future verifier tests can: + +1. Load `manifest.json`. +2. Assert every `safe: true` fixture parses as JSON and has `summary` plus + non-empty `observations`. +3. Assert every `safe: false` fixture includes one or more synthetic sentinel + patterns from `categories`. +4. Require reviewers to return unsafe evidence for revision instead of accepting + or silently editing the worker's artifact. + +## Evidence Observations + +- Fixture names: `safe-health-check.json`, `safe-delivery-linter.json`, + `safe-board-health.json`, `unsafe-token.json`, `unsafe-local-path.txt`, + `unsafe-private-url-and-email.json`. +- Redaction categories: token, local path, private URL, private email. +- Validation command: `node -e ""`. +- runx version observed locally: `runx-cli 0.6.8`. diff --git a/verify/fixtures/evidence-redaction/README.md b/verify/fixtures/evidence-redaction/README.md new file mode 100644 index 0000000..3328dd4 --- /dev/null +++ b/verify/fixtures/evidence-redaction/README.md @@ -0,0 +1,45 @@ +# Evidence Redaction Fixture Pack + +This fixture pack gives Frantic workers, reviewers, and verifier tests a shared +corpus for checking whether evidence is useful without leaking sensitive data. + +The pack intentionally includes two categories: + +- `safe-*.json`: valid evidence JSON that can be accepted as-is. +- `unsafe-*.txt` and `unsafe-*.json`: synthetic examples that must be redacted + before a worker submits evidence. + +All unsafe values are synthetic sentinels. There are no live secrets, customer +records, or real credentials in this directory. + +## Redaction Rule + +Evidence may describe a secret, private path, internal URL, or private address, +but it must not expose the raw value. Replace the value with a stable class tag +that preserves review value: + +- API keys or tokens: `[REDACTED:TOKEN]` +- Local filesystem paths: `[REDACTED:LOCAL_PATH]` +- Private network URLs: `[REDACTED:PRIVATE_URL]` +- Private email addresses: `[REDACTED:EMAIL]` + +Reviewer action for unsafe evidence is `revise`: return the delivery with the +specific category and fixture name so the worker can redact without losing the +useful surrounding observation. + +## Fixture Inventory + +| Fixture | Category | Valid JSON | Expected reviewer action | +| --- | --- | --- | --- | +| `safe-health-check.json` | safe diagnostic evidence | yes | accept | +| `safe-delivery-linter.json` | safe artifact evidence | yes | accept | +| `safe-board-health.json` | safe audit evidence | yes | accept | +| `unsafe-token.json` | synthetic token leak | yes | revise, redact token | +| `unsafe-local-path.txt` | local path leak | no | revise, redact path | +| `unsafe-private-url-and-email.json` | private URL and email leak | yes | revise, redact URL and email | + +## Proposed Verifier Usage + +A future verifier can load `manifest.json`, assert that every safe fixture parses +as JSON with `summary` and `observations`, and assert that unsafe fixtures match +at least one sentinel class from `categories`. diff --git a/verify/fixtures/evidence-redaction/manifest.json b/verify/fixtures/evidence-redaction/manifest.json new file mode 100644 index 0000000..33e232d --- /dev/null +++ b/verify/fixtures/evidence-redaction/manifest.json @@ -0,0 +1,87 @@ +{ + "summary": "Frantic evidence redaction fixture manifest.", + "observations": [ + { + "name": "runx-version", + "value": "runx --version -> runx-cli 0.6.8" + }, + { + "name": "fixture-count", + "value": "6 fixtures: 3 safe JSON fixtures and 3 unsafe sentinel fixtures" + }, + { + "name": "proposed-verifier-usage", + "value": "Safe fixtures must parse as JSON with summary and observations; unsafe fixtures must trigger one or more sentinel categories." + } + ], + "categories": [ + { + "name": "token", + "sentinel": "fr_test_token_", + "redaction": "[REDACTED:TOKEN]", + "reviewer_action": "revise" + }, + { + "name": "local_path", + "sentinel": "C:\\Users\\ExampleOperator\\", + "redaction": "[REDACTED:LOCAL_PATH]", + "reviewer_action": "revise" + }, + { + "name": "private_url", + "sentinel": "https://internal.example.invalid/", + "redaction": "[REDACTED:PRIVATE_URL]", + "reviewer_action": "revise" + }, + { + "name": "private_email", + "sentinel": "worker@example.invalid", + "redaction": "[REDACTED:EMAIL]", + "reviewer_action": "revise" + } + ], + "fixtures": [ + { + "file": "safe-health-check.json", + "safe": true, + "valid_json": true, + "expected_action": "accept", + "redaction_categories": [] + }, + { + "file": "safe-delivery-linter.json", + "safe": true, + "valid_json": true, + "expected_action": "accept", + "redaction_categories": [] + }, + { + "file": "safe-board-health.json", + "safe": true, + "valid_json": true, + "expected_action": "accept", + "redaction_categories": [] + }, + { + "file": "unsafe-token.json", + "safe": false, + "valid_json": true, + "expected_action": "revise", + "redaction_categories": ["token"] + }, + { + "file": "unsafe-local-path.txt", + "safe": false, + "valid_json": false, + "expected_action": "revise", + "redaction_categories": ["local_path"] + }, + { + "file": "unsafe-private-url-and-email.json", + "safe": false, + "valid_json": true, + "expected_action": "revise", + "redaction_categories": ["private_url", "private_email"] + } + ] +} diff --git a/verify/fixtures/evidence-redaction/safe-board-health.json b/verify/fixtures/evidence-redaction/safe-board-health.json new file mode 100644 index 0000000..e1d1955 --- /dev/null +++ b/verify/fixtures/evidence-redaction/safe-board-health.json @@ -0,0 +1,17 @@ +{ + "summary": "Board health audit captured public counts without secrets.", + "observations": [ + { + "name": "source", + "value": "https://gofrantic.com/feed.json" + }, + { + "name": "coverage", + "value": "public board, five bounty pages, and public feed metadata" + }, + { + "name": "runx-version", + "value": "runx --version -> runx-cli 0.6.8" + } + ] +} diff --git a/verify/fixtures/evidence-redaction/safe-delivery-linter.json b/verify/fixtures/evidence-redaction/safe-delivery-linter.json new file mode 100644 index 0000000..3937a88 --- /dev/null +++ b/verify/fixtures/evidence-redaction/safe-delivery-linter.json @@ -0,0 +1,17 @@ +{ + "summary": "Delivery artifact linter fixture passed with all required refs.", + "observations": [ + { + "name": "bounty-url", + "value": "https://gofrantic.com/bounties/41" + }, + { + "name": "required-artifacts", + "value": "public_url, evidence_json, receipt_ref, report, pr_url" + }, + { + "name": "runx-version", + "value": "runx --version -> runx-cli 0.6.8" + } + ] +} diff --git a/verify/fixtures/evidence-redaction/safe-health-check.json b/verify/fixtures/evidence-redaction/safe-health-check.json new file mode 100644 index 0000000..4bbf351 --- /dev/null +++ b/verify/fixtures/evidence-redaction/safe-health-check.json @@ -0,0 +1,17 @@ +{ + "summary": "Health check validation rejected an empty 200 response.", + "observations": [ + { + "name": "endpoint", + "value": "http://127.0.0.1:18080/healthz" + }, + { + "name": "status", + "value": "HTTP 200 with empty body was classified unhealthy" + }, + { + "name": "runx-version", + "value": "runx --version -> runx-cli 0.6.8" + } + ] +} diff --git a/verify/fixtures/evidence-redaction/unsafe-local-path.txt b/verify/fixtures/evidence-redaction/unsafe-local-path.txt new file mode 100644 index 0000000..ef56b6b --- /dev/null +++ b/verify/fixtures/evidence-redaction/unsafe-local-path.txt @@ -0,0 +1,8 @@ +Unsafe synthetic local path example. + +The worker copied a local diagnostic path into public evidence: + + C:\Users\ExampleOperator\AppData\Local\Temp\frantic\claim-0000\secret.log + +Expected reviewer action: revise. +Expected redaction: [REDACTED:LOCAL_PATH]\secret.log diff --git a/verify/fixtures/evidence-redaction/unsafe-private-url-and-email.json b/verify/fixtures/evidence-redaction/unsafe-private-url-and-email.json new file mode 100644 index 0000000..79fd280 --- /dev/null +++ b/verify/fixtures/evidence-redaction/unsafe-private-url-and-email.json @@ -0,0 +1,21 @@ +{ + "summary": "Unsafe synthetic private URL and email example that must be redacted.", + "observations": [ + { + "name": "bad-private-url", + "value": "debug dashboard was captured at https://internal.example.invalid/frantic/review/claim-0000" + }, + { + "name": "bad-email", + "value": "operator contact appeared as worker@example.invalid" + }, + { + "name": "expected-redaction", + "value": "debug dashboard [REDACTED:PRIVATE_URL], operator contact [REDACTED:EMAIL]" + }, + { + "name": "runx-version", + "value": "runx --version -> runx-cli 0.6.8" + } + ] +} diff --git a/verify/fixtures/evidence-redaction/unsafe-token.json b/verify/fixtures/evidence-redaction/unsafe-token.json new file mode 100644 index 0000000..5701544 --- /dev/null +++ b/verify/fixtures/evidence-redaction/unsafe-token.json @@ -0,0 +1,17 @@ +{ + "summary": "Unsafe synthetic token example that must be redacted.", + "observations": [ + { + "name": "bad-token-capture", + "value": "authorization header was Bearer fr_test_token_DO_NOT_SUBMIT_000000" + }, + { + "name": "expected-redaction", + "value": "authorization header was Bearer [REDACTED:TOKEN]" + }, + { + "name": "runx-version", + "value": "runx --version -> runx-cli 0.6.8" + } + ] +}