From 3b99aaca6e243370d64fa830e2e85b165f2fc48a Mon Sep 17 00:00:00 2001 From: Alimzy Date: Tue, 28 Jul 2026 06:59:52 +0100 Subject: [PATCH] feat: add GrantFox issue batch quality validator Adds scripts/validate-issue-batch.mts, a standalone dependency-free script (matching check-package-boundaries.mts's convention) that validates a GrantFox issue batch JSON file against docs/ISSUE_STANDARD.md and .github/LABELS.yml before any issues are created via gh issue create. Checks: - Required fields (title, labels, body) present and non-empty - Every label exists in .github/LABELS.yml (the #1 failure mode AUTOMATION_RUNBOOK.md's troubleshooting table already names) - Campaign-labelled issues have a scope label, a type label, and at most one difficulty label per ISSUE_STANDARD.md - Campaign-labelled issues include all 8 required ISSUE_STANDARD.md sections as markdown headings - Acceptance-criteria checkboxes aren't vague/unverifiable phrasing Usage: pnpm validate:issues path/to/batch.json (exit 0/1). Adds examples/issue-batches/{valid,invalid}-batch.json. The invalid fixture is modeled on this issue's own real labels (monorepo, feature, developer-experience), none of which exist in .github/LABELS.yml -- confirmed by running the validator against it. Adds scripts/validate-issue-batch.test.ts (6 tests: compliant batch, unsupported labels, missing fields, missing sections, weak acceptance criteria, non-campaign issues exempt from section checks). Adds docs/ISSUE_BATCH_VALIDATOR.md. Closes #64 --- docs/ISSUE_BATCH_VALIDATOR.md | 57 ++++++ examples/issue-batches/invalid-batch.json | 12 ++ examples/issue-batches/valid-batch.json | 7 + package.json | 1 + scripts/validate-issue-batch.mts | 207 ++++++++++++++++++++++ scripts/validate-issue-batch.test.ts | 109 ++++++++++++ 6 files changed, 393 insertions(+) create mode 100644 docs/ISSUE_BATCH_VALIDATOR.md create mode 100644 examples/issue-batches/invalid-batch.json create mode 100644 examples/issue-batches/valid-batch.json create mode 100644 scripts/validate-issue-batch.mts create mode 100644 scripts/validate-issue-batch.test.ts diff --git a/docs/ISSUE_BATCH_VALIDATOR.md b/docs/ISSUE_BATCH_VALIDATOR.md new file mode 100644 index 0000000..fa6990e --- /dev/null +++ b/docs/ISSUE_BATCH_VALIDATOR.md @@ -0,0 +1,57 @@ +# GrantFox Issue Batch Validator + +`scripts/validate-issue-batch.mts` validates a GrantFox issue batch JSON file +against [ISSUE_STANDARD.md](./ISSUE_STANDARD.md) and `.github/LABELS.yml` +*before* any issues are created via `gh issue create` (see +[AUTOMATION_RUNBOOK.md](./AUTOMATION_RUNBOOK.md#batch-issue-creation)). + +## Usage + +```bash +pnpm validate:issues path/to/batch.json +``` + +The batch file must be a top-level JSON array of `{title, labels, body}` +objects, matching the format `AUTOMATION_RUNBOOK.md` uses for batch issue +creation. + +Exits `0` and prints a success line if every entry passes. Exits `1` and +prints every violation, grouped by entry, if any entry fails. + +## What it checks + +1. **Required fields** — every entry must have a non-empty `title`, a + non-empty `labels` array, and a non-empty `body`. +2. **Unsupported labels** — every label must exist in `.github/LABELS.yml`. + This is the check `AUTOMATION_RUNBOOK.md`'s troubleshooting table already + names as the most common batch failure ("Issues created without labels"). +3. **Campaign label taxonomy** — for any entry carrying all three campaign + labels (`GrantFox OSS`, `Maybe Rewarded`, `Official Campaign | FWC26`), + checks it also has at least one scope label, at least one type label, and + at most one difficulty label, per `ISSUE_STANDARD.md`'s labelling rules. +4. **Required sections** — campaign-labelled entries must include all eight + `ISSUE_STANDARD.md` sections (Summary, Background, Proposed scope, + Acceptance criteria, Tests required, Docs required, Security and Stellar + correctness notes, Estimate), matched as markdown headings. +5. **Weak acceptance criteria** — flags checkbox lines under "Acceptance + criteria" containing vague, unverifiable phrasing (e.g. "improve UX", + "handle edge cases") that `ISSUE_STANDARD.md` explicitly calls out as + non-compliant. + +Non-campaign entries (missing one or more of the three campaign labels) skip +checks 3–5, since `ISSUE_STANDARD.md` only applies to GrantFox-ready issues. + +## Examples + +See `examples/issue-batches/valid-batch.json` and +`examples/issue-batches/invalid-batch.json`. The invalid example is modeled +on real label mistakes seen in practice (`monorepo`, `feature`, +`developer-experience` — none of which exist in `.github/LABELS.yml`). + +## Tests + +`scripts/validate-issue-batch.test.ts` covers: a fully compliant batch, +unsupported labels, missing required fields, missing standard sections, weak +acceptance criteria, and that non-campaign issues are exempt from the +section/criteria checks. Run via `pnpm test` (turbo runs it at the workspace +root) or `npx vitest run scripts/validate-issue-batch.test.ts` directly. diff --git a/examples/issue-batches/invalid-batch.json b/examples/issue-batches/invalid-batch.json new file mode 100644 index 0000000..65d9ab6 --- /dev/null +++ b/examples/issue-batches/invalid-batch.json @@ -0,0 +1,12 @@ +[ + { + "title": "Add GrantFox issue quality validator", + "labels": ["GrantFox OSS", "Maybe Rewarded", "Official Campaign | FWC26", "monorepo", "feature", "developer-experience", "expert"], + "body": "Summary\nAdd a local validator for GrantFox-style issue JSON files.\n\nAcceptance Criteria\n- [ ] Issue batch schema validator is implemented.\n- [ ] Weak acceptance criteria are flagged." + }, + { + "title": "", + "labels": [], + "body": "" + } +] diff --git a/examples/issue-batches/valid-batch.json b/examples/issue-batches/valid-batch.json new file mode 100644 index 0000000..92c2fd3 --- /dev/null +++ b/examples/issue-batches/valid-batch.json @@ -0,0 +1,7 @@ +[ + { + "title": "[GrantFox] Add retry backoff to Horizon polling", + "labels": ["GrantFox OSS", "Maybe Rewarded", "Official Campaign | FWC26", "stellar", "bug", "good first issue"], + "body": "### Summary\nHorizon polling has no backoff on 429s.\n\n### Background\nSee Horizon rate limit docs.\n\n### Proposed scope\n**In scope**: exponential backoff in stellar-kit poller\n**Out of scope**: switching away from polling\n\n### Acceptance criteria\n- [ ] Poller retries with exponential backoff on HTTP 429\n\n### Tests required\n- packages/stellar-kit/test/poller.test.ts\n\n### Docs required\nNone required\n\n### Security and Stellar correctness notes\nNo secret key handling.\n\n### Estimate\nsmall" + } +] diff --git a/package.json b/package.json index 1985cb7..26f0ece 100644 --- a/package.json +++ b/package.json @@ -20,6 +20,7 @@ "format:check": "prettier --check \"**/*.{ts,tsx,md,json,rs}\"", "check:examples": "tsx scripts/check-examples.mts", "check:boundaries": "tsx scripts/check-package-boundaries.mts", + "validate:issues": "tsx scripts/validate-issue-batch.mts", "clean": "turbo run clean && rm -rf node_modules && rm -rf .turbo", "contract:test": "cd contracts/treasury-escrow && cargo test", "contract:build": "cd contracts/treasury-escrow && cargo build --target wasm32-unknown-unknown --release", diff --git a/scripts/validate-issue-batch.mts b/scripts/validate-issue-batch.mts new file mode 100644 index 0000000..fa38030 --- /dev/null +++ b/scripts/validate-issue-batch.mts @@ -0,0 +1,207 @@ +/** + * GrantFox issue batch quality validator (issue #64). + * + * Validates a batch JSON file (array of {title, labels, body}) intended for + * `gh issue create` against docs/ISSUE_STANDARD.md before any issues are + * created on GitHub. Catches unsupported labels, missing required fields, + * missing ISSUE_STANDARD.md sections on campaign-labelled issues, and weak + * acceptance criteria. + * + * Usage: `pnpm validate:issues ` + */ +import { readFileSync } from "node:fs"; +import { resolve } from "node:path"; + +export interface IssueBatchEntry { + title: string; + labels: string[]; + body: string; +} + +export interface Violation { + entryIndex: number; + entryTitle: string; + field: string; + message: string; +} + +export interface ValidationResult { + valid: boolean; + violations: Violation[]; +} + +const REQUIRED_STANDARD_SECTIONS = [ + "Summary", + "Background", + "Proposed scope", + "Acceptance criteria", + "Tests required", + "Docs required", + "Security and Stellar correctness notes", + "Estimate", +] as const; + +const CAMPAIGN_LABELS = ["GrantFox OSS", "Maybe Rewarded", "Official Campaign | FWC26"] as const; + +const SCOPE_LABELS = ["stellar", "soroban", "anchor", "sep", "wallet", "payments", "escrow"]; +const TYPE_LABELS = ["security", "test", "documentation", "bug", "enhancement"]; +const DIFFICULTY_LABELS = ["good first issue", "expert"]; + +const WEAK_ACCEPTANCE_PHRASES = [ + "improve ux", + "improve performance", + "make it better", + "should work well", + "handle edge cases", + "is user friendly", + "looks good", + "works correctly", +]; + +function parseLabelsYaml(yamlText: string): Set { + const names = new Set(); + const lines = yamlText.split("\n"); + for (const line of lines) { + const match = line.match(/^-\s+name:\s+(.+)$/); + if (!match) continue; + let raw = match[1].trim(); + if (raw.startsWith('"') && raw.endsWith('"')) { + raw = raw.slice(1, -1); + } + names.add(raw); + } + return names; +} + +function checkRequiredFields(entry: unknown, index: number, violations: Violation[]): entry is IssueBatchEntry { + if (typeof entry !== "object" || entry === null) { + violations.push({ entryIndex: index, entryTitle: "(unknown)", field: "entry", message: "Entry is not an object." }); + return false; + } + const e = entry as Record; + let ok = true; + const titleForMessage = typeof e.title === "string" ? e.title : "(missing title)"; + + if (typeof e.title !== "string" || e.title.trim() === "") { + violations.push({ entryIndex: index, entryTitle: titleForMessage, field: "title", message: "Missing or empty required field: title." }); + ok = false; + } + if (!Array.isArray(e.labels) || e.labels.length === 0) { + violations.push({ entryIndex: index, entryTitle: titleForMessage, field: "labels", message: "Missing or empty required field: labels (must be a non-empty array)." }); + ok = false; + } + if (typeof e.body !== "string" || e.body.trim() === "") { + violations.push({ entryIndex: index, entryTitle: titleForMessage, field: "body", message: "Missing or empty required field: body." }); + ok = false; + } + return ok; +} + +function checkLabels(entry: IssueBatchEntry, index: number, knownLabels: Set, violations: Violation[]) { + for (const label of entry.labels) { + if (!knownLabels.has(label)) { + violations.push({ entryIndex: index, entryTitle: entry.title, field: "labels", message: `Unsupported label "${label}" — not defined in .github/LABELS.yml.` }); + } + } + + const isCampaignIssue = CAMPAIGN_LABELS.every((l) => entry.labels.includes(l)); + if (isCampaignIssue) { + const hasScope = SCOPE_LABELS.some((l) => entry.labels.includes(l)); + const hasType = TYPE_LABELS.some((l) => entry.labels.includes(l)); + const difficultyCount = DIFFICULTY_LABELS.filter((l) => entry.labels.includes(l)).length; + + if (!hasScope) { + violations.push({ entryIndex: index, entryTitle: entry.title, field: "labels", message: `Campaign issue missing a scope label (one of: ${SCOPE_LABELS.join(", ")}).` }); + } + if (!hasType) { + violations.push({ entryIndex: index, entryTitle: entry.title, field: "labels", message: `Campaign issue missing a type label (one of: ${TYPE_LABELS.join(", ")}).` }); + } + if (difficultyCount > 1) { + violations.push({ entryIndex: index, entryTitle: entry.title, field: "labels", message: `Campaign issue has both "good first issue" and "expert" — pick at most one.` }); + } + } +} + +function checkBodyStandard(entry: IssueBatchEntry, index: number, violations: Violation[]) { + const isCampaignIssue = CAMPAIGN_LABELS.every((l) => entry.labels.includes(l)); + if (!isCampaignIssue) return; + + for (const section of REQUIRED_STANDARD_SECTIONS) { + const headingPattern = new RegExp(`(^|\\n)#{1,4}\\s*${section}\\b`, "i"); + if (!headingPattern.test(entry.body)) { + violations.push({ entryIndex: index, entryTitle: entry.title, field: "body", message: `Campaign issue is missing the required "${section}" section per docs/ISSUE_STANDARD.md.` }); + } + } + + const acceptanceMatch = entry.body.match(/#{1,4}\s*Acceptance criteria\b([\s\S]*?)(\n#{1,4}\s|$)/i); + if (acceptanceMatch) { + const section = acceptanceMatch[1]; + const checkboxLines = section.split("\n").filter((l) => /^\s*-\s*\[[ x]\]/i.test(l)); + if (checkboxLines.length === 0) { + violations.push({ entryIndex: index, entryTitle: entry.title, field: "body", message: `Acceptance criteria section has no checkbox items ("- [ ] ...").` }); + } + for (const line of checkboxLines) { + const lower = line.toLowerCase(); + const weakPhrase = WEAK_ACCEPTANCE_PHRASES.find((p) => lower.includes(p)); + if (weakPhrase) { + violations.push({ entryIndex: index, entryTitle: entry.title, field: "body", message: `Weak, unverifiable acceptance criterion (contains "${weakPhrase}"): "${line.trim()}"` }); + } + } + } +} + +export function validateIssueBatch(batch: unknown[], labelsYamlText: string): ValidationResult { + const knownLabels = parseLabelsYaml(labelsYamlText); + const violations: Violation[] = []; + + batch.forEach((entry, index) => { + if (!checkRequiredFields(entry, index, violations)) return; + checkLabels(entry, index, knownLabels, violations); + checkBodyStandard(entry, index, violations); + }); + + return { valid: violations.length === 0, violations }; +} + +function main() { + const batchPath = process.argv[2]; + if (!batchPath) { + console.error("Usage: pnpm validate:issues "); + process.exit(1); + } + + const root = process.cwd(); + const labelsYamlText = readFileSync(resolve(root, ".github/LABELS.yml"), "utf-8"); + const batchRaw = readFileSync(resolve(root, batchPath), "utf-8"); + + let batch: unknown; + try { + batch = JSON.parse(batchRaw); + } catch (err) { + console.error(`❌ Failed to parse ${batchPath} as JSON: ${(err as Error).message}`); + process.exit(1); + } + + if (!Array.isArray(batch)) { + console.error(`❌ ${batchPath} must contain a top-level JSON array of issue entries.`); + process.exit(1); + } + + const result = validateIssueBatch(batch, labelsYamlText); + + if (result.valid) { + console.log(`\n✓ Issue batch "${batchPath}" passed validation (${batch.length} entries).\n`); + process.exit(0); + } else { + console.error(`\n❌ Found ${result.violations.length} issue(s) across ${batch.length} entries in "${batchPath}":\n`); + for (const v of result.violations) { + console.error(` [entry ${v.entryIndex}] "${v.entryTitle}" — ${v.field}`); + console.error(` ${v.message}\n`); + } + process.exit(1); + } +} + +if (import.meta.url === `file://${process.argv[1]}`) { + main(); +} diff --git a/scripts/validate-issue-batch.test.ts b/scripts/validate-issue-batch.test.ts new file mode 100644 index 0000000..7d72ca4 --- /dev/null +++ b/scripts/validate-issue-batch.test.ts @@ -0,0 +1,109 @@ +import { describe, test, expect } from "vitest"; +import { validateIssueBatch } from "./validate-issue-batch.mts"; + +const LABELS_YAML = ` +- name: GrantFox OSS + color: 9443fb +- name: Maybe Rewarded + color: f59e0b +- name: "Official Campaign | FWC26" + color: ec4899 +- name: stellar + color: 2563eb +- name: test + color: 0d9488 +- name: expert + color: 475569 +`; + +const VALID_BODY = ` +### Summary +Fixes a real bug. + +### Background +See docs. + +### Proposed scope +**In scope**: thing +**Out of scope**: other thing + +### Acceptance criteria +- [ ] Endpoint returns 404 when project does not exist + +### Tests required +- foo.test.ts + +### Docs required +None required + +### Security and Stellar correctness notes +No secret key handling. + +### Estimate +small +`; + +describe("validateIssueBatch", () => { + test("accepts a fully compliant campaign issue", () => { + const batch = [ + { + title: "Fix thing", + labels: ["GrantFox OSS", "Maybe Rewarded", "Official Campaign | FWC26", "stellar", "test"], + body: VALID_BODY, + }, + ]; + const result = validateIssueBatch(batch, LABELS_YAML); + expect(result.valid).toBe(true); + expect(result.violations).toHaveLength(0); + }); +}); + +test("flags unsupported labels", () => { + const batch = [{ title: "X", labels: ["GrantFox OSS", "monorepo", "feature"], body: "body text" }]; + const result = validateIssueBatch(batch, LABELS_YAML); + expect(result.valid).toBe(false); + const labelViolations = result.violations.filter((v) => v.message.includes("Unsupported label")); + expect(labelViolations).toHaveLength(2); +}); + +test("flags missing required fields", () => { + const batch = [{ title: "", labels: [], body: "" }]; + const result = validateIssueBatch(batch, LABELS_YAML); + expect(result.valid).toBe(false); + expect(result.violations.some((v) => v.field === "title")).toBe(true); + expect(result.violations.some((v) => v.field === "labels")).toBe(true); + expect(result.violations.some((v) => v.field === "body")).toBe(true); +}); + +test("flags missing ISSUE_STANDARD.md sections on campaign issues", () => { + const batch = [ + { + title: "X", + labels: ["GrantFox OSS", "Maybe Rewarded", "Official Campaign | FWC26", "stellar", "test"], + body: "### Summary\nJust a summary, nothing else.", + }, + ]; + const result = validateIssueBatch(batch, LABELS_YAML); + expect(result.valid).toBe(false); + const missingSection = result.violations.filter((v) => v.message.includes("missing the required")); + expect(missingSection.length).toBeGreaterThanOrEqual(6); +}); + +test("flags weak, unverifiable acceptance criteria", () => { + const batch = [ + { + title: "X", + labels: ["GrantFox OSS", "Maybe Rewarded", "Official Campaign | FWC26", "stellar", "test"], + body: VALID_BODY.replace("Endpoint returns 404 when project does not exist", "improve UX"), + }, + ]; + const result = validateIssueBatch(batch, LABELS_YAML); + expect(result.valid).toBe(false); + expect(result.violations.some((v) => v.message.includes("Weak, unverifiable"))).toBe(true); +}); + +test("does not require ISSUE_STANDARD.md sections on non-campaign issues", () => { + const batch = [{ title: "X", labels: ["stellar"], body: "just a short body" }]; + const result = validateIssueBatch(batch, LABELS_YAML); + expect(result.valid).toBe(true); +});