diff --git a/.github/workflows/auto-triage-issues.lock.yml b/.github/workflows/auto-triage-issues.lock.yml index e2896f38381..a8bc5891a08 100644 --- a/.github/workflows/auto-triage-issues.lock.yml +++ b/.github/workflows/auto-triage-issues.lock.yml @@ -1702,6 +1702,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"labels-applied","question":"Did the agent apply at least one label to an unlabeled issue, or correctly call noop when no unlabeled issues were found?"},{"id":"report-created","question":"Was a summary discussion created listing the issues processed and the labels applied?"}]' GH_AW_EVALS_MODEL: "copilot/gpt-5.4" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/breaking-change-checker.lock.yml b/.github/workflows/breaking-change-checker.lock.yml index 1588bdc1f3b..932924895d6 100644 --- a/.github/workflows/breaking-change-checker.lock.yml +++ b/.github/workflows/breaking-change-checker.lock.yml @@ -1789,6 +1789,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"analysis-completed","question":"Did the agent complete an analysis of recent commits and merged PRs for breaking CLI changes?"},{"id":"issue-created-or-noop","question":"Was a breaking change issue created when breaking changes were found, or was noop correctly called when no breaking changes were detected?"}]' GH_AW_EVALS_MODEL: "small" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/contribution-check.lock.yml b/.github/workflows/contribution-check.lock.yml index 3cea5078f72..86556705543 100644 --- a/.github/workflows/contribution-check.lock.yml +++ b/.github/workflows/contribution-check.lock.yml @@ -1833,6 +1833,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"prs-evaluated","question":"Did the agent dispatch PRs to the contribution-checker subagent and produce evaluation results for each PR?"},{"id":"report-created","question":"Was a report issue created summarizing PR compliance with the contributing guidelines?"}]' GH_AW_EVALS_MODEL: "small" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/.github/workflows/smoke-copilot.lock.yml b/.github/workflows/smoke-copilot.lock.yml index 920eabd58e9..325ccb8defa 100644 --- a/.github/workflows/smoke-copilot.lock.yml +++ b/.github/workflows/smoke-copilot.lock.yml @@ -2985,6 +2985,7 @@ jobs: GH_AW_EVALS_QUESTIONS: '[{"id":"smoke-passed","question":"Did all or most smoke tests pass? Look for an overall PASS status or the majority of tests showing ✅ in the agent output."},{"id":"build-succeeded","question":"Did the gh-aw binary build succeed? Look for a successful make build step or explicit mention that compilation succeeded."},{"id":"issue-created","question":"Was a smoke test issue created with test results? Look for a create_issue output containing ''Smoke Test'' in the title."}]' GH_AW_EVALS_MODEL: "gpt-5.4" GH_AW_EVALS_PHASE: parse + GITHUB_RUN_ID: ${{ github.run_id }} with: script: | const { setupGlobals } = require('${{ runner.temp }}/gh-aw/actions/setup_globals.cjs'); diff --git a/actions/setup/js/run_evals.cjs b/actions/setup/js/run_evals.cjs index c379df655ac..c0da1c7ba00 100644 --- a/actions/setup/js/run_evals.cjs +++ b/actions/setup/js/run_evals.cjs @@ -106,6 +106,7 @@ async function setupMain() { async function parseMain() { const questionsRaw = process.env.GH_AW_EVALS_QUESTIONS; const model = process.env.GH_AW_EVALS_MODEL || ""; + const runID = process.env.GITHUB_RUN_ID || "unknown"; /** @type {Array<{id: string, question: string}>} */ let questions = []; @@ -147,6 +148,7 @@ async function parseMain() { answer, model, timestamp, + runid: runID, }; results.push(record); core.info(`Q[${q.id}]: ${answer}`); diff --git a/actions/setup/js/run_evals.test.cjs b/actions/setup/js/run_evals.test.cjs new file mode 100644 index 00000000000..8c42b7bddae --- /dev/null +++ b/actions/setup/js/run_evals.test.cjs @@ -0,0 +1,77 @@ +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import fs from "fs"; +import { createRequire } from "module"; + +const EVALS_DIR = "/tmp/gh-aw/evals"; +const EVALS_LOG_PATH = `${EVALS_DIR}/evals.log`; +const EVALS_OUTPUT_PATH = "/tmp/gh-aw/evals.jsonl"; +const require = createRequire(import.meta.url); +const { parseMain } = require("./run_evals.cjs"); + +const mockCore = { + info: vi.fn(), + warning: vi.fn(), + setFailed: vi.fn(), + exportVariable: vi.fn(), + summary: { + addDetails: vi.fn().mockReturnThis(), + write: vi.fn().mockResolvedValue(), + }, +}; + +global.core = mockCore; + +describe("run_evals.cjs", () => { + beforeEach(() => { + vi.clearAllMocks(); + fs.mkdirSync(EVALS_DIR, { recursive: true }); + if (fs.existsSync(EVALS_LOG_PATH)) { + fs.unlinkSync(EVALS_LOG_PATH); + } + if (fs.existsSync(EVALS_OUTPUT_PATH)) { + fs.unlinkSync(EVALS_OUTPUT_PATH); + } + }); + + afterEach(() => { + vi.unstubAllEnvs(); + if (fs.existsSync(EVALS_LOG_PATH)) { + fs.unlinkSync(EVALS_LOG_PATH); + } + if (fs.existsSync(EVALS_OUTPUT_PATH)) { + fs.unlinkSync(EVALS_OUTPUT_PATH); + } + }); + + it("stores the workflow run id when writing eval records", async () => { + vi.stubEnv("GH_AW_EVALS_QUESTIONS", JSON.stringify([{ id: "labels-applied", question: "Did labels get applied?" }])); + vi.stubEnv("GH_AW_EVALS_MODEL", "small"); + vi.stubEnv("GITHUB_RUN_ID", "123456789"); + fs.writeFileSync(EVALS_LOG_PATH, "labels-applied: YES\n", "utf8"); + + await parseMain(); + + const lines = fs.readFileSync(EVALS_OUTPUT_PATH, "utf8").trim().split("\n"); + expect(lines).toHaveLength(1); + expect(JSON.parse(lines[0])).toEqual({ + id: "labels-applied", + question: "Did labels get applied?", + answer: "YES", + model: "small", + timestamp: expect.any(String), + runid: "123456789", + }); + }); + + it('falls back to "unknown" when the workflow run id is absent', async () => { + vi.stubEnv("GH_AW_EVALS_QUESTIONS", JSON.stringify([{ id: "labels-applied", question: "Did labels get applied?" }])); + vi.stubEnv("GH_AW_EVALS_MODEL", "small"); + vi.stubEnv("GITHUB_RUN_ID", ""); + fs.writeFileSync(EVALS_LOG_PATH, "labels-applied: YES\n", "utf8"); + + await parseMain(); + + const [line] = fs.readFileSync(EVALS_OUTPUT_PATH, "utf8").trim().split("\n"); + expect(JSON.parse(line).runid).toBe("unknown"); + }); +}); diff --git a/pkg/workflow/evals_steps.go b/pkg/workflow/evals_steps.go index 64718732e1f..8a41b0d7cca 100644 --- a/pkg/workflow/evals_steps.go +++ b/pkg/workflow/evals_steps.go @@ -275,6 +275,7 @@ await main();` fmt.Sprintf(" GH_AW_EVALS_QUESTIONS: '%s'\n", escapeYAMLSingleQuoted(questionsJSON)), fmt.Sprintf(" GH_AW_EVALS_MODEL: %q\n", model), " GH_AW_EVALS_PHASE: parse\n", + " GITHUB_RUN_ID: ${{ github.run_id }}\n", " with:\n", " script: |\n", } diff --git a/pkg/workflow/evals_steps_test.go b/pkg/workflow/evals_steps_test.go index be018a585fa..c8d845bb240 100644 --- a/pkg/workflow/evals_steps_test.go +++ b/pkg/workflow/evals_steps_test.go @@ -142,6 +142,7 @@ func TestBuildEvalsJobStepsRenderSummary(t *testing.T) { if renderIdx >= 0 && uploadIdx >= 0 && renderIdx >= uploadIdx { t.Errorf("expected render step to appear before upload step; renderIdx=%d uploadIdx=%d", renderIdx, uploadIdx) } + } func TestBuildEvalsJobStepsRedactionUsesEvalsSecretReferences(t *testing.T) { @@ -192,4 +193,7 @@ func TestBuildParseEvalsResultsStepUsesResolvedExecutionModel(t *testing.T) { if strings.Contains(steps, `GH_AW_EVALS_MODEL: "small"`) { t.Errorf("expected parse step to avoid default 'small' when engine model is resolved; got:\n%s", steps) } + if !strings.Contains(steps, "GITHUB_RUN_ID: ${{ github.run_id }}") { + t.Errorf("expected parse step to pass github.run_id through to the eval record writer; got:\n%s", steps) + } }