Add run/artifact metadata to persisted evals records

actions/setup/js/run_evals.cjs

@@ -106,6 +106,7 @@ async function setupMain() {
 async function parseMain() {
   const questionsRaw = process.env.GH_AW_EVALS_QUESTIONS;
   const model = process.env.GH_AW_EVALS_MODEL || "";
+  const runID = process.env.GITHUB_RUN_ID || "unknown";
 
   /** @type {Array<{id: string, question: string}>} */
   let questions = [];
@@ -147,6 +148,7 @@ async function parseMain() {
       answer,
       model,
       timestamp,
+      runid: runID,
     };
     results.push(record);
     core.info(`Q[${q.id}]: ${answer}`);

actions/setup/js/run_evals.test.cjs

@@ -0,0 +1,77 @@
+import { afterEach, beforeEach, describe, expect, it, vi } from "vitest";
+import fs from "fs";
+import { createRequire } from "module";
+
+const EVALS_DIR = "/tmp/gh-aw/evals";
+const EVALS_LOG_PATH = `${EVALS_DIR}/evals.log`;
+const EVALS_OUTPUT_PATH = "/tmp/gh-aw/evals.jsonl";
+const require = createRequire(import.meta.url);
+const { parseMain } = require("./run_evals.cjs");
+
+const mockCore = {
+  info: vi.fn(),
+  warning: vi.fn(),
+  setFailed: vi.fn(),
+  exportVariable: vi.fn(),
+  summary: {
+    addDetails: vi.fn().mockReturnThis(),
+    write: vi.fn().mockResolvedValue(),
+  },
+};
+
+global.core = mockCore;
+
+describe("run_evals.cjs", () => {
+  beforeEach(() => {
+    vi.clearAllMocks();
+    fs.mkdirSync(EVALS_DIR, { recursive: true });
+    if (fs.existsSync(EVALS_LOG_PATH)) {
+      fs.unlinkSync(EVALS_LOG_PATH);
+    }
+    if (fs.existsSync(EVALS_OUTPUT_PATH)) {
+      fs.unlinkSync(EVALS_OUTPUT_PATH);
+    }
+  });
+
+  afterEach(() => {
+    vi.unstubAllEnvs();
+    if (fs.existsSync(EVALS_LOG_PATH)) {
+      fs.unlinkSync(EVALS_LOG_PATH);
+    }
+    if (fs.existsSync(EVALS_OUTPUT_PATH)) {
+      fs.unlinkSync(EVALS_OUTPUT_PATH);
+    }
+  });
+
+  it("stores the workflow run id when writing eval records", async () => {
+    vi.stubEnv("GH_AW_EVALS_QUESTIONS", JSON.stringify([{ id: "labels-applied", question: "Did labels get applied?" }]));
+    vi.stubEnv("GH_AW_EVALS_MODEL", "small");
+    vi.stubEnv("GITHUB_RUN_ID", "123456789");
+    fs.writeFileSync(EVALS_LOG_PATH, "labels-applied: YES\n", "utf8");
+
+    await parseMain();
+
+    const lines = fs.readFileSync(EVALS_OUTPUT_PATH, "utf8").trim().split("\n");
+    expect(lines).toHaveLength(1);
+    expect(JSON.parse(lines[0])).toEqual({
+      id: "labels-applied",
+      question: "Did labels get applied?",
+      answer: "YES",
+      model: "small",
+      timestamp: expect.any(String),
+      runid: "123456789",
+    });
+  });
+
+  it('falls back to "unknown" when the workflow run id is absent', async () => {
+    vi.stubEnv("GH_AW_EVALS_QUESTIONS", JSON.stringify([{ id: "labels-applied", question: "Did labels get applied?" }]));
+    vi.stubEnv("GH_AW_EVALS_MODEL", "small");
+    vi.stubEnv("GITHUB_RUN_ID", "");
+    fs.writeFileSync(EVALS_LOG_PATH, "labels-applied: YES\n", "utf8");
+
+    await parseMain();
+
+    const [line] = fs.readFileSync(EVALS_OUTPUT_PATH, "utf8").trim().split("\n");
+    expect(JSON.parse(line).runid).toBe("unknown");
+  });
+});

pkg/workflow/evals_steps.go

@@ -275,6 +275,7 @@ await main();`
 		fmt.Sprintf("          GH_AW_EVALS_QUESTIONS: '%s'\n", escapeYAMLSingleQuoted(questionsJSON)),
 		fmt.Sprintf("          GH_AW_EVALS_MODEL: %q\n", model),
 		"          GH_AW_EVALS_PHASE: parse\n",
+		"          GITHUB_RUN_ID: ${{ github.run_id }}\n",
 		"        with:\n",
 		"          script: |\n",
 	}

pkg/workflow/evals_steps_test.go

@@ -142,6 +142,7 @@ func TestBuildEvalsJobStepsRenderSummary(t *testing.T) {
 	if renderIdx >= 0 && uploadIdx >= 0 && renderIdx >= uploadIdx {
 		t.Errorf("expected render step to appear before upload step; renderIdx=%d uploadIdx=%d", renderIdx, uploadIdx)
 	}
+
 }
 
 func TestBuildEvalsJobStepsRedactionUsesEvalsSecretReferences(t *testing.T) {
@@ -192,4 +193,7 @@ func TestBuildParseEvalsResultsStepUsesResolvedExecutionModel(t *testing.T) {
 	if strings.Contains(steps, `GH_AW_EVALS_MODEL: "small"`) {
 		t.Errorf("expected parse step to avoid default 'small' when engine model is resolved; got:\n%s", steps)
 	}
+	if !strings.Contains(steps, "GITHUB_RUN_ID: ${{ github.run_id }}") {
+		t.Errorf("expected parse step to pass github.run_id through to the eval record writer; got:\n%s", steps)
+	}
 }