test: version-agnostic SARIF fixture + observer server/screencast coverage

docs/integration/fixture-sarif.json

@@ -6,7 +6,7 @@
       "tool": {
         "driver": {
           "name": "pixelcheck",
-          "version": "1.3.0",
+          "version": "0.0.0-fixture",
           "informationUri": "https://github.com/xcodethink/pixelcheck",
           "rules": [
             {

scripts/gen-sarif-fixture.ts

@@ -97,6 +97,15 @@ const audit: AuditRun = {
 };
 
 const sarif = renderSarif(audit);
+// Neutralize the tool driver version (mirrors package.json) to a stable
+// sentinel so the committed fixture doesn't need a re-pin every release. The
+// matching test (wcag-axe.test.ts) normalizes the generated SARIF the same
+// way before comparing. The SARIF schema version (top-level) is untouched.
+const driver = (sarif as { runs?: Array<{ tool?: { driver?: { version?: string } } }> })
+  .runs?.[0]?.tool?.driver;
+if (driver && typeof driver.version === "string") {
+  driver.version = "0.0.0-fixture";
+}
 const out = path.join(process.cwd(), "docs/integration/fixture-sarif.json");
 fs.mkdirSync(path.dirname(out), { recursive: true });
 fs.writeFileSync(out, JSON.stringify(sarif, null, 2));

tests/integration/playwright/wcag-axe.test.ts

@@ -328,6 +328,23 @@ test.describe("SARIF render + write pipeline", () => {
     //   (a) renderSarif logic changed → review and update fixture via:
     //       npx tsx scripts/gen-sarif-fixture.ts
     //   (b) regression introduced — investigate before committing.
+    //
+    // The tool `driver.version` mirrors package.json, so it changes every
+    // release. We normalize it to a sentinel on BOTH sides before comparing,
+    // so a version bump no longer breaks this test (and the fixture never
+    // needs a per-release re-pin). The SARIF schema version (top-level
+    // `version`) is left untouched — only the nested driver version is
+    // neutralized, structurally, to avoid string-collision with it.
+    const VERSION_SENTINEL = "0.0.0-fixture";
+    const neutralizeDriverVersion = (raw: string): string => {
+      const obj = JSON.parse(raw);
+      const driver = obj?.runs?.[0]?.tool?.driver;
+      if (driver && typeof driver.version === "string") {
+        driver.version = VERSION_SENTINEL;
+      }
+      return JSON.stringify(obj, null, 2);
+    };
+
     const audit = makeAuditWithWcagIssues();
     const sarif = renderSarif(audit);
     const generated = JSON.stringify(sarif, null, 2);
@@ -338,6 +355,8 @@ test.describe("SARIF render + write pipeline", () => {
     );
     const committed = fs.readFileSync(fixturePath, "utf8").trim();
 
-    expect(generated.trim()).toBe(committed);
+    expect(neutralizeDriverVersion(generated)).toBe(
+      neutralizeDriverVersion(committed),
+    );
   });
 });

tests/observer-screencast.test.ts

@@ -0,0 +1,103 @@
+import { describe, it, expect, vi } from "vitest";
+import type { Page } from "playwright";
+import { startScreencast } from "../src/observer/screencast.js";
+
+/** Minimal fake CDP session: records sends, lets tests fire frame events. */
+function makeFakeCdp() {
+  const handlers: Record<string, (params: unknown) => void> = {};
+  const send = vi.fn(async (_method: string, _params?: unknown) => {});
+  const detach = vi.fn(async () => {});
+  return {
+    on: (evt: string, cb: (p: unknown) => void) => {
+      handlers[evt] = cb;
+    },
+    send,
+    detach,
+    emitFrame: (params: unknown) => handlers["Page.screencastFrame"]?.(params),
+  };
+}
+
+function makePage(cdp: ReturnType<typeof makeFakeCdp> | null): Page {
+  return {
+    context: () => ({
+      newCDPSession: vi.fn(async () => {
+        if (!cdp) throw new Error("CDP not available in this context");
+        return cdp;
+      }),
+    }),
+  } as unknown as Page;
+}
+
+describe("observer screencast (G3 follow-up)", () => {
+  it("starts the CDP screencast with documented defaults", async () => {
+    const cdp = makeFakeCdp();
+    await startScreencast(makePage(cdp), () => {});
+    const startCall = cdp.send.mock.calls.find((c) => c[0] === "Page.startScreencast");
+    expect(startCall).toBeDefined();
+    expect(startCall![1]).toEqual({
+      format: "jpeg",
+      quality: 50,
+      maxWidth: 800,
+      maxHeight: 600,
+      everyNthFrame: 3,
+    });
+  });
+
+  it("honors custom options", async () => {
+    const cdp = makeFakeCdp();
+    await startScreencast(makePage(cdp), () => {}, {
+      format: "png",
+      quality: 80,
+      maxWidth: 1280,
+      maxHeight: 720,
+      everyNthFrame: 1,
+    });
+    const startCall = cdp.send.mock.calls.find((c) => c[0] === "Page.startScreencast");
+    expect(startCall![1]).toEqual({
+      format: "png",
+      quality: 80,
+      maxWidth: 1280,
+      maxHeight: 720,
+      everyNthFrame: 1,
+    });
+  });
+
+  it("forwards each frame to onFrame and acks it", async () => {
+    const cdp = makeFakeCdp();
+    const frames: Array<{ data: string; ts: number }> = [];
+    await startScreencast(makePage(cdp), (data, meta) => frames.push({ data, ts: meta.timestamp }));
+
+    cdp.emitFrame({ data: "BASE64DATA", metadata: { timestamp: 1234 }, sessionId: "sess-1" });
+
+    expect(frames).toEqual([{ data: "BASE64DATA", ts: 1234 }]);
+    const ack = cdp.send.mock.calls.find((c) => c[0] === "Page.screencastFrameAck");
+    expect(ack).toBeDefined();
+    expect(ack![1]).toEqual({ sessionId: "sess-1" });
+  });
+
+  it("falls back to a timestamp when the frame metadata omits one", async () => {
+    const cdp = makeFakeCdp();
+    let ts: number | undefined;
+    await startScreencast(makePage(cdp), (_d, meta) => {
+      ts = meta.timestamp;
+    });
+    cdp.emitFrame({ data: "x", metadata: {}, sessionId: "s" });
+    expect(typeof ts).toBe("number");
+  });
+
+  it("stop() stops the screencast and detaches the CDP session", async () => {
+    const cdp = makeFakeCdp();
+    const handle = await startScreencast(makePage(cdp), () => {});
+    await handle.stop();
+    expect(cdp.send.mock.calls.some((c) => c[0] === "Page.stopScreencast")).toBe(true);
+    expect(cdp.detach).toHaveBeenCalledOnce();
+  });
+
+  it("degrades to a no-op when CDP is unavailable (no throw, no frames)", async () => {
+    const onFrame = vi.fn();
+    const handle = await startScreencast(makePage(null), onFrame);
+    // Returns a usable handle whose stop() is safe to call.
+    await expect(handle.stop()).resolves.toBeUndefined();
+    expect(onFrame).not.toHaveBeenCalled();
+  });
+});

tests/observer-server.test.ts

@@ -0,0 +1,181 @@
+import { describe, it, expect, beforeAll, afterAll, vi } from "vitest";
+import { WebSocket } from "ws";
+import type { Server } from "node:http";
+import { ObserverServer } from "../src/observer/server.js";
+import { AgentEventBus } from "../src/agent/events.js";
+import { SessionStore } from "../src/observer/session-store.js";
+import { SessionRegistry } from "../src/observer/session-registry.js";
+
+/** Read the OS-assigned port (server bound to :0) off the running instance. */
+function portOf(server: ObserverServer): number {
+  const http = (server as unknown as { _httpServer: Server })._httpServer;
+  const addr = http.address();
+  if (addr && typeof addr === "object" && addr.port) return addr.port;
+  throw new Error("observer server has no bound port");
+}
+
+function openWs(url: string): Promise<WebSocket> {
+  return new Promise((resolve, reject) => {
+    const ws = new WebSocket(url);
+    ws.on("open", () => resolve(ws));
+    ws.on("error", reject);
+  });
+}
+
+describe("observer HTTP/WS server (G3 follow-up)", () => {
+  let bus: AgentEventBus;
+  let store: SessionStore;
+  let server: ObserverServer;
+  let base: string;
+  let wsBase: string;
+  let token: string;
+
+  beforeAll(async () => {
+    bus = new AgentEventBus("test-session");
+    store = new SessionStore("test-session");
+    const registry = new SessionRegistry("test-root");
+    server = new ObserverServer({ port: 0, eventBus: bus, sessionStore: store, registry });
+    await server.start();
+    const p = portOf(server);
+    base = `http://127.0.0.1:${p}`;
+    wsBase = `ws://127.0.0.1:${p}`;
+    token = server.token;
+  });
+
+  afterAll(async () => {
+    await server.stop();
+  });
+
+  it("mints a 32-hex bearer token", () => {
+    expect(token).toMatch(/^[0-9a-f]{32}$/);
+  });
+
+  it("serves the dashboard at / without auth", async () => {
+    const res = await fetch(`${base}/`);
+    expect(res.status).toBe(200);
+    expect(res.headers.get("content-type")).toContain("text/html");
+    expect(await res.text()).toContain("<!DOCTYPE html>");
+  });
+
+  it("serves the grid dashboard + /api/grid snapshot (registry attached, no auth)", async () => {
+    const grid = await fetch(`${base}/grid`);
+    expect(grid.status).toBe(200);
+    expect(await grid.text()).toContain("PixelCheck");
+
+    const snap = await fetch(`${base}/api/grid`);
+    expect(snap.status).toBe(200);
+    expect(Array.isArray(await snap.json())).toBe(true);
+  });
+
+  it("404s an unknown session id", async () => {
+    expect((await fetch(`${base}/api/session/nope`)).status).toBe(404);
+  });
+
+  it("requires a token for /api/* data routes", async () => {
+    expect((await fetch(`${base}/api/state`)).status).toBe(401);
+  });
+
+  it("serves the exact-match data routes when authed via Authorization header", async () => {
+    // NOTE: /api/state, /api/events, /api/timeline are matched with `url ===`,
+    // which includes the query string — so they only resolve when there is NO
+    // query (i.e. token supplied via the Authorization header, not ?token=).
+    // The query-bearing routes below use startsWith and accept ?token=.
+    // (Flagged as a server routing quirk; not changed here — test task.)
+    const hdr = { headers: { authorization: `Bearer ${token}` } };
+    for (const route of ["/api/state", "/api/events", "/api/timeline"]) {
+      expect((await fetch(`${base}${route}`, hdr)).status, route).toBe(200);
+    }
+  });
+
+  it("serves the query-bearing data routes with ?token=", async () => {
+    for (const route of ["/api/events/all?start=0", "/api/screenshot?seq=0"]) {
+      const res = await fetch(`${base}${route}&token=${token}`);
+      expect(res.status, route).toBe(200);
+    }
+  });
+
+  it("404s unknown routes", async () => {
+    expect((await fetch(`${base}/totally-unknown`)).status).toBe(404);
+  });
+
+  it("closes a WS connection that presents a bad token (code 4001)", async () => {
+    const ws = new WebSocket(`${wsBase}/ws?token=WRONG`);
+    const code = await new Promise<number>((resolve) => {
+      ws.on("close", (c) => resolve(c));
+      ws.on("error", () => {});
+    });
+    expect(code).toBe(4001);
+  });
+
+  it("sends an init payload to a good-token WS client", async () => {
+    // Attach the message listener synchronously at construction — the server
+    // sends `init` immediately on connect, so awaiting `open` first can race
+    // past the first frame.
+    const ws = new WebSocket(`${wsBase}/ws?token=${token}`);
+    try {
+      const msg = await new Promise<{ type: string; payload: Record<string, unknown> }>(
+        (resolve, reject) => {
+          ws.on("message", (d) => resolve(JSON.parse(d.toString())));
+          ws.on("error", reject);
+        },
+      );
+      expect(msg.type).toBe("init");
+      expect(msg.payload).toHaveProperty("state");
+      expect(msg.payload).toHaveProperty("recentEvents");
+    } finally {
+      ws.close();
+    }
+  });
+
+  it("dispatches an allowlisted command to the event bus, ignores unknown ones", async () => {
+    const pauseSpy = vi.spyOn(bus, "pause");
+    const ws = await openWs(`${wsBase}/ws?token=${token}`);
+    try {
+      ws.send(JSON.stringify({ command: "nonsense" }));
+      ws.send(JSON.stringify({ command: "pause" }));
+      await new Promise((r) => setTimeout(r, 120));
+      expect(pauseSpy).toHaveBeenCalledTimes(1);
+    } finally {
+      ws.close();
+      pauseSpy.mockRestore();
+    }
+  });
+
+  it("broadcasts a screencast frame as binary to connected clients", async () => {
+    const ws = await openWs(`${wsBase}/ws?token=${token}`);
+    ws.binaryType = "nodebuffer";
+    try {
+      const binary = new Promise<Buffer>((resolve) => {
+        ws.on("message", (d, isBinary) => {
+          if (isBinary) resolve(d as Buffer);
+        });
+      });
+      // give the server a tick to register the client before broadcasting
+      await new Promise((r) => setTimeout(r, 50));
+      server.broadcastFrame(Buffer.from("frame-bytes").toString("base64"));
+      expect((await binary).toString()).toBe("frame-bytes");
+    } finally {
+      ws.close();
+    }
+  });
+
+  it("relays event-bus events to connected clients", async () => {
+    const ws = await openWs(`${wsBase}/ws?token=${token}`);
+    try {
+      const eventMsg = new Promise<{ type: string }>((resolve) => {
+        ws.on("message", (d, isBinary) => {
+          if (isBinary) return;
+          const m = JSON.parse(d.toString());
+          if (m.type === "event") resolve(m);
+        });
+      });
+      await new Promise((r) => setTimeout(r, 50));
+      // emitEvent does the dual-emit (type + "*"); the server listens on "*".
+      bus.emitEvent("session:start", {});
+      const m = await eventMsg;
+      expect(m.type).toBe("event");
+    } finally {
+      ws.close();
+    }
+  });
+});