Run the worker in e2e and verify a submitted job is processed

.github/workflows/e2e.yaml

@@ -1,12 +1,18 @@
 # End-to-End Integration Tests
 #
 # Verifies the open source local deployment works end-to-end.
-# Walks through the `elmo init --dev` CLI flow in CI mode, starts services
-# via Docker Compose (skipping worker to avoid execution costs),
-# seeds test data, and runs Playwright tests against the Dockerized web app.
+# Walks through the `elmo init --dev` CLI flow in CI mode, then runs in two
+# phases against the Dockerized stack: with the worker down, seed data and run
+# the fixture-dependent Playwright + Bruno suites; then start the worker
+# (pointed at the no-network `stub` provider) and run the `worker` Playwright
+# project, which submits a job and asserts the worker processes it. Starting
+# the worker only after the fixture suites finish keeps its self-healing
+# scheduler from mutating seeded data.
 #
 # This workflow is the SINGLE SOURCE OF TRUTH for E2E test orchestration.
-# Run locally via: pnpm test:e2e  (uses https://github.com/nektos/act)
+# Run locally via: pnpm test:e2e-local  (replays this workflow with
+# https://github.com/nektos/act); `pnpm test:e2e` runs just the fixture specs
+# against an already-running stack.
 
 name: E2E Tests
 
@@ -30,6 +36,12 @@ jobs:
     runs-on: blacksmith-4vcpu-ubuntu-2404
     timeout-minutes: 30
 
+    env:
+      # One file set for every `docker compose` invocation below, so no step
+      # can silently run against a different effective config. The override
+      # scopes the worker to the no-network stub provider.
+      COMPOSE_FILE: e2e/.elmo/elmo.yaml:e2e/worker-override.yaml
+
     steps:
       - name: Checkout
         uses: actions/checkout@v7
@@ -100,19 +112,16 @@ jobs:
           grep -E '^\s{2}\w' e2e/.elmo/elmo.yaml || true
 
       # Build all images so a broken worker (or db-migrate) Dockerfile fails CI
-      # here, even though we don't run the worker. web/worker/migrate share the
-      # base/deps/builder stages, so BuildKit builds those once — building all
-      # three costs little more than building web alone. postgres is an image
-      # pull, so `build` skips it.
+      # here. web/worker/migrate share the base/deps/builder stages, so BuildKit
+      # builds those once — building all three costs little more than building
+      # web alone. postgres is an image pull, so `build` skips it.
       - name: Build images
-        run: docker compose -f e2e/.elmo/elmo.yaml build
+        run: docker compose build
 
-      # Start only web and its deps (db-migrate, postgres). The worker is
-      # intentionally not started (we skip it to avoid execution costs); it was
-      # only ever started to be immediately stopped, which gave no runtime
-      # safety, so we just don't start it.
-      - name: Start services
-        run: docker compose -f e2e/.elmo/elmo.yaml up -d --no-build web
+      # Start web only (with its deps db-migrate + postgres). The worker stays
+      # down for now — it comes up in phase 2, after the fixture suites run.
+      - name: Start web
+        run: docker compose up -d --no-build web
 
       - name: Wait for web
         run: |
@@ -125,18 +134,35 @@ jobs:
             sleep 3
           done
           echo "ERROR: Web did not become ready in time."
-          docker compose -f e2e/.elmo/elmo.yaml logs
+          docker compose logs
           exit 1
 
       - name: Seed test data
         run: cd e2e && npx tsx seed.ts
 
+      # Phase 1 — worker still down. Run everything that reads the seeded
+      # fixtures, so no background job can mutate them mid-suite.
       - name: Run Playwright E2E tests
-        run: pnpm test:e2e
+        run: pnpm -C e2e exec playwright test --project=fixtures
 
       - name: Run Bruno API tests
         run: pnpm test:api
 
+      # Phase 2 — bring the worker up and prove it processes a submitted job.
+      # No readiness wait: the job the spec submits is durable in pg-boss, so
+      # the spec's own DB poll absorbs worker startup.
+      - name: Start worker
+        run: docker compose up -d --no-build worker
+
+      # `github` keeps PR annotations on failure; omitting `html` leaves phase
+      # 1's playwright-report/ untouched for the upload below.
+      - name: Run worker job-processing test
+        run: pnpm -C e2e exec playwright test --project=worker --reporter=list,github
+
+      - name: Dump worker logs
+        if: failure()
+        run: docker compose logs worker
+
       - name: Upload Playwright report
         uses: actions/upload-artifact@v7
         if: ${{ !cancelled() }}
@@ -150,5 +176,7 @@ jobs:
         if: ${{ !cancelled() }}
         with:
           name: test-results
-          path: e2e/test-results/
+          path: |
+            e2e/test-results/
+            e2e/test-results-worker/
           retention-days: 7

.gitignore

@@ -45,6 +45,7 @@ storybook-static/
 # playwright
 playwright-report/
 test-results/
+test-results-worker/
 blob-report/
 
 # e2e generated config (created by `elmo init --dev --defaults`)

e2e/auth-setup.ts

@@ -7,10 +7,9 @@
  */
 import { chromium, type FullConfig } from "@playwright/test";
 import pg from "pg";
-import { TEST_USER, TEST_BRAND_ID, TEST_BRAND_NAME } from "./seed";
+import { DATABASE_URL, TEST_USER, TEST_BRAND_ID, TEST_BRAND_NAME } from "./fixtures";
 
 const AUTH_STATE_PATH = "e2e/.auth/user.json";
-const DATABASE_URL = "postgres://postgres:postgres@localhost:5432/elmo";
 
 export default async function globalSetup(config: FullConfig) {
 	const baseURL = config.projects[0]?.use?.baseURL || "http://localhost:1515";

e2e/fixtures.ts

@@ -0,0 +1,55 @@
+/**
+ * Shared E2E fixture constants — the single place the seeder, auth setup, and
+ * Playwright specs agree on IDs and credentials. Unlike seed.ts this module
+ * has no side effects, so specs can import from it freely.
+ */
+
+// Hardcoded to localhost so the destructive seeder can never point at a
+// production database.
+export const DATABASE_URL = "postgres://postgres:postgres@localhost:5432/elmo";
+
+// Must match ADMIN_API_KEYS in the CI-patched .env (.github/workflows/e2e.yaml)
+// and bruno/environments/local.bru.
+export const TEST_API_KEY = "test-api-key-e2e";
+
+export const TEST_USER = {
+  email: "e2e@test.local",
+  password: "e2e-test-password-123",
+  name: "E2E Test User",
+} as const;
+
+export const TEST_BRAND_ID = "default";
+export const TEST_BRAND_NAME = "Test Organization";
+export const TEST_BRAND_WEBSITE = "https://example.com";
+
+export const PROMPT_IDS = {
+  branded1: "00000000-0000-0000-0000-000000000001",
+  branded2: "00000000-0000-0000-0000-000000000002",
+  unbranded1: "00000000-0000-0000-0000-000000000003",
+  branded3: "00000000-0000-0000-0000-000000000004",
+  unbranded2: "00000000-0000-0000-0000-000000000005",
+} as const;
+
+export const COMPETITOR_IDS = {
+  competitorA: "00000000-0000-0000-0000-100000000001",
+  competitorB: "00000000-0000-0000-0000-100000000002",
+} as const;
+
+export const REPORT_IDS = {
+  completed: "00000000-0000-0000-0000-300000000001",
+  pending: "00000000-0000-0000-0000-300000000002",
+  processing: "00000000-0000-0000-0000-300000000003",
+  failed: "00000000-0000-0000-0000-300000000004",
+} as const;
+
+// Second tenant — a brand in an org the E2E user is NOT a member of.
+export const NIKE_ORG_ID = "nike";
+export const NIKE_BRAND_ID = "nike";
+export const NIKE_PROMPT_IDS = {
+  training: "00000000-0000-0000-0000-400000000001",
+  lifestyle: "00000000-0000-0000-0000-400000000002",
+} as const;
+export const NIKE_COMPETITOR_IDS = {
+  adidas: "00000000-0000-0000-0000-410000000001",
+  puma: "00000000-0000-0000-0000-410000000002",
+} as const;

e2e/package.json

@@ -5,7 +5,7 @@
   "license": "MIT",
   "type": "module",
   "scripts": {
-    "test:e2e": "playwright test",
+    "test:e2e": "playwright test --project=fixtures",
     "test:api": "mkdir -p test-results && cd bruno && bru run -r --env local --reporter-junit ../test-results/api-junit.xml --reporter-json ../test-results/api-report.json",
     "seed": "tsx seed.ts"
   },

e2e/playwright.config.ts

@@ -30,7 +30,22 @@ export default defineConfig({
 
   projects: [
     {
-      name: "chromium",
+      name: "fixtures",
+      testIgnore: /worker\.spec\.ts/,
+      use: { ...devices["Desktop Chrome"] },
+    },
+    // Run explicitly by CI phase 2 (--project=worker) once the worker
+    // container is up; `pnpm test:e2e` stays worker-free so a bare local run
+    // doesn't hang on (or feed a paid job to) whatever worker happens to be
+    // running. Separate outputDir because Playwright wipes the output dir of
+    // every project it runs — sharing test-results/ would delete phase 1's
+    // traces and the Bruno reports before CI uploads them. The timeout leaves
+    // room for the spec's 120s poll (worker startup + one pg-boss retry).
+    {
+      name: "worker",
+      testMatch: /worker\.spec\.ts/,
+      outputDir: "test-results-worker",
+      timeout: 150_000,
       use: { ...devices["Desktop Chrome"] },
     },
   ],

e2e/seed.ts

@@ -3,59 +3,26 @@
  *
  * Seeds the LOCAL test database with realistic fixture data for E2E testing.
  *
- * SAFETY: Database URL is hardcoded to localhost to prevent
- * accidentally running this against a production database (it DELETEs all data).
+ * SAFETY: the database URL (see fixtures.ts) is hardcoded to localhost to
+ * prevent accidentally running this against a production database (it DELETEs
+ * all data).
  *
  * Usage: tsx seed.ts
  */
 import pg from "pg";
-
-const DATABASE_URL = "postgres://postgres:postgres@localhost:5432/elmo";
-
-// ---------------------------------------------------------------------------
-// Fixed IDs for test fixtures (so tests can reference them directly)
-// ---------------------------------------------------------------------------
-export const TEST_USER = {
-  email: "e2e@test.local",
-  password: "e2e-test-password-123",
-  name: "E2E Test User",
-} as const;
-
-export const TEST_BRAND_ID = "default";
-export const TEST_BRAND_NAME = "Test Organization";
-export const TEST_BRAND_WEBSITE = "https://example.com";
-
-export const PROMPT_IDS = {
-  branded1: "00000000-0000-0000-0000-000000000001",
-  branded2: "00000000-0000-0000-0000-000000000002",
-  unbranded1: "00000000-0000-0000-0000-000000000003",
-  branded3: "00000000-0000-0000-0000-000000000004",
-  unbranded2: "00000000-0000-0000-0000-000000000005",
-} as const;
-
-export const COMPETITOR_IDS = {
-  competitorA: "00000000-0000-0000-0000-100000000001",
-  competitorB: "00000000-0000-0000-0000-100000000002",
-} as const;
-
-export const REPORT_IDS = {
-  completed: "00000000-0000-0000-0000-300000000001",
-  pending: "00000000-0000-0000-0000-300000000002",
-  processing: "00000000-0000-0000-0000-300000000003",
-  failed: "00000000-0000-0000-0000-300000000004",
-} as const;
-
-// Second tenant — a brand in an org the E2E user is NOT a member of.
-export const NIKE_ORG_ID = "nike";
-export const NIKE_BRAND_ID = "nike";
-export const NIKE_PROMPT_IDS = {
-  training: "00000000-0000-0000-0000-400000000001",
-  lifestyle: "00000000-0000-0000-0000-400000000002",
-} as const;
-export const NIKE_COMPETITOR_IDS = {
-  adidas: "00000000-0000-0000-0000-410000000001",
-  puma: "00000000-0000-0000-0000-410000000002",
-} as const;
+import {
+  COMPETITOR_IDS,
+  DATABASE_URL,
+  NIKE_BRAND_ID,
+  NIKE_COMPETITOR_IDS,
+  NIKE_ORG_ID,
+  NIKE_PROMPT_IDS,
+  PROMPT_IDS,
+  REPORT_IDS,
+  TEST_BRAND_ID,
+  TEST_BRAND_NAME,
+  TEST_BRAND_WEBSITE,
+} from "./fixtures";
 
 // Prompt run IDs (for prompt detail page testing)
 const RUN_IDS = [

e2e/tests/worker.spec.ts

@@ -0,0 +1,60 @@
+/**
+ * Worker Job Processing E2E Test
+ *
+ * Proves the background worker is wired up end to end: submitting a prompt
+ * through the public API enqueues a `process-prompt` pg-boss job, and the
+ * worker (running the no-network `stub` provider) dequeues it and records the
+ * run. We assert on the `prompt_runs` side effect rather than the job's own
+ * state so the test only passes if the handler actually did its work.
+ *
+ * This spec is the `worker` Playwright project and runs in its own phase,
+ * after the worker is started. The fixture-dependent specs (the `fixtures`
+ * project) run earlier while the worker is still down, so the worker's
+ * self-healing scheduler can't re-enqueue the seeded prompts and mutate data
+ * they assert on (see .github/workflows/e2e.yaml).
+ */
+import { test, expect } from "@playwright/test";
+import pg from "pg";
+import { DATABASE_URL, TEST_API_KEY, TEST_BRAND_ID } from "../fixtures";
+
+test.describe("Worker job processing", () => {
+  test("a submitted prompt is dequeued and processed by the worker", async ({ request }) => {
+    // Submit: creating a prompt enqueues an immediate process-prompt job.
+    const value = "worker e2e — does a submitted job get processed?";
+    const createRes = await request.post("/api/v1/prompts", {
+      headers: { Authorization: `Bearer ${TEST_API_KEY}` },
+      data: { brandId: TEST_BRAND_ID, value },
+    });
+    expect(createRes.status(), await createRes.text()).toBe(201);
+    const prompt = (await createRes.json()) as { id: string };
+    expect(prompt.id).toBeTruthy();
+
+    // Assert: poll until the worker records a run for this prompt. The stub
+    // provider is the worker's only configured target, so the run must carry
+    // its version — proof the job ran, not a stray write. The job is durable
+    // in pg-boss, so the poll budget absorbs worker startup (there is no
+    // separate readiness wait in CI) and covers one retry cycle of the
+    // process-prompt queue (retryDelay 60s).
+    const client = new pg.Client({ connectionString: DATABASE_URL });
+    await client.connect();
+    try {
+      await expect
+        .poll(
+          async () => {
+            const { rows } = await client.query<{ version: string }>(
+              `SELECT version FROM prompt_runs WHERE prompt_id = $1 LIMIT 1`,
+              [prompt.id],
+            );
+            return rows[0]?.version;
+          },
+          {
+            message: `worker did not record a prompt_run for ${prompt.id}`,
+            timeout: 120_000,
+          },
+        )
+        .toBe("stub");
+    } finally {
+      await client.end();
+    }
+  });
+});

e2e/worker-override.yaml

@@ -0,0 +1,9 @@
+# Compose override layered on top of the generated .elmo/elmo.yaml so the E2E
+# worker processes jobs without any paid API calls: SCRAPE_TARGETS points at the
+# no-network `stub` provider, so a submitted job records a prompt_run without
+# reaching out to any LLM API. Worker-only — the web service keeps the real
+# SCRAPE_TARGETS from .env.
+services:
+  worker:
+    environment:
+      SCRAPE_TARGETS: stub:stub