diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 745ec274..b0138b2d 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -1,12 +1,18 @@ # End-to-End Integration Tests # # Verifies the open source local deployment works end-to-end. -# Walks through the `elmo init --dev` CLI flow in CI mode, starts services -# via Docker Compose (skipping worker to avoid execution costs), -# seeds test data, and runs Playwright tests against the Dockerized web app. +# Walks through the `elmo init --dev` CLI flow in CI mode, then runs in two +# phases against the Dockerized stack: with the worker down, seed data and run +# the fixture-dependent Playwright + Bruno suites; then start the worker +# (pointed at the no-network `stub` provider) and run the `worker` Playwright +# project, which submits a job and asserts the worker processes it. Starting +# the worker only after the fixture suites finish keeps its self-healing +# scheduler from mutating seeded data. # # This workflow is the SINGLE SOURCE OF TRUTH for E2E test orchestration. -# Run locally via: pnpm test:e2e (uses https://github.com/nektos/act) +# Run locally via: pnpm test:e2e-local (replays this workflow with +# https://github.com/nektos/act); `pnpm test:e2e` runs just the fixture specs +# against an already-running stack. name: E2E Tests @@ -30,6 +36,12 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 timeout-minutes: 30 + env: + # One file set for every `docker compose` invocation below, so no step + # can silently run against a different effective config. The override + # scopes the worker to the no-network stub provider. + COMPOSE_FILE: e2e/.elmo/elmo.yaml:e2e/worker-override.yaml + steps: - name: Checkout uses: actions/checkout@v7 @@ -100,19 +112,16 @@ jobs: grep -E '^\s{2}\w' e2e/.elmo/elmo.yaml || true # Build all images so a broken worker (or db-migrate) Dockerfile fails CI - # here, even though we don't run the worker. web/worker/migrate share the - # base/deps/builder stages, so BuildKit builds those once — building all - # three costs little more than building web alone. postgres is an image - # pull, so `build` skips it. + # here. web/worker/migrate share the base/deps/builder stages, so BuildKit + # builds those once — building all three costs little more than building + # web alone. postgres is an image pull, so `build` skips it. - name: Build images - run: docker compose -f e2e/.elmo/elmo.yaml build + run: docker compose build - # Start only web and its deps (db-migrate, postgres). The worker is - # intentionally not started (we skip it to avoid execution costs); it was - # only ever started to be immediately stopped, which gave no runtime - # safety, so we just don't start it. - - name: Start services - run: docker compose -f e2e/.elmo/elmo.yaml up -d --no-build web + # Start web only (with its deps db-migrate + postgres). The worker stays + # down for now — it comes up in phase 2, after the fixture suites run. + - name: Start web + run: docker compose up -d --no-build web - name: Wait for web run: | @@ -125,18 +134,35 @@ jobs: sleep 3 done echo "ERROR: Web did not become ready in time." - docker compose -f e2e/.elmo/elmo.yaml logs + docker compose logs exit 1 - name: Seed test data run: cd e2e && npx tsx seed.ts + # Phase 1 — worker still down. Run everything that reads the seeded + # fixtures, so no background job can mutate them mid-suite. - name: Run Playwright E2E tests - run: pnpm test:e2e + run: pnpm -C e2e exec playwright test --project=fixtures - name: Run Bruno API tests run: pnpm test:api + # Phase 2 — bring the worker up and prove it processes a submitted job. + # No readiness wait: the job the spec submits is durable in pg-boss, so + # the spec's own DB poll absorbs worker startup. + - name: Start worker + run: docker compose up -d --no-build worker + + # `github` keeps PR annotations on failure; omitting `html` leaves phase + # 1's playwright-report/ untouched for the upload below. + - name: Run worker job-processing test + run: pnpm -C e2e exec playwright test --project=worker --reporter=list,github + + - name: Dump worker logs + if: failure() + run: docker compose logs worker + - name: Upload Playwright report uses: actions/upload-artifact@v7 if: ${{ !cancelled() }} @@ -150,5 +176,7 @@ jobs: if: ${{ !cancelled() }} with: name: test-results - path: e2e/test-results/ + path: | + e2e/test-results/ + e2e/test-results-worker/ retention-days: 7 diff --git a/.gitignore b/.gitignore index e8ce94f4..22832aa6 100644 --- a/.gitignore +++ b/.gitignore @@ -45,6 +45,7 @@ storybook-static/ # playwright playwright-report/ test-results/ +test-results-worker/ blob-report/ # e2e generated config (created by `elmo init --dev --defaults`) diff --git a/e2e/auth-setup.ts b/e2e/auth-setup.ts index b68bea98..24d74462 100644 --- a/e2e/auth-setup.ts +++ b/e2e/auth-setup.ts @@ -7,10 +7,9 @@ */ import { chromium, type FullConfig } from "@playwright/test"; import pg from "pg"; -import { TEST_USER, TEST_BRAND_ID, TEST_BRAND_NAME } from "./seed"; +import { DATABASE_URL, TEST_USER, TEST_BRAND_ID, TEST_BRAND_NAME } from "./fixtures"; const AUTH_STATE_PATH = "e2e/.auth/user.json"; -const DATABASE_URL = "postgres://postgres:postgres@localhost:5432/elmo"; export default async function globalSetup(config: FullConfig) { const baseURL = config.projects[0]?.use?.baseURL || "http://localhost:1515"; diff --git a/e2e/fixtures.ts b/e2e/fixtures.ts new file mode 100644 index 00000000..ff05abad --- /dev/null +++ b/e2e/fixtures.ts @@ -0,0 +1,55 @@ +/** + * Shared E2E fixture constants — the single place the seeder, auth setup, and + * Playwright specs agree on IDs and credentials. Unlike seed.ts this module + * has no side effects, so specs can import from it freely. + */ + +// Hardcoded to localhost so the destructive seeder can never point at a +// production database. +export const DATABASE_URL = "postgres://postgres:postgres@localhost:5432/elmo"; + +// Must match ADMIN_API_KEYS in the CI-patched .env (.github/workflows/e2e.yaml) +// and bruno/environments/local.bru. +export const TEST_API_KEY = "test-api-key-e2e"; + +export const TEST_USER = { + email: "e2e@test.local", + password: "e2e-test-password-123", + name: "E2E Test User", +} as const; + +export const TEST_BRAND_ID = "default"; +export const TEST_BRAND_NAME = "Test Organization"; +export const TEST_BRAND_WEBSITE = "https://example.com"; + +export const PROMPT_IDS = { + branded1: "00000000-0000-0000-0000-000000000001", + branded2: "00000000-0000-0000-0000-000000000002", + unbranded1: "00000000-0000-0000-0000-000000000003", + branded3: "00000000-0000-0000-0000-000000000004", + unbranded2: "00000000-0000-0000-0000-000000000005", +} as const; + +export const COMPETITOR_IDS = { + competitorA: "00000000-0000-0000-0000-100000000001", + competitorB: "00000000-0000-0000-0000-100000000002", +} as const; + +export const REPORT_IDS = { + completed: "00000000-0000-0000-0000-300000000001", + pending: "00000000-0000-0000-0000-300000000002", + processing: "00000000-0000-0000-0000-300000000003", + failed: "00000000-0000-0000-0000-300000000004", +} as const; + +// Second tenant — a brand in an org the E2E user is NOT a member of. +export const NIKE_ORG_ID = "nike"; +export const NIKE_BRAND_ID = "nike"; +export const NIKE_PROMPT_IDS = { + training: "00000000-0000-0000-0000-400000000001", + lifestyle: "00000000-0000-0000-0000-400000000002", +} as const; +export const NIKE_COMPETITOR_IDS = { + adidas: "00000000-0000-0000-0000-410000000001", + puma: "00000000-0000-0000-0000-410000000002", +} as const; diff --git a/e2e/package.json b/e2e/package.json index fa3e8d5a..7b74fc2e 100644 --- a/e2e/package.json +++ b/e2e/package.json @@ -5,7 +5,7 @@ "license": "MIT", "type": "module", "scripts": { - "test:e2e": "playwright test", + "test:e2e": "playwright test --project=fixtures", "test:api": "mkdir -p test-results && cd bruno && bru run -r --env local --reporter-junit ../test-results/api-junit.xml --reporter-json ../test-results/api-report.json", "seed": "tsx seed.ts" }, diff --git a/e2e/playwright.config.ts b/e2e/playwright.config.ts index c2b26de5..3aa1761b 100644 --- a/e2e/playwright.config.ts +++ b/e2e/playwright.config.ts @@ -30,7 +30,22 @@ export default defineConfig({ projects: [ { - name: "chromium", + name: "fixtures", + testIgnore: /worker\.spec\.ts/, + use: { ...devices["Desktop Chrome"] }, + }, + // Run explicitly by CI phase 2 (--project=worker) once the worker + // container is up; `pnpm test:e2e` stays worker-free so a bare local run + // doesn't hang on (or feed a paid job to) whatever worker happens to be + // running. Separate outputDir because Playwright wipes the output dir of + // every project it runs — sharing test-results/ would delete phase 1's + // traces and the Bruno reports before CI uploads them. The timeout leaves + // room for the spec's 120s poll (worker startup + one pg-boss retry). + { + name: "worker", + testMatch: /worker\.spec\.ts/, + outputDir: "test-results-worker", + timeout: 150_000, use: { ...devices["Desktop Chrome"] }, }, ], diff --git a/e2e/seed.ts b/e2e/seed.ts index c46850ef..0f612c5a 100644 --- a/e2e/seed.ts +++ b/e2e/seed.ts @@ -3,59 +3,26 @@ * * Seeds the LOCAL test database with realistic fixture data for E2E testing. * - * SAFETY: Database URL is hardcoded to localhost to prevent - * accidentally running this against a production database (it DELETEs all data). + * SAFETY: the database URL (see fixtures.ts) is hardcoded to localhost to + * prevent accidentally running this against a production database (it DELETEs + * all data). * * Usage: tsx seed.ts */ import pg from "pg"; - -const DATABASE_URL = "postgres://postgres:postgres@localhost:5432/elmo"; - -// --------------------------------------------------------------------------- -// Fixed IDs for test fixtures (so tests can reference them directly) -// --------------------------------------------------------------------------- -export const TEST_USER = { - email: "e2e@test.local", - password: "e2e-test-password-123", - name: "E2E Test User", -} as const; - -export const TEST_BRAND_ID = "default"; -export const TEST_BRAND_NAME = "Test Organization"; -export const TEST_BRAND_WEBSITE = "https://example.com"; - -export const PROMPT_IDS = { - branded1: "00000000-0000-0000-0000-000000000001", - branded2: "00000000-0000-0000-0000-000000000002", - unbranded1: "00000000-0000-0000-0000-000000000003", - branded3: "00000000-0000-0000-0000-000000000004", - unbranded2: "00000000-0000-0000-0000-000000000005", -} as const; - -export const COMPETITOR_IDS = { - competitorA: "00000000-0000-0000-0000-100000000001", - competitorB: "00000000-0000-0000-0000-100000000002", -} as const; - -export const REPORT_IDS = { - completed: "00000000-0000-0000-0000-300000000001", - pending: "00000000-0000-0000-0000-300000000002", - processing: "00000000-0000-0000-0000-300000000003", - failed: "00000000-0000-0000-0000-300000000004", -} as const; - -// Second tenant — a brand in an org the E2E user is NOT a member of. -export const NIKE_ORG_ID = "nike"; -export const NIKE_BRAND_ID = "nike"; -export const NIKE_PROMPT_IDS = { - training: "00000000-0000-0000-0000-400000000001", - lifestyle: "00000000-0000-0000-0000-400000000002", -} as const; -export const NIKE_COMPETITOR_IDS = { - adidas: "00000000-0000-0000-0000-410000000001", - puma: "00000000-0000-0000-0000-410000000002", -} as const; +import { + COMPETITOR_IDS, + DATABASE_URL, + NIKE_BRAND_ID, + NIKE_COMPETITOR_IDS, + NIKE_ORG_ID, + NIKE_PROMPT_IDS, + PROMPT_IDS, + REPORT_IDS, + TEST_BRAND_ID, + TEST_BRAND_NAME, + TEST_BRAND_WEBSITE, +} from "./fixtures"; // Prompt run IDs (for prompt detail page testing) const RUN_IDS = [ diff --git a/e2e/tests/worker.spec.ts b/e2e/tests/worker.spec.ts new file mode 100644 index 00000000..aa49925a --- /dev/null +++ b/e2e/tests/worker.spec.ts @@ -0,0 +1,60 @@ +/** + * Worker Job Processing E2E Test + * + * Proves the background worker is wired up end to end: submitting a prompt + * through the public API enqueues a `process-prompt` pg-boss job, and the + * worker (running the no-network `stub` provider) dequeues it and records the + * run. We assert on the `prompt_runs` side effect rather than the job's own + * state so the test only passes if the handler actually did its work. + * + * This spec is the `worker` Playwright project and runs in its own phase, + * after the worker is started. The fixture-dependent specs (the `fixtures` + * project) run earlier while the worker is still down, so the worker's + * self-healing scheduler can't re-enqueue the seeded prompts and mutate data + * they assert on (see .github/workflows/e2e.yaml). + */ +import { test, expect } from "@playwright/test"; +import pg from "pg"; +import { DATABASE_URL, TEST_API_KEY, TEST_BRAND_ID } from "../fixtures"; + +test.describe("Worker job processing", () => { + test("a submitted prompt is dequeued and processed by the worker", async ({ request }) => { + // Submit: creating a prompt enqueues an immediate process-prompt job. + const value = "worker e2e — does a submitted job get processed?"; + const createRes = await request.post("/api/v1/prompts", { + headers: { Authorization: `Bearer ${TEST_API_KEY}` }, + data: { brandId: TEST_BRAND_ID, value }, + }); + expect(createRes.status(), await createRes.text()).toBe(201); + const prompt = (await createRes.json()) as { id: string }; + expect(prompt.id).toBeTruthy(); + + // Assert: poll until the worker records a run for this prompt. The stub + // provider is the worker's only configured target, so the run must carry + // its version — proof the job ran, not a stray write. The job is durable + // in pg-boss, so the poll budget absorbs worker startup (there is no + // separate readiness wait in CI) and covers one retry cycle of the + // process-prompt queue (retryDelay 60s). + const client = new pg.Client({ connectionString: DATABASE_URL }); + await client.connect(); + try { + await expect + .poll( + async () => { + const { rows } = await client.query<{ version: string }>( + `SELECT version FROM prompt_runs WHERE prompt_id = $1 LIMIT 1`, + [prompt.id], + ); + return rows[0]?.version; + }, + { + message: `worker did not record a prompt_run for ${prompt.id}`, + timeout: 120_000, + }, + ) + .toBe("stub"); + } finally { + await client.end(); + } + }); +}); diff --git a/e2e/worker-override.yaml b/e2e/worker-override.yaml new file mode 100644 index 00000000..a73107ab --- /dev/null +++ b/e2e/worker-override.yaml @@ -0,0 +1,9 @@ +# Compose override layered on top of the generated .elmo/elmo.yaml so the E2E +# worker processes jobs without any paid API calls: SCRAPE_TARGETS points at the +# no-network `stub` provider, so a submitted job records a prompt_run without +# reaching out to any LLM API. Worker-only — the web service keeps the real +# SCRAPE_TARGETS from .env. +services: + worker: + environment: + SCRAPE_TARGETS: stub:stub