chore: migrate to chromatic for CI testing

Signed-off-by: Ulises Gascon <[email protected]>

Author: UlisesGascon

.github/workflows/playwright.yml

@@ -9,23 +9,31 @@ jobs:
   test-docs:
     name: E2E Testing
     runs-on: ubuntu-latest
-    env:
-      CI: true
     defaults:
       run:
         working-directory: scorecards-site
 
     steps:
       - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 #v5
+        with:
+          fetch-depth: 0
       - uses: actions/setup-node@a0853c24544627f65ddf259abe73b1d18a591444 #v5
         with:
           node-version: 22
 
       - name: Install dependencies
         run: |
           yarn install --frozen-lockfile
-          yarn add -D @playwright/test@latest
-          npx playwright install --with-deps
+          yarn add -D @playwright/test chromatic @chromatic-com/playwright
+          yarn playwright install --with-deps
 
       - name: Run Playwright tests
-        run: npx playwright test
+        run: yarn playwright test
+        env:
+          CHROMATIC_PROJECT_TOKEN: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
+
+      - name: Run test and Upload the generated Chromatic archives
+        run: yarn chromatic --playwright -t="$CHROMATIC_PROJECT_TOKEN"
+        env:
+          CHROMATIC_PROJECT_TOKEN: ${{ secrets.CHROMATIC_PROJECT_TOKEN }}
+          CI: true

scorecards-site/README.md

@@ -37,8 +37,7 @@ To run tests locally, you must install Playwright manually and avoid committing
 ### Temporary local installation
 
 ```sh
-yarn add -D @playwright/test@latest
-yarn playwright install --with-deps
+yarn add -D @playwright/test chromatic @chromatic-com/playwright
 ```
 
 These changes are temporary and should not be committed.

scorecards-site/tests-e2e/home.spec.js

@@ -1,7 +1,7 @@
 // @ts-check
 import path from 'path'
 import fs from 'fs'
-import { test, expect } from '@playwright/test'
+import { test, expect } from '@chromatic-com/playwright'
 
 const mockRepoDataPath = path.resolve(
   __dirname,
@@ -58,5 +58,7 @@ test('Home Page', async ({ page }) => {
   })
 
   // Prevent Visual Regressions
-  expect(await page.screenshot({ fullPage: true })).toMatchSnapshot('home.png')
+  if (process.env.CI) {
+    expect(await page.screenshot({ fullPage: true })).toMatchSnapshot('home.png')
+  }
 })

scorecards-site/tests-e2e/home.spec.js-snapshots/home-Mobile-Chrome-linux.png

@@ -1,7 +1,7 @@
 // @ts-check
 import path from 'path'
 import fs from 'fs'
-import { test, expect } from '@playwright/test'
+import { test, expect } from '@chromatic-com/playwright'
 
 const mockScorecardDataPath = path.resolve(
   __dirname,
@@ -14,9 +14,11 @@ test('viewer welcome', async ({ page }) => {
   await page.goto('http://localhost:3000/viewer')
 
   // Prevent Visual Regressions
-  expect(await page.screenshot({ fullPage: true })).toMatchSnapshot(
-    'viewer-welcome.png'
-  )
+  if (process.env.CI) {
+    expect(await page.screenshot({ fullPage: true })).toMatchSnapshot(
+      'viewer-welcome.png'
+    )
+  }
 
   // Find input field and type the repo URL.
   const input = page.locator('input[placeholder="github.com/ossf/scorecard"]')
@@ -57,7 +59,9 @@ test('viewer details', async ({ page }) => {
   await page.waitForTimeout(5 * 1000)
 
   // Prevent Visual Regressions
-  expect(await page.screenshot({ fullPage: true })).toMatchSnapshot(
-    'viewer-details.png'
-  )
+  if (process.env.CI) {
+    expect(await page.screenshot({ fullPage: true })).toMatchSnapshot(
+      'viewer-details.png'
+    )
+  }
 })