From c500c432b4be6f3630d7d7abf7a89d142d2cbd67 Mon Sep 17 00:00:00 2001 From: Theo Ephraim Date: Fri, 1 May 2026 15:53:11 -0700 Subject: [PATCH 1/3] perf: add quick mode for nextjs framework tests Run only v16+turbopack for most PRs, full suite (v14/v15/v16, both bundlers) only when the nextjs integration package or test files change. - Restructure CI matrix from string arrays to {name, testPath} objects - Add QUICK_TEST_PATHS config in detect-changed-integrations - Add NEXTJS_TURBO_ONLY env var to skip webpack bundler in quick mode --- .github/workflows/framework-tests.yaml | 10 +-- .../frameworks/nextjs/nextjs-shared.ts | 7 ++- scripts/detect-changed-integrations.ts | 62 ++++++++++++------- 3 files changed, 52 insertions(+), 27 deletions(-) diff --git a/.github/workflows/framework-tests.yaml b/.github/workflows/framework-tests.yaml index db58fe860..a9556cbb9 100644 --- a/.github/workflows/framework-tests.yaml +++ b/.github/workflows/framework-tests.yaml @@ -81,7 +81,7 @@ jobs: retention-days: 1 test: - name: Framework Tests (${{ matrix.integration }}) + name: Framework Tests (${{ matrix.integration.name }}) needs: [detect-integrations, build] if: needs.detect-integrations.outputs.any-changed == 'true' runs-on: ubuntu-latest @@ -122,8 +122,8 @@ jobs: name: built-repo path: packages/ - - name: Run ${{ matrix.integration }} framework tests - run: cd framework-tests && bun install && bun run test -- --reporter=verbose frameworks/${{ matrix.integration }}/ + - name: Run ${{ matrix.integration.name }} framework tests + run: cd framework-tests && bun install && bun run test -- --reporter=verbose frameworks/${{ matrix.integration.testPath }} timeout-minutes: 15 - # env: - # FRAMEWORK_TEST_VERBOSE: '1' + env: + NEXTJS_TURBO_ONLY: ${{ contains(matrix.integration.name, '(quick)') && '1' || '' }} diff --git a/framework-tests/frameworks/nextjs/nextjs-shared.ts b/framework-tests/frameworks/nextjs/nextjs-shared.ts index 81dc33f04..4703247b7 100644 --- a/framework-tests/frameworks/nextjs/nextjs-shared.ts +++ b/framework-tests/frameworks/nextjs/nextjs-shared.ts @@ -3,11 +3,16 @@ import { } from 'vitest'; import { FrameworkTestEnv } from '../../harness/index'; -const BUNDLERS = [ +const ALL_BUNDLERS = [ 'webpack', 'turbopack', ]; +// When running quick mode (just v16), skip webpack to cut build time in half +const BUNDLERS = process.env.NEXTJS_TURBO_ONLY + ? ALL_BUNDLERS.filter((b) => b === 'turbopack') + : ALL_BUNDLERS; + const EXPORT_CONFIG = { path: '_base/next.config.mjs' as const, replacements: { '// OUTPUT-MODE': "output: 'export'," }, diff --git a/scripts/detect-changed-integrations.ts b/scripts/detect-changed-integrations.ts index 74f235b80..3dd68c1c0 100644 --- a/scripts/detect-changed-integrations.ts +++ b/scripts/detect-changed-integrations.ts @@ -31,6 +31,15 @@ const INTEGRATION_PACKAGES: Record> = { astro: ['@varlock/astro-integration', '@varlock/vite-integration'], }; +// Quick test paths for integrations where full suite is expensive. +// When only core varlock or harness changes trigger the integration, run the quick +// path. When the integration's own package or test files change, run full suite. +const QUICK_TEST_PATHS: Record = { + nextjs: 'nextjs/nextjs-v16.test.ts', +}; + +type IntegrationEntry = { name: string; testPath: string }; + const ALL_INTEGRATIONS = Object.keys(INTEGRATION_PACKAGES); const forceAll = process.argv.includes('--all'); const isReleasePR = process.argv.includes('--release-pr'); @@ -44,20 +53,27 @@ function writeGithubOutputs(outputs: Record) { const REPO_ROOT = join(import.meta.dirname, '..'); -function writeResults(integrations: Array) { - const anyChanged = integrations.length > 0; +function toEntry(name: string, mode: 'full' | 'quick' = 'full'): IntegrationEntry { + if (mode === 'quick' && QUICK_TEST_PATHS[name]) { + return { name: `${name} (quick)`, testPath: QUICK_TEST_PATHS[name] }; + } + return { name, testPath: `${name}/` }; +} + +function writeResults(entries: Array) { + const anyChanged = entries.length > 0; writeGithubOutputs({ 'any-changed': String(anyChanged), - integrations: JSON.stringify(integrations), + integrations: JSON.stringify(entries), }); if (!process.env.GITHUB_OUTPUT) { - console.log('Integrations to test:', integrations.length > 0 ? integrations : '(none)'); + console.log('Integrations to test:', entries.length > 0 ? entries.map((e) => e.name) : '(none)'); } } -// --all flag: test everything +// --all flag: test everything (full suite) if (forceAll) { - writeResults(ALL_INTEGRATIONS); + writeResults(ALL_INTEGRATIONS.map((name) => toEntry(name, 'full'))); process.exit(0); } @@ -78,7 +94,7 @@ if (isReleasePR) { }); } catch (e) { console.error('Failed to diff against origin/main:', e); - writeResults(ALL_INTEGRATIONS); + writeResults(ALL_INTEGRATIONS.map((name) => toEntry(name, 'full'))); process.exit(0); } @@ -113,7 +129,7 @@ if (isReleasePR) { } catch (e) { console.error('Failed to diff against origin/main:', e); // If we can't diff, fall back to running all tests - writeResults(ALL_INTEGRATIONS); + writeResults(ALL_INTEGRATIONS.map((name) => toEntry(name, 'full'))); process.exit(0); } @@ -153,7 +169,7 @@ if (isReleasePR) { if (changedPackages.has('varlock')) { if (isReleasePR) { console.log('Core varlock package changed on release PR — running all integration tests'); - writeResults(ALL_INTEGRATIONS); + writeResults(ALL_INTEGRATIONS.map((name) => toEntry(name, 'full'))); process.exit(0); } else { console.log('Core varlock package changed — triggering core-only test suites'); @@ -176,7 +192,7 @@ try { } else if (filePath.startsWith('framework-tests/harness/') || filePath === 'framework-tests/vitest.config.ts') { // Shared test infrastructure changed — trigger all console.log(`Shared framework test file changed (${filePath}) — running all integration tests`); - writeResults(ALL_INTEGRATIONS); + writeResults(ALL_INTEGRATIONS.map((name) => toEntry(name, 'quick'))); process.exit(0); } } @@ -188,14 +204,18 @@ try { } // Match changed packages to integration test suites -const integrationsList = Object.entries(INTEGRATION_PACKAGES) - .filter(([name, packages]) => { - // triggered if the test files themselves changed - if (changedTestIntegrations.has(name)) return true; - // suites with no integration packages are triggered by core varlock changes - if (packages.length === 0) return changedPackages.has('varlock'); - return packages.some((pkg) => changedPackages.has(pkg)); - }) - .map(([name]) => name); - -writeResults(integrationsList); +const entries: Array = []; +for (const [name, packages] of Object.entries(INTEGRATION_PACKAGES)) { + // Test files changed or integration package changed → full suite + if (changedTestIntegrations.has(name) || packages.some((pkg) => changedPackages.has(pkg))) { + entries.push(toEntry(name, 'full')); + continue; + } + // Suites with no integration packages are triggered by core varlock changes (quick) + if (packages.length === 0 && changedPackages.has('varlock')) { + entries.push(toEntry(name, 'quick')); + continue; + } +} + +writeResults(entries); From 76883114e414958bc7f6f6df9e15ab80470b71d9 Mon Sep 17 00:00:00 2001 From: Theo Ephraim Date: Fri, 1 May 2026 16:05:12 -0700 Subject: [PATCH 2/3] perf: preserve build cache between scenarios and improve CI caching - Preserve .next/cache between scenario runs so turbopack/webpack compilation cache speeds up subsequent builds within the same fixture - Add NEXT_TELEMETRY_DISABLED=1 to avoid telemetry overhead - Cache pnpm store in CI for framework test project installs --- .github/workflows/framework-tests.yaml | 10 ++++++++++ framework-tests/harness/test-fixture.ts | 14 ++++++++++++-- 2 files changed, 22 insertions(+), 2 deletions(-) diff --git a/.github/workflows/framework-tests.yaml b/.github/workflows/framework-tests.yaml index a9556cbb9..be8495ea4 100644 --- a/.github/workflows/framework-tests.yaml +++ b/.github/workflows/framework-tests.yaml @@ -113,6 +113,15 @@ jobs: with: version: 10 + - name: Cache pnpm store + uses: actions/cache@v5 + with: + path: ~/.local/share/pnpm/store/v3 + key: pnpm-${{ runner.os }}-${{ matrix.integration.name }}-${{ hashFiles('framework-tests/**/package.json') }} + restore-keys: | + pnpm-${{ runner.os }}-${{ matrix.integration.name }}- + pnpm-${{ runner.os }}- + - name: Install dependencies run: bun install @@ -127,3 +136,4 @@ jobs: timeout-minutes: 15 env: NEXTJS_TURBO_ONLY: ${{ contains(matrix.integration.name, '(quick)') && '1' || '' }} + NEXT_TELEMETRY_DISABLED: '1' diff --git a/framework-tests/harness/test-fixture.ts b/framework-tests/harness/test-fixture.ts index 4886c37f2..b8f3403dd 100644 --- a/framework-tests/harness/test-fixture.ts +++ b/framework-tests/harness/test-fixture.ts @@ -1,5 +1,5 @@ import { - cpSync, writeFileSync, readFileSync, + cpSync, writeFileSync, readFileSync, readdirSync, rmSync, existsSync, mkdirSync, } from 'node:fs'; import { @@ -389,7 +389,17 @@ export class FrameworkTestEnv { * Clean build artifacts between scenarios. */ cleanBuildArtifacts(): void { - const artifactDirs = ['.next', 'out', 'dist', '.turbo', '.wrangler']; + // Clean .next but preserve the cache directory so turbopack/webpack + // compilation cache speeds up subsequent builds within the same fixture + const nextDir = join(this.dir, '.next'); + if (existsSync(nextDir)) { + for (const entry of readdirSync(nextDir)) { + if (entry === 'cache') continue; + rmSync(join(nextDir, entry), { recursive: true, force: true }); + } + } + + const artifactDirs = ['out', 'dist', '.turbo', '.wrangler']; for (const dir of artifactDirs) { const fullPath = join(this.dir, dir); if (existsSync(fullPath)) { From 57f7ad7ebc795b4f2f663419439737ca4fd35df5 Mon Sep 17 00:00:00 2001 From: Theo Ephraim Date: Fri, 1 May 2026 16:07:05 -0700 Subject: [PATCH 3/3] perf: disable typescript checking during next build in tests ESLint was already disabled; also skip TS type-checking since these are framework integration tests, not type-correctness tests. --- framework-tests/frameworks/nextjs/files/_base/next.config.mjs | 1 + 1 file changed, 1 insertion(+) diff --git a/framework-tests/frameworks/nextjs/files/_base/next.config.mjs b/framework-tests/frameworks/nextjs/files/_base/next.config.mjs index 8c2619005..e773d6b1f 100644 --- a/framework-tests/frameworks/nextjs/files/_base/next.config.mjs +++ b/framework-tests/frameworks/nextjs/files/_base/next.config.mjs @@ -5,6 +5,7 @@ const nextConfig = { // OUTPUT-MODE productionBrowserSourceMaps: true, eslint: { ignoreDuringBuilds: true }, + typescript: { ignoreBuildErrors: true }, }; export default varlockNextConfigPlugin()(nextConfig);