diff --git a/.github/workflows/benchmark-comment-external.yml b/.github/workflows/benchmark-comment-external.yml new file mode 100644 index 0000000000..5b5b78dc10 --- /dev/null +++ b/.github/workflows/benchmark-comment-external.yml @@ -0,0 +1,26 @@ +name: Benchmark Comment External + +on: + workflow_run: + workflows: ["Benchmark External"] + types: + - completed + +permissions: + actions: read + pull-requests: write + +jobs: + comment: + if: > + github.event.workflow_run.event == 'pull_request' && + github.event.workflow_run.conclusion == 'success' + uses: ./.github/workflows/benchmark-comment-run.yml + with: + run-id: ${{ github.event.workflow_run.id }} + header: benchmark-results-external + comment-file: benchmark-comment.external.md + permissions: + actions: read + pull-requests: write + secrets: inherit diff --git a/.github/workflows/benchmark-comment-run.yml b/.github/workflows/benchmark-comment-run.yml new file mode 100644 index 0000000000..0c0b8600a5 --- /dev/null +++ b/.github/workflows/benchmark-comment-run.yml @@ -0,0 +1,46 @@ +name: Benchmark Comment Run + +# Reusable workflow that posts one sticky benchmark comment from a Benchmark run. +on: + workflow_call: + inputs: + run-id: + description: "The Benchmark workflow run id to download the comment artifact from" + required: true + type: string + header: + description: "Sticky comment header (identifies the comment to update)" + required: true + type: string + comment-file: + description: "Comment markdown file name inside the artifact" + required: true + type: string + +permissions: + actions: read + pull-requests: write + +jobs: + comment: + name: Post Benchmark Comment + runs-on: ubuntu-latest + steps: + - name: Download benchmark comment artifact + uses: actions/download-artifact@v8 + with: + name: benchmark-comment + run-id: ${{ inputs.run-id }} + github-token: ${{ secrets.GITHUB_TOKEN }} + + - name: Get PR number + id: get-pr + run: echo "pr_number=$(cat benchmark-pr-number.txt)" >> "$GITHUB_OUTPUT" + + - name: Post PR comment + uses: marocchino/sticky-pull-request-comment@v3 + with: + header: ${{ inputs.header }} + number: ${{ steps.get-pr.outputs.pr_number }} + path: ${{ inputs.comment-file }} + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} diff --git a/.github/workflows/benchmark-comment.yml b/.github/workflows/benchmark-comment.yml index ae4bfa0d36..f00d5e032f 100644 --- a/.github/workflows/benchmark-comment.yml +++ b/.github/workflows/benchmark-comment.yml @@ -7,31 +7,20 @@ on: - completed permissions: + actions: read pull-requests: write jobs: comment: - name: Post Benchmark Comment - runs-on: ubuntu-latest if: > github.event.workflow_run.event == 'pull_request' && github.event.workflow_run.conclusion == 'success' - steps: - - name: Download benchmark comment artifact - uses: actions/download-artifact@v8 - with: - name: benchmark-comment - run-id: ${{ github.event.workflow_run.id }} - github-token: ${{ secrets.GITHUB_TOKEN }} - - - name: Get PR number - id: get-pr - run: echo "pr_number=$(cat benchmark-pr-number.txt)" >> "$GITHUB_OUTPUT" - - - name: Post PR comment - uses: marocchino/sticky-pull-request-comment@v3 - with: - header: benchmark-results - number: ${{ steps.get-pr.outputs.pr_number }} - path: benchmark-comment.md - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + uses: ./.github/workflows/benchmark-comment-run.yml + with: + run-id: ${{ github.event.workflow_run.id }} + header: benchmark-results + comment-file: benchmark-comment.md + permissions: + actions: read + pull-requests: write + secrets: inherit diff --git a/.github/workflows/benchmark-external.yml b/.github/workflows/benchmark-external.yml new file mode 100644 index 0000000000..412e015faa --- /dev/null +++ b/.github/workflows/benchmark-external.yml @@ -0,0 +1,41 @@ +name: Benchmark External + +on: + push: + branches: + - main + pull_request: + branches: + - main + - release/* + workflow_dispatch: + inputs: + runner: + description: "Runner label (for stable runs prefer self-hosted or larger dedicated runner)" + required: false + type: string + default: "ubuntu-latest" + +permissions: + contents: write + +concurrency: + group: benchmark-external-${{ github.ref }} + cancel-in-progress: true + +jobs: + run: + uses: ./.github/workflows/benchmark-run.yml + with: + specs-dir: packages/benchmark/external-spec + results-dir: external-results + title: "⚡ External Spec Benchmark Results" + comment-file: benchmark-comment.external.md + iterations: 3 + warmup: 1 + noise-cv: 0.1 + rerun-iterations: 2 + runner: ${{ github.event_name == 'workflow_dispatch' && inputs.runner || vars.BENCHMARK_RUNNER || 'ubuntu-latest' }} + permissions: + contents: write + secrets: inherit diff --git a/.github/workflows/benchmark-run.yml b/.github/workflows/benchmark-run.yml new file mode 100644 index 0000000000..5fe130502b --- /dev/null +++ b/.github/workflows/benchmark-run.yml @@ -0,0 +1,133 @@ +name: Benchmark Run + +# Reusable workflow that benchmarks a single spec group. Instantiate it once per +# spec group (see benchmark.yml and benchmark-external.yml). +on: + workflow_call: + inputs: + specs-dir: + description: "Directory of benchmark specs to run" + required: true + type: string + results-dir: + description: "Directory on the data branch to store results/history" + required: true + type: string + title: + description: "Heading for the PR comment" + required: true + type: string + comment-file: + description: "Comment markdown file name (must match the comment instance)" + required: true + type: string + iterations: + required: false + type: number + default: 25 + warmup: + required: false + type: number + default: 3 + noise-cv: + required: false + type: number + default: 0.08 + rerun-iterations: + required: false + type: number + default: 10 + runner: + required: false + type: string + default: "ubuntu-latest" + backfill_from: + description: "Backfill from: commit SHA or number of recent commits (main only)" + required: false + type: string + default: "" + branch: + description: "Data branch to store results" + required: false + type: string + default: "benchmark-data" + +permissions: + contents: write + +jobs: + benchmark: + name: Run Benchmarks + runs-on: ${{ inputs.runner }} + env: + TYPESPEC_VS_CI_BUILD: true + TYPESPEC_SKIP_WEBSITE_BUILD: true + steps: + - uses: actions/checkout@v7 + with: + submodules: recursive + fetch-depth: 0 + + - uses: ./.github/actions/setup + with: + node-version: 24.15.0 + + - name: Install dependencies + run: pnpm install + + - name: Build + run: pnpm -r --filter "@azure-tools/typespec-benchmark..." build + + - name: Run backfill + if: ${{ inputs.backfill_from != '' }} + run: | + node --max-old-space-size=6144 packages/benchmark/dist/src/cli.js backfill \ + --from ${{ inputs.backfill_from }} \ + --specs-dir ${{ inputs.specs-dir }} \ + --iterations ${{ inputs.iterations }} \ + --warmup ${{ inputs.warmup }} \ + --branch ${{ inputs.branch }} \ + --push + + - name: Run benchmarks + if: ${{ inputs.backfill_from == '' }} + run: | + node --max-old-space-size=6144 packages/benchmark/dist/src/cli.js run \ + --specs-dir ${{ inputs.specs-dir }} \ + --iterations ${{ inputs.iterations }} \ + --warmup ${{ inputs.warmup }} \ + --noise-cv-threshold ${{ inputs.noise-cv }} \ + --max-reruns 1 \ + --rerun-iterations ${{ inputs.rerun-iterations }} \ + --commit ${{ github.sha }} \ + --output /tmp/benchmark-results.json + + # On push to main: store results to the data branch. + - name: Store benchmark results + if: github.event_name == 'push' && github.ref == 'refs/heads/main' + run: | + node packages/benchmark/dist/src/cli.js store-results \ + --results /tmp/benchmark-results.json \ + --commit ${{ github.sha }} \ + --results-dir ${{ inputs.results-dir }} + + # On PR: fetch baseline, compare, and upload as artifact for the comment workflow. + - name: Generate PR comment + if: github.event_name == 'pull_request' + run: | + node packages/benchmark/dist/src/cli.js upload-pr-comment \ + --results /tmp/benchmark-results.json \ + --pr-number ${{ github.event.number }} \ + --results-dir ${{ inputs.results-dir }} \ + --baseline-window 20 \ + --title "${{ inputs.title }}" \ + --comment-file ${{ inputs.comment-file }} \ + --output-dir /tmp/benchmark-artifacts + + - name: Upload benchmark comment + if: github.event_name == 'pull_request' + uses: actions/upload-artifact@v7 + with: + name: benchmark-comment + path: /tmp/benchmark-artifacts/ + retention-days: 1 diff --git a/.github/workflows/benchmark.yml b/.github/workflows/benchmark.yml index 06ac622637..5ad0058667 100644 --- a/.github/workflows/benchmark.yml +++ b/.github/workflows/benchmark.yml @@ -29,78 +29,24 @@ permissions: contents: write concurrency: - group: benchmark-${{ github.ref }} + group: benchmark-main-${{ github.ref }} cancel-in-progress: true jobs: - benchmark: - name: Run Benchmarks - runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.runner || vars.BENCHMARK_RUNNER || 'ubuntu-latest' }} - env: - TYPESPEC_VS_CI_BUILD: true - TYPESPEC_SKIP_WEBSITE_BUILD: true - steps: - - uses: actions/checkout@v7 - with: - submodules: recursive - fetch-depth: 0 - - - uses: ./.github/actions/setup - with: - node-version: 24.15.0 - - - name: Install dependencies - run: pnpm install - - - name: Build - run: pnpm -r --filter "@azure-tools/typespec-benchmark..." build - - - name: Run backfill - if: ${{ github.event_name == 'workflow_dispatch' && inputs.backfill_from }} - run: | - node --max-old-space-size=6144 packages/benchmark/dist/src/cli.js backfill \ - --from ${{ inputs.backfill_from }} \ - --specs-dir packages/benchmark/specs \ - --iterations 25 \ - --warmup 3 \ - --branch ${{ inputs.branch }} \ - --push - - - name: Run benchmarks - if: ${{ github.event_name != 'workflow_dispatch' || !inputs.backfill_from }} - run: | - node --max-old-space-size=6144 packages/benchmark/dist/src/cli.js run \ - --specs-dir packages/benchmark/specs \ - --iterations 25 \ - --warmup 3 \ - --noise-cv-threshold 0.08 \ - --max-reruns 1 \ - --rerun-iterations 10 \ - --commit ${{ github.sha }} \ - --output /tmp/benchmark-results.json - - # On push to main: store results to benchmark-data branch - - name: Store benchmark results - if: github.event_name == 'push' && github.ref == 'refs/heads/main' - run: | - node packages/benchmark/dist/src/cli.js store-results \ - --results /tmp/benchmark-results.json \ - --commit ${{ github.sha }} - - # On PR: fetch baseline, compare, and upload as artifact for the comment workflow - - name: Generate PR comment - if: github.event_name == 'pull_request' - run: | - node packages/benchmark/dist/src/cli.js upload-pr-comment \ - --results /tmp/benchmark-results.json \ - --pr-number ${{ github.event.number }} \ - --baseline-window 20 \ - --output-dir /tmp/benchmark-artifacts - - - name: Upload benchmark comment - if: github.event_name == 'pull_request' - uses: actions/upload-artifact@v7 - with: - name: benchmark-comment - path: /tmp/benchmark-artifacts/ - retention-days: 1 + run: + uses: ./.github/workflows/benchmark-run.yml + with: + specs-dir: packages/benchmark/specs + results-dir: results + title: "⚡ Benchmark Results" + comment-file: benchmark-comment.md + iterations: 25 + warmup: 3 + noise-cv: 0.08 + rerun-iterations: 10 + runner: ${{ github.event_name == 'workflow_dispatch' && inputs.runner || vars.BENCHMARK_RUNNER || 'ubuntu-latest' }} + backfill_from: ${{ github.event_name == 'workflow_dispatch' && inputs.backfill_from || '' }} + branch: ${{ github.event_name == 'workflow_dispatch' && inputs.branch || 'benchmark-data' }} + permissions: + contents: write + secrets: inherit diff --git a/packages/benchmark/.gitignore b/packages/benchmark/.gitignore index d4086fa950..9e3518e402 100644 --- a/packages/benchmark/.gitignore +++ b/packages/benchmark/.gitignore @@ -1,3 +1,4 @@ dist/ tsp-output/ node_modules/ +.external/ diff --git a/packages/benchmark/README.md b/packages/benchmark/README.md index f2a031a0e7..6a087eae78 100644 --- a/packages/benchmark/README.md +++ b/packages/benchmark/README.md @@ -132,9 +132,77 @@ The TypeSpec compiler provides built-in `Stats` covering: - `linter` — per-rule timing (e.g., `@azure-tools/typespec-azure-core/auth-required`) - `emit` — per-emitter timing with per-step breakdown -## Adding a new benchmark spec +## Benchmark specs + +A benchmark spec is anything the runner can compile and measure. There are two kinds, discovered from whatever directory `--specs-dir` points at: + +- **Local spec** — a subdirectory containing a `main.tsp` (and usually a `tspconfig.yaml`). The built-in specs under `specs/` are local specs. +- **External spec** — a directory with a `spec.json` (and a `tspconfig.yaml`) describing a spec that lives in another repository (see below). + +The runner treats both kinds uniformly and writes a single results file. What to run and how to display it is decided by the caller (CI job / dashboard), not by the runner — e.g. CI runs `--specs-dir specs` and `--specs-dir external-spec` separately and stores them in separate results directories. + +### Adding a local spec 1. Create a new directory under `specs/` (e.g., `specs/my-new-spec/`) 2. Add a `main.tsp` file with your TypeSpec code 3. Add a `tspconfig.yaml` with emitter and linter configuration 4. The runner auto-discovers spec directories — no registration needed + +### Adding an external spec (e.g. from azure-rest-api-specs) + +External specs live in another repository (such as [azure-rest-api-specs](https://github.com/Azure/azure-rest-api-specs)) and are not copied into this repo. They are sparse-checked-out into `packages/benchmark/.external/` (git-ignored) and compiled against **this workspace's** packages, so the measurement reflects your local source changes (e.g. to TCGC or a client emitter) rather than the published npm versions. + +Each external spec is **one directory** under `external-spec/` (the directory name is the benchmark name), containing two files: + +- `spec.json` — where to get the spec source. +- `tspconfig.yaml` — the compiler config (emitters + linter) to measure; it is copied into the checkout, replacing the spec's own tspconfig. + +`external-spec/web/spec.json`: + +```json +{ + "repository": "https://github.com/Azure/azure-rest-api-specs.git", + "ref": "main", + "path": "specification/web/resource-manager/Microsoft.Web/AppService" +} +``` + +- `repository` / `ref` — the repo and git ref to sparse-checkout; `ref` tracks latest (e.g. `main`), no pinning. +- `path` — the entry directory (containing `main.tsp`) relative to the repo root. The spec's own `main.tsp` is the entrypoint (it imports `client.tsp`). +- `checkoutPath` (optional) — directory to sparse-checkout, when the spec imports sibling folders (e.g. `../common`). Defaults to `path`. +- `name` (optional) — overrides the benchmark name (defaults to the directory name). + +`external-spec/web/tspconfig.yaml` selects what to measure — e.g. TCGC plus the ARM ruleset: + +```yaml +emit: + - "@azure-tools/typespec-client-generator-core" +linter: + extends: + - "@azure-tools/typespec-azure-rulesets/resource-manager" +``` + +Run the external specs (fewer iterations, since they are heavy): + +```bash +node packages/benchmark/dist/src/cli.js run \ + --specs-dir packages/benchmark/external-spec \ + --warmup 0 --iterations 1 \ + --output external.json +``` + +To evaluate the impact of a change, rebuild the workspace at each revision and compare: + +```bash +# revision A: build, then run +node packages/benchmark/dist/src/cli.js run --specs-dir packages/benchmark/external-spec --output before.json +# revision B: rebuild, then run +node packages/benchmark/dist/src/cli.js run --specs-dir packages/benchmark/external-spec --output after.json +node packages/benchmark/dist/src/cli.js compare --baseline before.json --current after.json --detailed +``` + +> **Notes** +> +> - The spec's own `tspconfig.yaml` is replaced by the one in the external-spec directory, so the emitters and linter measured are exactly what you configure there. +> - A spec that imports a package not declared in `packages/benchmark/package.json` will fail to resolve — add the package to the benchmark's dependencies or pick a compatible spec. +> - Per-emitter time is reported under `emit/` (the top-level `total` metric excludes emit time). diff --git a/packages/benchmark/external-spec/compute/spec.json b/packages/benchmark/external-spec/compute/spec.json new file mode 100644 index 0000000000..2d9e469d8e --- /dev/null +++ b/packages/benchmark/external-spec/compute/spec.json @@ -0,0 +1,6 @@ +{ + "repository": "https://github.com/Azure/azure-rest-api-specs.git", + "ref": "main", + "path": "specification/compute/resource-manager/Microsoft.Compute/Compute/Compute", + "checkoutPath": "specification/compute/resource-manager/Microsoft.Compute/Compute" +} diff --git a/packages/benchmark/external-spec/compute/tspconfig.yaml b/packages/benchmark/external-spec/compute/tspconfig.yaml new file mode 100644 index 0000000000..f562b813f2 --- /dev/null +++ b/packages/benchmark/external-spec/compute/tspconfig.yaml @@ -0,0 +1,5 @@ +emit: + - "@azure-tools/typespec-client-generator-core" +linter: + extends: + - "@azure-tools/typespec-azure-rulesets/resource-manager" diff --git a/packages/benchmark/external-spec/network/spec.json b/packages/benchmark/external-spec/network/spec.json new file mode 100644 index 0000000000..ac0c413e96 --- /dev/null +++ b/packages/benchmark/external-spec/network/spec.json @@ -0,0 +1,6 @@ +{ + "repository": "https://github.com/Azure/azure-rest-api-specs.git", + "ref": "main", + "path": "specification/network/resource-manager/Microsoft.Network/Network/Network", + "checkoutPath": "specification/network/resource-manager/Microsoft.Network/Network" +} diff --git a/packages/benchmark/external-spec/network/tspconfig.yaml b/packages/benchmark/external-spec/network/tspconfig.yaml new file mode 100644 index 0000000000..f562b813f2 --- /dev/null +++ b/packages/benchmark/external-spec/network/tspconfig.yaml @@ -0,0 +1,5 @@ +emit: + - "@azure-tools/typespec-client-generator-core" +linter: + extends: + - "@azure-tools/typespec-azure-rulesets/resource-manager" diff --git a/packages/benchmark/external-spec/web/spec.json b/packages/benchmark/external-spec/web/spec.json new file mode 100644 index 0000000000..5294ed78ba --- /dev/null +++ b/packages/benchmark/external-spec/web/spec.json @@ -0,0 +1,5 @@ +{ + "repository": "https://github.com/Azure/azure-rest-api-specs.git", + "ref": "main", + "path": "specification/web/resource-manager/Microsoft.Web/AppService" +} diff --git a/packages/benchmark/external-spec/web/tspconfig.yaml b/packages/benchmark/external-spec/web/tspconfig.yaml new file mode 100644 index 0000000000..f562b813f2 --- /dev/null +++ b/packages/benchmark/external-spec/web/tspconfig.yaml @@ -0,0 +1,5 @@ +emit: + - "@azure-tools/typespec-client-generator-core" +linter: + extends: + - "@azure-tools/typespec-azure-rulesets/resource-manager" diff --git a/packages/benchmark/src/cli.ts b/packages/benchmark/src/cli.ts index 82380f99cf..9aa7a88469 100644 --- a/packages/benchmark/src/cli.ts +++ b/packages/benchmark/src/cli.ts @@ -35,7 +35,8 @@ Commands: backfill Backfill benchmark data for historical commits Run options: - --specs-dir Directory containing benchmark specs (default: built-in specs) + --specs-dir Directory of benchmark specs: subdirectories with a main.tsp (local specs) + and/or with a spec.json (external specs) (default: built-in specs) --iterations Number of measured iterations (default: 5) --warmup Number of warmup iterations (default: 1) --noise-cv-threshold @@ -63,14 +64,18 @@ Store-results options: --results Path to the benchmark results JSON file --commit Git commit SHA --branch Branch name for storing results (default: benchmark-data) + --results-dir Directory on the data branch to store results/history (default: results) Upload-pr-comment options: --results Path to the current benchmark results JSON file --pr-number Pull request number --output-dir Output directory for artifacts --branch Branch name for fetching baseline (default: benchmark-data) + --results-dir Directory on the data branch holding the baseline (default: results) --baseline-window Number of recent main results to build rolling baseline (default: 20) --threshold Percent threshold for notable changes (default: 5) + --title Heading for the comment (default: "⚡ Benchmark Results") + --comment-file Comment markdown file name (default: benchmark-comment.md) Backfill options: --from Start point: a commit SHA or number of recent commits (default: 100) @@ -140,6 +145,9 @@ async function runCommand(args: Record): Promise { ? parseInt(args["rerun-iterations"], 10) : undefined; + // A spec source is either a local spec directory (with main.tsp) or an + // external spec directory (with spec.json). `--specs-dir` selects which set + // to run; both kinds run uniformly and produce a single result file. const result = await runBenchmarks({ specsDir, iterations, @@ -201,6 +209,7 @@ function storeResultsCommand(args: Record): void { resultsFile, commit, branch: args["branch"], + resultsDir: args["results-dir"], }); } @@ -219,8 +228,11 @@ function uploadPrCommentCommand(args: Record): void { prNumber, outputDir, branch: args["branch"], + resultsDir: args["results-dir"], threshold: args["threshold"] ? parseFloat(args["threshold"]) : undefined, baselineWindow: args["baseline-window"] ? parseInt(args["baseline-window"], 10) : undefined, + title: args["title"], + commentFile: args["comment-file"], }); } diff --git a/packages/benchmark/src/external-specs.ts b/packages/benchmark/src/external-specs.ts new file mode 100644 index 0000000000..fa949c33d7 --- /dev/null +++ b/packages/benchmark/src/external-specs.ts @@ -0,0 +1,163 @@ +/* eslint-disable no-console */ +import { execFileSync } from "child_process"; +import { copyFileSync, existsSync, mkdirSync, readFileSync } from "fs"; +import { basename, join, resolve } from "path"; +import { fileURLToPath } from "url"; + +const __dirname = fileURLToPath(new URL(".", import.meta.url)); +/** Root where external spec repositories are sparse-checked-out. */ +const EXTERNAL_ROOT = resolve(__dirname, "..", "..", ".external"); + +/** File name of the source config inside an external spec directory. */ +export const EXTERNAL_SPEC_CONFIG = "spec.json"; +/** File name of the tspconfig copied into the checkout. */ +const TSPCONFIG = "tspconfig.yaml"; + +/** + * Source config for an external spec, stored as `spec.json` inside the spec's + * directory (e.g. `external-spec/web/spec.json`). The sibling `tspconfig.yaml` + * in the same directory controls the emitters and linter to measure. External + * specs are sparse-checked-out from another repository and compiled against + * this workspace's packages. + */ +export interface ExternalSpecConfig { + /** Git URL of the repository to sparse-checkout (e.g. azure-rest-api-specs). */ + repository: string; + /** Git branch or tag to check out. Defaults to "main". Tracks latest (no pinning). */ + ref?: string; + /** Entry directory (containing main.tsp), relative to the repository root. */ + path: string; + /** + * Directory to sparse-checkout, relative to the repository root. Defaults to + * `path`. Set to a parent directory when the spec imports sibling folders + * (e.g. `../common`). + */ + checkoutPath?: string; + /** Benchmark name (defaults to the spec directory name). */ + name?: string; +} + +/** An external spec config with its resolved name and source directory. */ +export interface NamedExternalSpecConfig extends ExternalSpecConfig { + name: string; + /** The local directory holding `spec.json` and `tspconfig.yaml`. */ + sourceDir: string; +} + +/** A resolved external spec ready to be benchmarked. */ +export interface ResolvedExternalSpec { + name: string; + dir: string; +} + +function git(cwd: string, args: string[]): void { + execFileSync("git", args, { cwd, stdio: "inherit" }); +} + +/** Derive a stable, filesystem-safe directory name for a repository URL. */ +function repoDirName(repository: string): string { + return basename(repository).replace(/\.git$/, "") || "external-repo"; +} + +/** + * Clone (or reuse) a repository using a sparse checkout limited to the given + * paths. The checkout lives under `packages/benchmark/.external/` so that + * TypeSpec imports resolve to this workspace's packages, letting the benchmark + * measure the impact of local source changes on external specs. + */ +function sparseCheckout(repository: string, ref: string, paths: string[]): string { + const dest = join(EXTERNAL_ROOT, repoDirName(repository)); + + mkdirSync(EXTERNAL_ROOT, { recursive: true }); + + if (!existsSync(join(dest, ".git"))) { + console.log(`Cloning ${repository} (sparse, ref ${ref}) into ${dest}...`); + git(EXTERNAL_ROOT, [ + "clone", + "--filter=blob:none", + "--no-checkout", + "--depth", + "1", + "--branch", + ref, + repository, + dest, + ]); + git(dest, ["sparse-checkout", "init", "--cone"]); + git(dest, ["sparse-checkout", "set", ...paths]); + git(dest, ["checkout", ref]); + } else { + console.log(`Reusing existing checkout at ${dest} (fetching ref ${ref})...`); + git(dest, ["fetch", "--depth", "1", "origin", ref]); + git(dest, ["sparse-checkout", "set", ...paths]); + // Check out the freshly fetched commit (not a stale local branch) so the + // reuse path always lands on the latest of `ref`. --force discards any + // tspconfig.yaml we copied into the checkout on a previous run. + git(dest, ["checkout", "--force", "FETCH_HEAD"]); + } + + return dest; +} + +/** Load and validate an external spec's `spec.json` from its directory. */ +export function loadExternalSpecConfig(specDir: string): NamedExternalSpecConfig { + const configFile = join(specDir, EXTERNAL_SPEC_CONFIG); + const config = JSON.parse(readFileSync(configFile, "utf-8")) as ExternalSpecConfig; + if (!config.repository) { + throw new Error(`External spec config ${configFile} is missing "repository"`); + } + if (!config.path) { + throw new Error(`External spec config ${configFile} is missing "path"`); + } + const name = config.name ?? basename(specDir); + return { ...config, name, sourceDir: specDir }; +} + +/** + * Resolve external specs: sparse-checkout their sources (grouped per + * repository+ref so a shared repo is cloned once), copy each spec's + * `tspconfig.yaml` into the checkout so the compiler uses our emitter/linter + * config, and return each as a spec ready to benchmark. Fails if any spec's + * entry directory has no main.tsp. + */ +export function resolveExternalSpecs(configs: NamedExternalSpecConfig[]): ResolvedExternalSpec[] { + // Group by repository + ref so a shared repo is checked out once. + const groups = new Map(); + for (const config of configs) { + const ref = config.ref ?? "main"; + const key = `${config.repository}\n${ref}`; + const group = groups.get(key); + if (group) { + group.push(config); + } else { + groups.set(key, [config]); + } + } + + const resolved: ResolvedExternalSpec[] = []; + for (const [key, groupConfigs] of groups) { + const [repository, ref] = key.split("\n"); + const paths = groupConfigs.map((c) => c.checkoutPath ?? c.path); + const checkoutDir = sparseCheckout(repository, ref, paths); + + for (const config of groupConfigs) { + const dir = join(checkoutDir, config.path); + if (!existsSync(join(dir, "main.tsp"))) { + throw new Error( + `External spec "${config.name}" has no main.tsp at ${dir}. ` + + `Check "path"/"checkoutPath" and that the sparse checkout succeeded.`, + ); + } + // Copy our tspconfig.yaml into the checkout entry dir (overwriting the + // repo's) so the compiler uses our emitters/linter. Done after checkout + // so it isn't discarded by the checkout above. + const tspconfig = join(config.sourceDir, TSPCONFIG); + if (existsSync(tspconfig)) { + copyFileSync(tspconfig, join(dir, TSPCONFIG)); + } + resolved.push({ name: config.name, dir }); + } + } + + return resolved; +} diff --git a/packages/benchmark/src/format-comment.ts b/packages/benchmark/src/format-comment.ts index 311ca57542..6833009917 100644 --- a/packages/benchmark/src/format-comment.ts +++ b/packages/benchmark/src/format-comment.ts @@ -156,6 +156,8 @@ const LEGEND = export interface FormatOptions { /** Change threshold for highlighting (default: 5%). */ threshold?: number; + /** Heading title (default: "⚡ Benchmark Results"). */ + title?: string; } /** Format comparison results as a GitHub PR comment markdown. */ @@ -168,7 +170,7 @@ export function formatPrComment( const threshold = options.threshold ?? DEFAULT_THRESHOLD; const lines: string[] = []; - lines.push("## ⚡ Benchmark Results\n"); + lines.push(`## ${options.title ?? "⚡ Benchmark Results"}\n`); // Average metrics across all specs const averaged = averageComparisonMetrics(comparisons); diff --git a/packages/benchmark/src/run.ts b/packages/benchmark/src/run.ts index 039a8982de..e68aec1332 100644 --- a/packages/benchmark/src/run.ts +++ b/packages/benchmark/src/run.ts @@ -1,10 +1,16 @@ /* eslint-disable no-console */ import { execSync, spawn } from "child_process"; +import { existsSync } from "fs"; import { readdir } from "fs/promises"; import os from "os"; import { join, resolve } from "path"; import { fileURLToPath } from "url"; import { aggregateDurations } from "./aggregate.js"; +import { + EXTERNAL_SPEC_CONFIG, + loadExternalSpecConfig, + resolveExternalSpecs, +} from "./external-specs.js"; import { summarize } from "./statistics.js"; import type { BenchmarkResult, @@ -37,17 +43,41 @@ export interface RunOptions { rerunIterations?: number; } -/** Discover benchmark spec directories under the given path. */ -async function discoverSpecs(specsDir: string, filter?: string[]): Promise { - const entries = await readdir(specsDir, { withFileTypes: true }); - const dirs = entries - .filter((e) => e.isDirectory()) - .map((e) => e.name) - .sort((a, b) => a.localeCompare(b)); +/** A benchmark spec source: a name and the directory containing its main.tsp. */ +interface SpecSource { + name: string; + dir: string; +} + +/** Discover benchmark spec sources under the given directory. + * + * Each subdirectory is a spec: one containing a `main.tsp` is a local spec; one + * containing a `spec.json` is an external spec (sparse-checked-out from another + * repository). Both are treated uniformly as spec sources. */ +async function discoverSpecSources(specsDir: string, filter?: string[]): Promise { + const dirs = (await readdir(specsDir, { withFileTypes: true })).filter((e) => e.isDirectory()); + + const localSources: SpecSource[] = dirs + .filter((e) => existsSync(join(specsDir, e.name, "main.tsp"))) + .map((e) => ({ name: e.name, dir: join(specsDir, e.name) })); + + const externalConfigs = dirs + .filter((e) => existsSync(join(specsDir, e.name, EXTERNAL_SPEC_CONFIG))) + .map((e) => loadExternalSpecConfig(join(specsDir, e.name))); + + const externalSources: SpecSource[] = + externalConfigs.length > 0 + ? resolveExternalSpecs(externalConfigs).map((s) => ({ name: s.name, dir: s.dir })) + : []; + + const sources = [...localSources, ...externalSources].sort((a, b) => + a.name.localeCompare(b.name), + ); + if (filter && filter.length > 0) { - return dirs.filter((d) => filter.includes(d)); + return sources.filter((s) => filter.includes(s.name)); } - return dirs; + return sources; } const compileOncePath = fileURLToPath(new URL("./compile-once.js", import.meta.url)); @@ -203,13 +233,13 @@ export async function runBenchmarks(options: RunOptions): Promise = {}; @@ -217,8 +247,9 @@ export async function runBenchmarks(options: RunOptions): Promise/` (default "results"); a + // distinct directory keeps a spec group separate from the main baseline. + const resultsDir = join(worktreeDir, resultsDirName); mkdirSync(resultsDir, { recursive: true }); copyFileSync(resultsFile, join(resultsDir, `${commit}.json`)); copyFileSync(resultsFile, join(resultsDir, "latest.json")); @@ -58,12 +65,36 @@ export function storeResults(options: StoreResultsOptions): void { writeFileSync(join(resultsDir, "history.json"), JSON.stringify(history, null, 2)); // Commit and push - git("add results/", worktreeDir); - git(`commit -m "Benchmark results for ${commit}"`, worktreeDir); - git(`push origin HEAD:${branch}`, worktreeDir); + git(`add ${resultsDirName}/`, worktreeDir); + git(`commit -m "Benchmark results (${resultsDirName}) for ${commit}"`, worktreeDir); + pushWithRetry(branch, worktreeDir); - console.log(`Benchmark results stored on ${branch} branch for commit ${commit}`); + console.log( + `Benchmark results stored on ${branch} branch for commit ${commit} (dir: ${resultsDirName})`, + ); } finally { gitSilent(`worktree remove ${worktreeDir} --force`); } } + +/** + * Push to the data branch, retrying on non-fast-forward rejection. Multiple + * benchmark workflows (e.g. main and external) may push to the same branch + * concurrently; since each writes a distinct results directory, rebasing our + * commit onto the latest remote branch and retrying resolves the race cleanly. + */ +function pushWithRetry(branch: string, worktreeDir: string, attempts = 5): void { + for (let attempt = 1; attempt <= attempts; attempt++) { + if (gitSilent(`push origin HEAD:${branch}`, worktreeDir)) { + return; + } + if (attempt === attempts) { + throw new Error(`Failed to push to ${branch} after ${attempts} attempts`); + } + console.log( + `Push to ${branch} rejected (attempt ${attempt}); rebasing on latest and retrying...`, + ); + git(`fetch origin ${branch}`, worktreeDir); + git(`rebase origin/${branch}`, worktreeDir); + } +} diff --git a/packages/benchmark/src/upload-pr-comment.ts b/packages/benchmark/src/upload-pr-comment.ts index 53e447bcc2..298ade6a4f 100644 --- a/packages/benchmark/src/upload-pr-comment.ts +++ b/packages/benchmark/src/upload-pr-comment.ts @@ -22,10 +22,16 @@ export interface UploadPrCommentOptions { outputDir: string; /** Branch name for fetching baseline results. */ branch?: string; + /** Directory on the data branch holding the baseline (default: "results"). */ + resultsDir?: string; /** Percent threshold for notable changes. */ threshold?: number; /** Number of latest entries to use for rolling baseline. */ baselineWindow?: number; + /** Heading title for the comment (default: "⚡ Benchmark Results"). */ + title?: string; + /** Comment markdown file name written to outputDir (default: "benchmark-comment.md"). */ + commentFile?: string; } interface BaselineResult { @@ -151,6 +157,7 @@ function buildRollingBaseline( function fetchBaseline( branch: string, + resultsDir: string, current: BenchmarkResult, baselineWindow: number, ): BaselineResult | undefined { @@ -170,7 +177,7 @@ function fetchBaseline( execSync(`git fetch origin ${branch}`, { stdio: "ignore" }); try { - const historyContent = execSync(`git show origin/${branch}:results/history.json`, { + const historyContent = execSync(`git show origin/${branch}:${resultsDir}/history.json`, { encoding: "utf-8", maxBuffer: 50_000_000, }); @@ -183,7 +190,7 @@ function fetchBaseline( // ignore and fallback to latest.json } - const latestContent = execSync(`git show origin/${branch}:results/latest.json`, { + const latestContent = execSync(`git show origin/${branch}:${resultsDir}/latest.json`, { encoding: "utf-8", maxBuffer: 50_000_000, }); @@ -196,9 +203,9 @@ function fetchBaseline( } } -function generateNoBaselineComment(): string { +function generateNoBaselineComment(title: string): string { return [ - "## ⚡ Benchmark Results", + `## ${title}`, "", "No baseline found on the `benchmark-data` branch. Benchmark results will be stored after merging to `main`.", "", @@ -209,15 +216,18 @@ function generateNoBaselineComment(): string { export function uploadPrComment(options: UploadPrCommentOptions): void { const { resultsFile, prNumber, outputDir } = options; const branch = options.branch ?? DEFAULT_BRANCH; + const resultsDir = options.resultsDir ?? "results"; const threshold = options.threshold; const baselineWindow = options.baselineWindow ?? 20; + const title = options.title ?? "⚡ Benchmark Results"; + const commentFile = options.commentFile ?? "benchmark-comment.md"; if (!existsSync(resultsFile)) { throw new Error(`Results file not found: ${resultsFile}`); } const current = JSON.parse(readFileSync(resolve(resultsFile), "utf-8")) as BenchmarkResult; - const baselineResult = fetchBaseline(branch, current, baselineWindow); + const baselineResult = fetchBaseline(branch, resultsDir, current, baselineWindow); mkdirSync(outputDir, { recursive: true }); @@ -233,6 +243,7 @@ export function uploadPrComment(options: UploadPrCommentOptions): void { current.commit, { threshold, + title, }, ); githubSummary = formatComparisonSummary( @@ -250,10 +261,10 @@ export function uploadPrComment(options: UploadPrCommentOptions): void { } } else { console.log("No baseline found — generating placeholder comment."); - commentMarkdown = generateNoBaselineComment(); + commentMarkdown = generateNoBaselineComment(title); } - writeFileSync(join(outputDir, "benchmark-comment.md"), commentMarkdown); + writeFileSync(join(outputDir, commentFile), commentMarkdown); writeFileSync(join(outputDir, "benchmark-pr-number.txt"), prNumber); // Write GitHub Actions job summary if available diff --git a/website/src/components/benchmarks/benchmark-dashboard.tsx b/website/src/components/benchmarks/benchmark-dashboard.tsx index f0e5be5051..221e2947ed 100644 --- a/website/src/components/benchmarks/benchmark-dashboard.tsx +++ b/website/src/components/benchmarks/benchmark-dashboard.tsx @@ -66,9 +66,12 @@ function getUrlParams(): { repo: string; branch: string } { }; } -function getHistoryUrl(): string { +type Dataset = "main" | "external"; + +function getHistoryUrl(dataset: Dataset): string { const { repo, branch } = getUrlParams(); - return `https://raw.githubusercontent.com/${repo}/${branch}/results/history.json`; + const dir = dataset === "external" ? "external-results" : "results"; + return `https://raw.githubusercontent.com/${repo}/${branch}/${dir}/history.json`; } function getCommitUrl(): string { @@ -118,16 +121,19 @@ function seriesColors(count: number): string[] { } /** Parse URL search params to get initial state */ -function getInitialParams(): { tab: Tab; spec: string; range: TimeRange } { - if (typeof window === "undefined") return { tab: "stages", spec: "all", range: "all" }; +function getInitialParams(): { tab: Tab; spec: string; range: TimeRange; dataset: Dataset } { + if (typeof window === "undefined") + return { tab: "stages", spec: "all", range: "all", dataset: "main" }; const params = new URLSearchParams(window.location.search); const tab = (params.get("tab") as Tab) || "stages"; const spec = params.get("spec") || "all"; const range = (params.get("range") as TimeRange) || "all"; + const dataset = (params.get("dataset") as Dataset) || "main"; return { tab: TABS.some((t) => t.key === tab) ? tab : "stages", spec, range: TIME_RANGES.some((r) => r.key === range) ? range : "all", + dataset: dataset === "external" ? "external" : "main", }; } @@ -136,7 +142,7 @@ function updateUrlParams(params: Record) { if (typeof window === "undefined") return; const url = new URL(window.location.href); for (const [key, value] of Object.entries(params)) { - if (value && value !== "all" && value !== "stages") { + if (value && value !== "all" && value !== "stages" && value !== "main") { url.searchParams.set(key, value); } else { url.searchParams.delete(key); @@ -182,32 +188,43 @@ const highlightPlugin: Plugin<"line"> = { // ─── Sub-components ─────────────────────────────────────────────────────────── -interface ChartSection { - title: string; - filter: (label: string) => boolean; +interface Series { + label: string; + data: (number | null)[]; } -function BenchmarkChart({ data, section }: { data: FilteredData; section: ChartSection }) { - const metricLabels = useMemo( - () => data.labels.filter(section.filter).sort(), - [data, section.filter], - ); - const colors = useMemo(() => seriesColors(metricLabels.length), [metricLabels.length]); - const isDense = metricLabels.length >= 8; +/** A point on the x axis: the commit + timestamp behind each entry. */ +interface ChartPoint { + commit: string; + timestamp: string; +} + +/** Render a time-series line chart from pre-built series. */ +function MultiSeriesChart({ + title, + points, + series, +}: { + title: string; + points: ChartPoint[]; + series: Series[]; +}) { + const colors = useMemo(() => seriesColors(series.length), [series.length]); + const isDense = series.length >= 8; const xLabels = useMemo( () => - data.entries.map((e) => { - const date = new Date(e.timestamp); + points.map((p) => { + const date = new Date(p.timestamp); return `${(date.getMonth() + 1).toString().padStart(2, "0")}/${date.getDate().toString().padStart(2, "0")}`; }), - [data], + [points], ); const chartData = useMemo(() => { - const datasets = metricLabels.map((label, i) => ({ - label: shortLabel(label), - data: data.entries.map((e) => e.metrics[label] ?? null), + const datasets = series.map((s, i) => ({ + label: s.label, + data: s.data, borderColor: colors[i], backgroundColor: colors[i], borderWidth: 1.5, @@ -216,7 +233,7 @@ function BenchmarkChart({ data, section }: { data: FilteredData; section: ChartS tension: 0.2, })); return { labels: xLabels, datasets }; - }, [data, metricLabels, colors, xLabels]); + }, [series, colors, xLabels]); const options: ChartOptions<"line"> = useMemo( () => ({ @@ -228,7 +245,7 @@ function BenchmarkChart({ data, section }: { data: FilteredData; section: ChartS plugins: { title: { display: true, - text: section.title, + text: title, font: { size: 16 }, color: "currentcolor", }, @@ -244,15 +261,15 @@ function BenchmarkChart({ data, section }: { data: FilteredData; section: ChartS callbacks: { title: (items) => { if (items.length === 0) return ""; - const entry = data.entries[items[0].dataIndex]; - const date = new Date(entry.timestamp).toLocaleDateString(); - return `${entry.commit.slice(0, 7)} — ${date}`; + const point = points[items[0].dataIndex]; + const date = new Date(point.timestamp).toLocaleDateString(); + return `${point.commit.slice(0, 7)} — ${date}`; }, label: (item) => `${item.dataset.label}: ${item.parsed.y?.toFixed(2)}ms`, afterTitle: (items) => { if (items.length === 0) return ""; - const entry = data.entries[items[0].dataIndex]; - return `${getCommitUrl()}${entry.commit}`; + const point = points[items[0].dataIndex]; + return `${getCommitUrl()}${point.commit}`; }, }, }, @@ -274,10 +291,10 @@ function BenchmarkChart({ data, section }: { data: FilteredData; section: ChartS }, }, }), - [data, section, isDense], + [title, points, isDense], ); - if (metricLabels.length === 0) return null; + if (series.length === 0) return null; return (
@@ -286,6 +303,36 @@ function BenchmarkChart({ data, section }: { data: FilteredData; section: ChartS ); } +interface ChartSection { + title: string; + filter: (label: string) => boolean; +} + +function BenchmarkChart({ data, section }: { data: FilteredData; section: ChartSection }) { + const metricLabels = useMemo( + () => data.labels.filter(section.filter).sort(), + [data, section.filter], + ); + + const points = useMemo( + () => data.entries.map((e) => ({ commit: e.commit, timestamp: e.timestamp })), + [data], + ); + + const series = useMemo( + () => + metricLabels.map((label) => ({ + label: shortLabel(label), + data: data.entries.map((e) => e.metrics[label] ?? null), + })), + [data, metricLabels], + ); + + if (metricLabels.length === 0) return null; + + return ; +} + function MetricSummary({ data }: { data: FilteredData }) { const latest = data.entries[data.entries.length - 1]; const previous = data.entries.length > 1 ? data.entries[data.entries.length - 2] : null; @@ -384,6 +431,167 @@ function EmitterSection({ data, emitterName }: { data: FilteredData; emitterName ); } +// ─── External specs view ────────────────────────────────────────────────────── + +type ExternalMode = "by-spec" | "by-emitter"; + +/** Collect the union of metric labels seen across all specs' specMetrics. */ +function collectSpecMetricLabels(entries: HistoryEntry[]): string[] { + const labels = new Set(); + for (const entry of entries) { + for (const metrics of Object.values(entry.specMetrics ?? {})) { + for (const label of Object.keys(metrics)) labels.add(label); + } + } + return [...labels]; +} + +/** + * External-specs view: a single chart whose comparison axis is switchable. + * - by-spec: fix one spec, one line per emitter (cross-emitter). + * - by-emitter: fix one emitter, one line per spec (cross-spec). + */ +function ExternalView({ + data, + timeRange, + onTimeRange, +}: { + data: HistoryData; + timeRange: TimeRange; + onTimeRange: (r: TimeRange) => void; +}) { + const [mode, setMode] = useState("by-emitter"); + const [selectedSpec, setSelectedSpec] = useState(""); + const [selectedEmitter, setSelectedEmitter] = useState(""); + + const entries = useMemo(() => { + if (timeRange === "all") return data.entries; + const days = timeRange === "30d" ? 30 : 90; + const cutoff = Date.now() - days * 24 * 60 * 60 * 1000; + return data.entries.filter((e) => new Date(e.timestamp).getTime() >= cutoff); + }, [data, timeRange]); + + const specNames = useMemo(() => { + if (data.specNames && data.specNames.length > 0) return data.specNames; + const names = new Set(); + for (const e of entries) for (const n of Object.keys(e.specMetrics ?? {})) names.add(n); + return [...names].sort(); + }, [data, entries]); + + const emitterNames = useMemo(() => getEmitterNames(collectSpecMetricLabels(entries)), [entries]); + + // Default selections once data is available. + useEffect(() => { + if (!selectedSpec && specNames.length > 0) setSelectedSpec(specNames[0]); + }, [specNames, selectedSpec]); + useEffect(() => { + if (!selectedEmitter && emitterNames.length > 0) setSelectedEmitter(emitterNames[0]); + }, [emitterNames, selectedEmitter]); + + const points = useMemo( + () => entries.map((e) => ({ commit: e.commit, timestamp: e.timestamp })), + [entries], + ); + + const series = useMemo(() => { + if (mode === "by-spec") { + if (!selectedSpec) return []; + return emitterNames.map((emitter) => ({ + label: shortLabel(`emit/${emitter}`), + data: entries.map((e) => e.specMetrics?.[selectedSpec]?.[`emit/${emitter}`] ?? null), + })); + } + if (!selectedEmitter) return []; + return specNames.map((spec) => ({ + label: spec, + data: entries.map((e) => e.specMetrics?.[spec]?.[`emit/${selectedEmitter}`] ?? null), + })); + }, [mode, selectedSpec, selectedEmitter, emitterNames, specNames, entries]); + + const title = + mode === "by-spec" + ? `${selectedSpec} — emitters (ms)` + : `${shortLabel(`emit/${selectedEmitter}`)} — specs (ms)`; + + if (entries.length === 0) { + return

No external benchmark data available for the selected filters.

; + } + + return ( +
+
+
+ + +
+ + {mode === "by-spec" ? ( +
+ Spec: + +
+ ) : ( +
+ Emitter: + +
+ )} + +
+ Range: + +
+ + + {entries.length} data points · Updated {new Date(data.generated).toLocaleDateString()} + +
+ + +
+ ); +} + // ─── Main Dashboard ─────────────────────────────────────────────────────────── export function BenchmarkDashboard() { @@ -394,12 +602,14 @@ export function BenchmarkDashboard() { const [activeTab, setActiveTab] = useState(initialParams.tab); const [selectedSpec, setSelectedSpec] = useState(initialParams.spec); const [timeRange, setTimeRange] = useState(initialParams.range); + const [dataset, setDataset] = useState(initialParams.dataset); - const historyUrl = useMemo(getHistoryUrl, []); + const historyUrl = useMemo(() => getHistoryUrl(dataset), [dataset]); const fetchData = useCallback(() => { setLoading(true); setError(null); + setData(null); fetch(historyUrl) .then((res) => { if (!res.ok) throw new Error(`HTTP ${res.status}`); @@ -421,8 +631,8 @@ export function BenchmarkDashboard() { // Sync state to URL useEffect(() => { - updateUrlParams({ tab: activeTab, spec: selectedSpec, range: timeRange }); - }, [activeTab, selectedSpec, timeRange]); + updateUrlParams({ tab: activeTab, spec: selectedSpec, range: timeRange, dataset }); + }, [activeTab, selectedSpec, timeRange, dataset]); // Derive filtered data const filteredData: FilteredData | null = useMemo(() => { @@ -467,138 +677,167 @@ export function BenchmarkDashboard() { [filteredData], ); - // Loading state + const datasetSwitch = ( +
+ + +
+ ); + + let body: React.ReactNode; if (loading) { - return ( + body = (

Loading benchmark data...

); - } - - // Error state - if (error) { - return ( + } else if (error) { + body = (

Failed to load benchmark data: {error}

- Make sure the benchmark-data branch exists and has been pushed to the remote repository. + {dataset === "external" + ? "External-specs results appear once the external group has run on main (external-results/history.json)." + : "Make sure the benchmark-data branch exists and has been pushed to the remote repository."}

); - } - - if (!filteredData || filteredData.entries.length === 0) { - return ( + } else if (dataset === "external") { + body = data ? ( + + ) : ( +
+

No external benchmark data available.

+
+ ); + } else if (!filteredData || filteredData.entries.length === 0) { + body = (

No benchmark data available for the selected filters.

); - } - - return ( -
- {/* Controls: spec selector + time range */} -
- {specNames.length > 0 && ( + } else { + body = ( + <> + {/* Controls: spec selector + time range */} +
+ {specNames.length > 0 && ( +
+ Spec: + +
+ )}
- Spec: + Range:
- )} -
- Range: - + + {filteredData.entries.length} data points · Updated{" "} + {new Date(filteredData.generated).toLocaleDateString()} +
- - {filteredData.entries.length} data points · Updated{" "} - {new Date(filteredData.generated).toLocaleDateString()} - -
- + - {/* Tab navigation */} -
- {TABS.map((tab) => ( - - ))} -
+ {/* Tab navigation */} +
+ {TABS.map((tab) => ( + + ))} +
- {/* Tab content */} - {activeTab === "stages" && ( - - ["total", "loader", "resolver", "checker", "linter", "validation"].includes(l), - }} - /> - )} + {/* Tab content */} + {activeTab === "stages" && ( + + ["total", "loader", "resolver", "checker", "linter", "validation"].includes(l), + }} + /> + )} - {activeTab === "linter" && ( - l.startsWith("linter/") && l !== "linter", - }} - /> - )} + {activeTab === "linter" && ( + l.startsWith("linter/") && l !== "linter", + }} + /> + )} - {activeTab === "validation" && ( - l.startsWith("validation/") && l !== "validation", - }} - /> - )} + {activeTab === "validation" && ( + l.startsWith("validation/") && l !== "validation", + }} + /> + )} - {activeTab === "emitters" && ( -
- {emitterNames.length === 0 ? ( -

No emitter data available.

- ) : ( - emitterNames.map((name) => ( - - )) - )} -
- )} + {activeTab === "emitters" && ( +
+ {emitterNames.length === 0 ? ( +

No emitter data available.

+ ) : ( + emitterNames.map((name) => ( + + )) + )} +
+ )} + + ); + } + + return ( +
+ {datasetSwitch} + {body}
); } diff --git a/website/src/pages/benchmarks.astro b/website/src/pages/benchmarks.astro index 68839cf3ce..42a970f476 100644 --- a/website/src/pages/benchmarks.astro +++ b/website/src/pages/benchmarks.astro @@ -13,7 +13,8 @@ import { BenchmarkDashboard } from "../components/benchmarks/benchmark-dashboard href="https://github.com/Azure/typespec-azure/tree/benchmark-data" target="_blank" rel="noopener noreferrer">benchmark-data branch. + > branch. Use the Main baseline / External specs switch to view + the consistent in-repo baseline or the separate external-specs track.