Benchmark external specs (azure-rest-api-specs) with a separate dashboard

.github/workflows/benchmark-comment-external.yml

@@ -0,0 +1,26 @@
+name: Benchmark Comment External
+
+on:
+  workflow_run:
+    workflows: ["Benchmark External"]
+    types:
+      - completed
+
+permissions:
+  actions: read
+  pull-requests: write
+
+jobs:
+  comment:
+    if: >
+      github.event.workflow_run.event == 'pull_request' &&
+      github.event.workflow_run.conclusion == 'success'
+    uses: ./.github/workflows/benchmark-comment-run.yml
+    with:
+      run-id: ${{ github.event.workflow_run.id }}
+      header: benchmark-results-external
+      comment-file: benchmark-comment.external.md
+    permissions:
+      actions: read
+      pull-requests: write
+    secrets: inherit

.github/workflows/benchmark-comment-run.yml

@@ -0,0 +1,46 @@
+name: Benchmark Comment Run
+
+# Reusable workflow that posts one sticky benchmark comment from a Benchmark run.
+on:
+  workflow_call:
+    inputs:
+      run-id:
+        description: "The Benchmark workflow run id to download the comment artifact from"
+        required: true
+        type: string
+      header:
+        description: "Sticky comment header (identifies the comment to update)"
+        required: true
+        type: string
+      comment-file:
+        description: "Comment markdown file name inside the artifact"
+        required: true
+        type: string
+
+permissions:
+  actions: read
+  pull-requests: write
+
+jobs:
+  comment:
+    name: Post Benchmark Comment
+    runs-on: ubuntu-latest
+    steps:
+      - name: Download benchmark comment artifact
+        uses: actions/download-artifact@v8
+        with:
+          name: benchmark-comment
+          run-id: ${{ inputs.run-id }}
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Get PR number
+        id: get-pr
+        run: echo "pr_number=$(cat benchmark-pr-number.txt)" >> "$GITHUB_OUTPUT"
+
+      - name: Post PR comment
+        uses: marocchino/sticky-pull-request-comment@v3
+        with:
+          header: ${{ inputs.header }}
+          number: ${{ steps.get-pr.outputs.pr_number }}
+          path: ${{ inputs.comment-file }}
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/benchmark-comment.yml

@@ -7,31 +7,20 @@ on:
       - completed
 
 permissions:
+  actions: read
   pull-requests: write
 
 jobs:
   comment:
-    name: Post Benchmark Comment
-    runs-on: ubuntu-latest
     if: >
       github.event.workflow_run.event == 'pull_request' &&
       github.event.workflow_run.conclusion == 'success'
-    steps:
-      - name: Download benchmark comment artifact
-        uses: actions/download-artifact@v8
-        with:
-          name: benchmark-comment
-          run-id: ${{ github.event.workflow_run.id }}
-          github-token: ${{ secrets.GITHUB_TOKEN }}
-
-      - name: Get PR number
-        id: get-pr
-        run: echo "pr_number=$(cat benchmark-pr-number.txt)" >> "$GITHUB_OUTPUT"
-
-      - name: Post PR comment
-        uses: marocchino/sticky-pull-request-comment@v3
-        with:
-          header: benchmark-results
-          number: ${{ steps.get-pr.outputs.pr_number }}
-          path: benchmark-comment.md
-          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+    uses: ./.github/workflows/benchmark-comment-run.yml
+    with:
+      run-id: ${{ github.event.workflow_run.id }}
+      header: benchmark-results
+      comment-file: benchmark-comment.md
+    permissions:
+      actions: read
+      pull-requests: write
+    secrets: inherit

.github/workflows/benchmark-external.yml

@@ -0,0 +1,41 @@
+name: Benchmark External
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+      - release/*
+  workflow_dispatch:
+    inputs:
+      runner:
+        description: "Runner label (for stable runs prefer self-hosted or larger dedicated runner)"
+        required: false
+        type: string
+        default: "ubuntu-latest"
+
+permissions:
+  contents: write
+
+concurrency:
+  group: benchmark-external-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  run:
+    uses: ./.github/workflows/benchmark-run.yml
+    with:
+      specs-dir: packages/benchmark/external-spec
+      results-dir: external-results
+      title: "⚡ External Spec Benchmark Results"
+      comment-file: benchmark-comment.external.md
+      iterations: 3
+      warmup: 1
+      noise-cv: 0.1
+      rerun-iterations: 2
+      runner: ${{ github.event_name == 'workflow_dispatch' && inputs.runner || vars.BENCHMARK_RUNNER || 'ubuntu-latest' }}
+    permissions:
+      contents: write
+    secrets: inherit

.github/workflows/benchmark-run.yml

@@ -0,0 +1,133 @@
+name: Benchmark Run
+
+# Reusable workflow that benchmarks a single spec group. Instantiate it once per
+# spec group (see benchmark.yml and benchmark-external.yml).
+on:
+  workflow_call:
+    inputs:
+      specs-dir:
+        description: "Directory of benchmark specs to run"
+        required: true
+        type: string
+      results-dir:
+        description: "Directory on the data branch to store results/history"
+        required: true
+        type: string
+      title:
+        description: "Heading for the PR comment"
+        required: true
+        type: string
+      comment-file:
+        description: "Comment markdown file name (must match the comment instance)"
+        required: true
+        type: string
+      iterations:
+        required: false
+        type: number
+        default: 25
+      warmup:
+        required: false
+        type: number
+        default: 3
+      noise-cv:
+        required: false
+        type: number
+        default: 0.08
+      rerun-iterations:
+        required: false
+        type: number
+        default: 10
+      runner:
+        required: false
+        type: string
+        default: "ubuntu-latest"
+      backfill_from:
+        description: "Backfill from: commit SHA or number of recent commits (main only)"
+        required: false
+        type: string
+        default: ""
+      branch:
+        description: "Data branch to store results"
+        required: false
+        type: string
+        default: "benchmark-data"
+
+permissions:
+  contents: write
+
+jobs:
+  benchmark:
+    name: Run Benchmarks
+    runs-on: ${{ inputs.runner }}
+    env:
+      TYPESPEC_VS_CI_BUILD: true
+      TYPESPEC_SKIP_WEBSITE_BUILD: true
+    steps:
+      - uses: actions/checkout@v7
+        with:
+          submodules: recursive
+          fetch-depth: 0
+
+      - uses: ./.github/actions/setup
+        with:
+          node-version: 24.15.0
+
+      - name: Install dependencies
+        run: pnpm install
+
+      - name: Build
+        run: pnpm -r --filter "@azure-tools/typespec-benchmark..." build
+
+      - name: Run backfill
+        if: ${{ inputs.backfill_from != '' }}
+        run: |
+          node --max-old-space-size=6144 packages/benchmark/dist/src/cli.js backfill \
+            --from ${{ inputs.backfill_from }} \
+            --specs-dir ${{ inputs.specs-dir }} \
+            --iterations ${{ inputs.iterations }} \
+            --warmup ${{ inputs.warmup }} \
+            --branch ${{ inputs.branch }} \
+            --push
+
+      - name: Run benchmarks
+        if: ${{ inputs.backfill_from == '' }}
+        run: |
+          node --max-old-space-size=6144 packages/benchmark/dist/src/cli.js run \
+            --specs-dir ${{ inputs.specs-dir }} \
+            --iterations ${{ inputs.iterations }} \
+            --warmup ${{ inputs.warmup }} \
+            --noise-cv-threshold ${{ inputs.noise-cv }} \
+            --max-reruns 1 \
+            --rerun-iterations ${{ inputs.rerun-iterations }} \
+            --commit ${{ github.sha }} \
+            --output /tmp/benchmark-results.json
+
+      # On push to main: store results to the data branch.
+      - name: Store benchmark results
+        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
+        run: |
+          node packages/benchmark/dist/src/cli.js store-results \
+            --results /tmp/benchmark-results.json \
+            --commit ${{ github.sha }} \
+            --results-dir ${{ inputs.results-dir }}
+
+      # On PR: fetch baseline, compare, and upload as artifact for the comment workflow.
+      - name: Generate PR comment
+        if: github.event_name == 'pull_request'
+        run: |
+          node packages/benchmark/dist/src/cli.js upload-pr-comment \
+            --results /tmp/benchmark-results.json \
+            --pr-number ${{ github.event.number }} \
+            --results-dir ${{ inputs.results-dir }} \
+            --baseline-window 20 \
+            --title "${{ inputs.title }}" \
+            --comment-file ${{ inputs.comment-file }} \
+            --output-dir /tmp/benchmark-artifacts
+
+      - name: Upload benchmark comment
+        if: github.event_name == 'pull_request'
+        uses: actions/upload-artifact@v7
+        with:
+          name: benchmark-comment
+          path: /tmp/benchmark-artifacts/
+          retention-days: 1

.github/workflows/benchmark.yml

@@ -29,78 +29,24 @@ permissions:
   contents: write
 
 concurrency:
-  group: benchmark-${{ github.ref }}
+  group: benchmark-main-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  benchmark:
-    name: Run Benchmarks
-    runs-on: ${{ github.event_name == 'workflow_dispatch' && inputs.runner || vars.BENCHMARK_RUNNER || 'ubuntu-latest' }}
-    env:
-      TYPESPEC_VS_CI_BUILD: true
-      TYPESPEC_SKIP_WEBSITE_BUILD: true
-    steps:
-      - uses: actions/checkout@v7
-        with:
-          submodules: recursive
-          fetch-depth: 0
-
-      - uses: ./.github/actions/setup
-        with:
-          node-version: 24.15.0
-
-      - name: Install dependencies
-        run: pnpm install
-
-      - name: Build
-        run: pnpm -r --filter "@azure-tools/typespec-benchmark..." build
-
-      - name: Run backfill
-        if: ${{ github.event_name == 'workflow_dispatch' && inputs.backfill_from }}
-        run: |
-          node --max-old-space-size=6144 packages/benchmark/dist/src/cli.js backfill \
-            --from ${{ inputs.backfill_from }} \
-            --specs-dir packages/benchmark/specs \
-            --iterations 25 \
-            --warmup 3 \
-            --branch ${{ inputs.branch }} \
-            --push
-
-      - name: Run benchmarks
-        if: ${{ github.event_name != 'workflow_dispatch' || !inputs.backfill_from }}
-        run: |
-          node --max-old-space-size=6144 packages/benchmark/dist/src/cli.js run \
-            --specs-dir packages/benchmark/specs \
-            --iterations 25 \
-            --warmup 3 \
-            --noise-cv-threshold 0.08 \
-            --max-reruns 1 \
-            --rerun-iterations 10 \
-            --commit ${{ github.sha }} \
-            --output /tmp/benchmark-results.json
-
-      # On push to main: store results to benchmark-data branch
-      - name: Store benchmark results
-        if: github.event_name == 'push' && github.ref == 'refs/heads/main'
-        run: |
-          node packages/benchmark/dist/src/cli.js store-results \
-            --results /tmp/benchmark-results.json \
-            --commit ${{ github.sha }}
-
-      # On PR: fetch baseline, compare, and upload as artifact for the comment workflow
-      - name: Generate PR comment
-        if: github.event_name == 'pull_request'
-        run: |
-          node packages/benchmark/dist/src/cli.js upload-pr-comment \
-            --results /tmp/benchmark-results.json \
-            --pr-number ${{ github.event.number }} \
-            --baseline-window 20 \
-            --output-dir /tmp/benchmark-artifacts
-
-      - name: Upload benchmark comment
-        if: github.event_name == 'pull_request'
-        uses: actions/upload-artifact@v7
-        with:
-          name: benchmark-comment
-          path: /tmp/benchmark-artifacts/
-          retention-days: 1
+  run:
+    uses: ./.github/workflows/benchmark-run.yml
+    with:
+      specs-dir: packages/benchmark/specs
+      results-dir: results
+      title: "⚡ Benchmark Results"
+      comment-file: benchmark-comment.md
+      iterations: 25
+      warmup: 3
+      noise-cv: 0.08
+      rerun-iterations: 10
+      runner: ${{ github.event_name == 'workflow_dispatch' && inputs.runner || vars.BENCHMARK_RUNNER || 'ubuntu-latest' }}
+      backfill_from: ${{ github.event_name == 'workflow_dispatch' && inputs.backfill_from || '' }}
+      branch: ${{ github.event_name == 'workflow_dispatch' && inputs.branch || 'benchmark-data' }}
+    permissions:
+      contents: write
+    secrets: inherit

packages/benchmark/.gitignore

@@ -1,3 +1,4 @@
 dist/
 tsp-output/
 node_modules/
+.external/