Organize workloads into model directories

.buildkite/generate_pipeline.py

@@ -3,8 +3,9 @@
 
 Always emits one GPU-profiled step per selected workload. Selection rules:
 
-  WORKLOADS env var set?  → run exactly those paths (comma- or newline-
-                             separated; resolved against workloads/*.yaml)
+  WORKLOADS env var set?  → run exactly those paths or stable aliases (comma-
+                             or newline-separated; resolved recursively below
+                             workloads/)
   Otherwise               → run every workload with ``nightly: true``
 
 Override env vars are propagated to each step:
@@ -52,8 +53,16 @@ def setup_command(packages):
 
 DEFAULT_TIMEOUT = 120
 PROFILES_PATH = os.path.join(os.path.dirname(__file__), "..", "lib", "gpu_profiles.yaml")
+WORKLOADS_GLOB = "workloads/**/*.yaml"
 DEFAULT_IMAGE_REPO = "vllm/vllm-openai"
 
+# Preserve the one historical filename that cannot be derived mechanically
+# from <model>/<hardware>.yaml. All other flat stems map by replacing "/" with
+# "_" (for example qwen3_5/h200 -> qwen3_5_h200).
+LEGACY_WORKLOAD_ALIASES = {
+    "deepseek_v4_pro_5_h200": "workloads/deepseek_v4_pro/h200.yaml",
+}
+
 GPU_EMOJI = {
     "H200": ":h200:",
     "B200": ":b200:",
@@ -240,13 +249,41 @@ def load_profiles():
 
 def load_workloads():
     workloads = []
-    for path in sorted(glob.glob("workloads/*.yaml")):
+    for path in sorted(glob.glob(WORKLOADS_GLOB, recursive=True)):
         with open(path) as f:
             data = yaml.safe_load(f)
         workloads.append({"path": path, "data": data})
     return workloads
 
 
+def workload_aliases(workload):
+    """Return stable selectors for a nested workload path.
+
+    The canonical selector is ``model/hardware``. Exact paths, YAML names, and
+    the former flat filename stem remain accepted so existing Buildkite
+    triggers do not break when files move into model directories.
+    """
+    path = workload["path"]
+    rel = os.path.relpath(path, "workloads")
+    rel_stem = os.path.splitext(rel)[0]
+    legacy_stem = rel_stem.replace(os.sep, "_")
+    aliases = {
+        path,
+        os.path.splitext(path)[0],
+        rel,
+        rel_stem,
+        legacy_stem,
+        f"workloads/{legacy_stem}.yaml",
+    }
+    name = workload["data"].get("name")
+    if name:
+        aliases.add(name)
+    for alias, target in LEGACY_WORKLOAD_ALIASES.items():
+        if target == path:
+            aliases.update({alias, f"workloads/{alias}.yaml"})
+    return aliases
+
+
 def queue_for_gpu(gpu, profile):
     override = (os.environ.get(f"{gpu.upper()}_QUEUE") or "").strip()
     return override or profile["queue"]
@@ -307,21 +344,24 @@ def make_step(path, data, profiles):
 def select_workloads(workloads):
     raw = (os.environ.get("WORKLOADS") or "").strip()
     if raw:
-        # Accept comma- or newline-separated. Each entry is a workload path
-        # (e.g. workloads/qwen3_5_h200.yaml) or a bare name (qwen3_5_h200).
+        # Accept comma- or newline-separated canonical paths, model/hardware
+        # selectors, YAML names, and legacy flat filename stems.
         entries = [e.strip() for e in raw.replace(",", "\n").split("\n") if e.strip()]
-        by_path = {w["path"]: w for w in workloads}
-        by_stem = {os.path.basename(w["path"]).removesuffix(".yaml"): w for w in workloads}
+        by_alias = {}
+        for workload in workloads:
+            for alias in workload_aliases(workload):
+                by_alias.setdefault(alias, []).append(workload)
         selected = []
         for e in entries:
-            if e in by_path:
-                selected.append(by_path[e])
-            elif e in by_stem:
-                selected.append(by_stem[e])
-            elif (f"workloads/{e}.yaml") in by_path:
-                selected.append(by_path[f"workloads/{e}.yaml"])
-            else:
-                sys.exit(f"WORKLOADS entry {e!r} did not match any file in workloads/")
+            matches = by_alias.get(e, [])
+            if not matches:
+                sys.exit(
+                    f"WORKLOADS entry {e!r} did not match any workload below workloads/"
+                )
+            if len(matches) > 1:
+                paths = ", ".join(w["path"] for w in matches)
+                sys.exit(f"WORKLOADS entry {e!r} is ambiguous; matches {paths}")
+            selected.append(matches[0])
         return selected
     return [w for w in workloads if w["data"].get("nightly") is True]
 

.buildkite/test_generate_pipeline.py

@@ -4,9 +4,10 @@
 Run with ``python3 .buildkite/test_generate_pipeline.py`` (needs only pyyaml,
 which the pipeline already installs). No pytest / GPU / network required.
 
-Guards the HF-cache volume behaviour: the AMD k8s plugin must NOT emit a
-root-disk hostPath by default (that leaked model caches onto node root disks),
-and the volume source must be overridable per-cluster.
+Guards workload discovery/selection and the HF-cache volume behaviour: nested
+model directories must remain discoverable without breaking existing
+WORKLOADS selectors, and the AMD k8s plugin must not emit a root-disk hostPath
+by default.
 """
 
 import importlib.util
@@ -100,6 +101,67 @@ def test_shipped_amd_profiles_have_no_rootdisk_hostpath():
         )
 
 
+def _select(workloads, value):
+    previous = os.environ.get("WORKLOADS")
+    os.environ["WORKLOADS"] = value
+    try:
+        return g.select_workloads(workloads)
+    finally:
+        if previous is None:
+            os.environ.pop("WORKLOADS", None)
+        else:
+            os.environ["WORKLOADS"] = previous
+
+
+def test_load_workloads_discovers_nested_model_directories():
+    workloads = g.load_workloads()
+    paths = {w["path"] for w in workloads}
+    assert "workloads/qwen3_5/h200.yaml" in paths
+    assert "workloads/minimax_m3/b200.yaml" in paths
+    assert all(os.path.dirname(path) != "workloads" for path in paths), paths
+
+
+def test_select_workloads_accepts_nested_and_legacy_aliases():
+    workload = {
+        "path": "workloads/qwen3_5/h200.yaml",
+        "data": {"name": "qwen3_5-h200", "nightly": True},
+    }
+    selectors = (
+        "workloads/qwen3_5/h200.yaml",
+        "workloads/qwen3_5/h200",
+        "qwen3_5/h200.yaml",
+        "qwen3_5/h200",
+        "qwen3_5-h200",
+        "qwen3_5_h200",
+        "workloads/qwen3_5_h200.yaml",
+    )
+    for selector in selectors:
+        selected = _select([workload], selector)
+        assert selected == [workload], (selector, selected)
+
+
+def test_select_workloads_preserves_irregular_legacy_alias():
+    workload = {
+        "path": "workloads/deepseek_v4_pro/h200.yaml",
+        "data": {"name": "deepseek_v4_pro-h200", "nightly": True},
+    }
+    assert _select([workload], "deepseek_v4_pro_5_h200") == [workload]
+    assert _select([workload], "workloads/deepseek_v4_pro_5_h200.yaml") == [workload]
+
+
+def test_select_workloads_rejects_ambiguous_yaml_names():
+    workloads = [
+        {"path": "workloads/alpha/h200.yaml", "data": {"name": "duplicate"}},
+        {"path": "workloads/beta/h200.yaml", "data": {"name": "duplicate"}},
+    ]
+    try:
+        _select(workloads, "duplicate")
+    except SystemExit as exc:
+        assert "ambiguous" in str(exc), exc
+    else:
+        raise AssertionError("ambiguous WORKLOADS selector was accepted")
+
+
 def main():
     tests = [v for k, v in sorted(globals().items()) if k.startswith("test_")]
     failed = 0

CLAUDE.md

@@ -1,6 +1,6 @@
 # Agent instructions for perf-eval
 
-This repo orchestrates lm-evaluation-harness runs against vLLM via YAML workloads in `workloads/`. `lib/run.sh` parses a workload, brings up vLLM in Docker, and dispatches each task to a helper in `lib/`. Real runs need GPUs and are exercised on Buildkite.
+This repo orchestrates lm-evaluation-harness runs against vLLM via YAML workloads in `workloads/<model>/<hardware>.yaml`. `lib/run.sh` parses a workload, brings up vLLM in Docker, and dispatches each task to a helper in `lib/`. Real runs need GPUs and are exercised on Buildkite.
 
 ## Keep the README in sync
 
@@ -29,7 +29,7 @@ m = types.ModuleType('lm_eval'); t = types.ModuleType('lm_eval.tasks')
 class TM: all_tasks = ['gsm8k', 'aime25']
 t.TaskManager = TM
 sys.modules['lm_eval'] = m; sys.modules['lm_eval.tasks'] = t
-sys.argv = ['parse_workload.py', 'workloads/qwen3_5_h200.yaml']
+sys.argv = ['parse_workload.py', 'workloads/qwen3_5/h200.yaml']
 exec(open('lib/parse_workload.py').read())
 "
 ```
@@ -55,7 +55,7 @@ Pipeline metadata:
 - **pipeline**: `perf-eval`
 - **repo**: `github.com/vllm-project/perf-eval`
 - **default branch**: `main`
-- **what it runs**: a dynamic pipeline. A bootstrap step runs `.buildkite/generate_pipeline.py` and generates per-workload steps using each workload's GPU profile. When `WORKLOADS` is set, it runs exactly those workload paths or stems. Otherwise it discovers all `workloads/*.yaml` with `nightly: true`.
+- **what it runs**: a dynamic pipeline. A bootstrap step runs `.buildkite/generate_pipeline.py` and generates per-workload steps using each workload's GPU profile. When `WORKLOADS` is set, it runs exactly those workload paths or `model/hardware` selectors (legacy flat stems remain supported). Otherwise it recursively discovers all `workloads/**/*.yaml` with `nightly: true`.
 
 ### Workflow
 

README.md

@@ -7,7 +7,7 @@ Each recipe is one `(model, hardware, set of tasks)` combination. The Buildkite
 ## Repo layout
 
 ```
-workloads/        one YAML per (model, hardware) recipe
+workloads/        one directory per model, with one YAML recipe per hardware
 lib/              orchestrator (run.sh), helpers, GPU profiles
 .buildkite/       pipeline bootstrap, step generator, and its tests
 CLAUDE.md         agent conventions and detailed Buildkite workflow
@@ -17,10 +17,10 @@ CLAUDE.md         agent conventions and detailed Buildkite workflow
 
 ### Add a new recipe
 
-1. Copy an existing workload that targets the same GPU — e.g. `workloads/qwen3_5_h200.yaml` for H200 or `workloads/minimax_m3_b200.yaml` for B200.
-2. Name the file `<model>_<hardware>.yaml`. Keep hardware variants in separate files.
+1. Copy an existing workload that targets the same GPU — e.g. `workloads/qwen3_5/h200.yaml` for H200 or `workloads/minimax_m3/b200.yaml` for B200.
+2. Create `workloads/<model>/` when adding a new model, then name each hardware variant `<hardware>.yaml` inside it.
 3. Edit the fields to match your model and tasks. Set `nightly: true` if it should run in the nightly schedule; leave it off for opt-in recipes.
-4. Open a PR. The pipeline auto-discovers `workloads/*.yaml` — no Buildkite YAML edits needed.
+4. Open a PR. The pipeline recursively discovers `workloads/**/*.yaml` — no Buildkite YAML edits needed.
 
 B200 workloads run in a single Kubernetes pod. `num_gpus` controls the pod's
 GPU allocation; use at most 8 GPUs to keep the workload on one B200 node.
@@ -101,7 +101,7 @@ A few things worth knowing:
 - **`bfcl.maximum_step_limit`** caps how many inference steps BFCL allows per multi-turn turn (default 10 in perf-eval; BFCL upstream defaults to 20). Set it in the workload YAML, or override per-run with the `BFCL_MAXIMUM_STEP_LIMIT` env var (env wins over YAML). Useful for agentic / long multi-turn categories.
 - **`bfcl.max_test_cases`** subsamples a category instead of running the full set — e.g. `multi_turn` (~800 cases) down to 300. For aggregate groups with multiple subcategories, the cap is split evenly across subcategories (by BFCL id order within each). Set a single integer to cap every category, or a map per category (`multi_turn: 240`). Override per-run with `BFCL_MAX_TEST_CASES`. Scores are partial-eval only and are not comparable to full BFCL leaderboard numbers.
 
-For everything else (the full set of supported fields, defaults, validation rules), the existing files in `workloads/` are the working reference and `lib/parse_workload.py` is the source of truth.
+For everything else (the full set of supported fields, defaults, validation rules), the existing files below `workloads/<model>/` are the working reference and `lib/parse_workload.py` is the source of truth.
 
 ### HF cache volume (Kubernetes profiles)
 
@@ -134,7 +134,7 @@ The pipeline is [**`vllm/perf-eval`**](https://buildkite.com/vllm/perf-eval). Wi
 
 **Optional env vars:**
 
-- `WORKLOADS` — comma- or newline-separated list of workload paths or stems. Runs exactly those instead of the default `nightly: true` set.
+- `WORKLOADS` — comma- or newline-separated list of workload paths or `model/hardware` selectors. Runs exactly those instead of the default `nightly: true` set. YAML `name` values and former flat filename stems such as `qwen3_5_h200` remain supported for existing triggers.
 - `NIGHTLY` — set to `1` to tag every ingested row with `nightly: true`. The dashboard's `/nightly` view filters on this to pair adjacent nightly builds; only the scheduled nightly cron should set it.
 
 **Example — trigger a build from the Buildkite UI:**
@@ -145,11 +145,11 @@ The pipeline is [**`vllm/perf-eval`**](https://buildkite.com/vllm/perf-eval). Wi
    ```
    VLLM_COMMIT=abc1234def5678
    VLLM_IMAGE=vllm/vllm-openai:nightly-abc1234def5678
-   WORKLOADS=qwen3_5_h200
+   WORKLOADS=qwen3_5/h200
    ```
 4. Click **Create Build**.
 
-This runs the `qwen3_5_h200` workload against the specified vLLM nightly image. Omit `WORKLOADS` to run all `nightly: true` workloads.
+This runs `workloads/qwen3_5/h200.yaml` against the specified vLLM nightly image. Omit `WORKLOADS` to run all `nightly: true` workloads.
 
 **From an agent:** see `CLAUDE.md` for the Buildkite MCP and authenticated
 `bk` workflows. Don't make raw Buildkite API calls with `curl`.
@@ -159,7 +159,7 @@ This runs the `qwen3_5_h200` workload against the specified vLLM nightly image.
 A real run needs a GPU host with Docker, vLLM, and lm-eval available:
 
 ```bash
-./lib/run.sh workloads/qwen3_5_h200.yaml
+./lib/run.sh workloads/qwen3_5/h200.yaml
 ```
 
 Locally, you can smoke-test recipe changes without a GPU — see `CLAUDE.md` for the parser stub and shell-syntax checks.

lib/parse_workload.py

@@ -1,6 +1,6 @@
 """Read a workload YAML and emit `WORKLOAD_*` shell exports for run.sh.
 
-Usage: eval "$(python3 lib/parse_workload.py workloads/foo.yaml)"
+Usage: eval "$(python3 lib/parse_workload.py workloads/foo/h200.yaml)"
 
 The README documents the recipe schema. This script validates it and
 projects it into shell variables: top-level metadata, server config
@@ -41,6 +41,8 @@
     "maximum_step_limit", "max_test_cases",
 }
 BFCL_DEFAULT_MAXIMUM_STEP_LIMIT = 10
+REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
+PROFILES_PATH = os.path.join(REPO_ROOT, "lib", "gpu_profiles.yaml")
 BFCL_KNOWN_CATEGORIES = {
     "simple_python", "simple_java", "simple_javascript",
     "multiple", "parallel", "parallel_multiple", "irrelevance",
@@ -92,13 +94,11 @@ def known_task_names() -> set:
     return set(TaskManager().all_tasks)
 
 
-def load_profile(gpu: str, workload_path: str) -> dict:
-    repo_root = os.path.dirname(os.path.dirname(os.path.abspath(workload_path)))
-    profiles_path = os.path.join(repo_root, "lib", "gpu_profiles.yaml")
-    with open(profiles_path) as f:
+def load_profile(gpu: str) -> dict:
+    with open(PROFILES_PATH) as f:
         profiles = yaml.safe_load(f)
     if gpu not in profiles:
-        sys.exit(f"unknown gpu {gpu!r} in {profiles_path} (have {', '.join(profiles)})")
+        sys.exit(f"unknown gpu {gpu!r} in {PROFILES_PATH} (have {', '.join(profiles)})")
     return profiles[gpu]
 
 
@@ -337,7 +337,7 @@ def main(path: str) -> None:
     gpu = data.get("gpu")
     if not gpu:
         sys.exit(f"{path}: missing required 'gpu' field")
-    profile = load_profile(gpu, path)
+    profile = load_profile(gpu)
     vllm = data.get("vllm") or {}
     lm_eval = data.get("lm_eval") or {}
     bench = data.get("vllm_bench") or {}

lib/run.sh

@@ -2,7 +2,7 @@
 # Orchestrate a workload: bring up vLLM, then dispatch each task to the
 # helper script for its type.
 #
-# Usage: ./lib/run.sh workloads/qwen3_5_h200.yaml
+# Usage: ./lib/run.sh workloads/qwen3_5/h200.yaml
 set -euo pipefail
 
 WORKLOAD="${1:?usage: $0 <workload.yaml>}"