Sudhu/megatron ci benchmarking

.github/scripts/benchmark-report-to-markdown.py

@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""
+Convert benchmark results to markdown table.
+Reads from output/benchmark_report.json or output/benchmark_results.tmp
+"""
+import json
+import math
+import os
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+def _fmt_num(val, ndigits=4):
+    """Round floats for GitHub markdown tables; pass through N/A and strings."""
+    if val is None or val == "N/A":
+        return str(val)
+    try:
+        f = float(val)
+        if math.isnan(f) or math.isinf(f):
+            return str(val)
+        return f"{f:.{ndigits}g}"
+    except (TypeError, ValueError):
+        return str(val)
+
+
+def parse_results():
+    rows = []
+    json_path = os.environ.get("BENCHMARK_JSON", "output/benchmark_report.json")
+    results_path = os.environ.get("BENCHMARK_RESULTS", "output/benchmark_results.tmp")
+
+    if Path(json_path).exists():
+        try:
+            with open(json_path) as f:
+                data = json.load(f)
+            for b in data.get("benchmarks", []):
+                rows.append({
+                    "benchmark": b.get("benchmark", ""),
+                    "throughput": b.get("throughput_tflop_s_per_gpu") or "N/A",
+                    "elapsed": b.get("elapsed_ms_per_iter") or "N/A",
+                    "tokens": b.get("tokens_per_gpu_s") or "N/A",
+                    "mem": b.get("mem_gb") or "N/A",
+                })
+        except Exception as e:
+            print(f"Warning: Could not parse JSON: {e}", file=os.sys.stderr)
+
+    if not rows and Path(results_path).exists():
+        with open(results_path) as f:
+            for line in f:
+                m = re.match(r"^([^|]+)\|([^|]*)\|([^|]*)\|([^|]*)\|([^|]*)$", line.strip())
+                first = m.group(1).strip()
+                if m and first != "Benchmark" and not first.startswith("-"):
+                    rows.append({
+                        "benchmark": m.group(1).strip(),
+                        "throughput": m.group(2).strip() or "N/A",
+                        "elapsed": m.group(3).strip() or "N/A",
+                        "tokens": m.group(4).strip() or "N/A",
+                        "mem": m.group(5).strip() or "N/A",
+                    })
+    return rows
+
+
+def to_markdown_table(rows):
+    if not rows:
+        return "No benchmark results available."
+    header = "| Benchmark | Throughput (TFLOP/s/GPU) | Elapsed (ms/iter) | Tokens/GPU/s | Mem (GB) |"
+    sep = "|-----------|--------------------------|-------------------|--------------|----------|"
+    data_rows = [
+        "| "
+        + " | ".join(
+            [
+                r["benchmark"],
+                _fmt_num(r["throughput"], 5),
+                _fmt_num(r["elapsed"], 5),
+                _fmt_num(r["tokens"], 5),
+                _fmt_num(r["mem"], 4),
+            ]
+        )
+        + " |"
+        for r in rows
+    ]
+    return "\n".join([header, sep] + data_rows)
+
+
+def main():
+    try:
+        rows = parse_results()
+    except Exception as e:
+        print(f"Warning: Could not parse benchmark results: {e}", file=os.sys.stderr)
+        rows = []
+    table = to_markdown_table(rows)
+    report = f"""## Megatron-LM Benchmark Performance Report
+
+{table}
+
+*Generated at {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}*"""
+    out_path = os.environ.get("BENCHMARK_MARKDOWN_OUT", "output/benchmark_summary.md")
+    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+    Path(out_path).write_text(report)
+    print(report)
+
+
+if __name__ == "__main__":
+    main()