diff --git a/.github/scripts/benchmark-report-to-markdown.py b/.github/scripts/benchmark-report-to-markdown.py
new file mode 100644
index 00000000000..c3548418493
--- /dev/null
+++ b/.github/scripts/benchmark-report-to-markdown.py
@@ -0,0 +1,104 @@
+#!/usr/bin/env python3
+"""
+Convert benchmark results to markdown table.
+Reads from output/benchmark_report.json or output/benchmark_results.tmp
+"""
+import json
+import math
+import os
+import re
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+def _fmt_num(val, ndigits=4):
+    """Round floats for GitHub markdown tables; pass through N/A and strings."""
+    if val is None or val == "N/A":
+        return str(val)
+    try:
+        f = float(val)
+        if math.isnan(f) or math.isinf(f):
+            return str(val)
+        return f"{f:.{ndigits}g}"
+    except (TypeError, ValueError):
+        return str(val)
+
+
+def parse_results():
+    rows = []
+    json_path = os.environ.get("BENCHMARK_JSON", "output/benchmark_report.json")
+    results_path = os.environ.get("BENCHMARK_RESULTS", "output/benchmark_results.tmp")
+
+    if Path(json_path).exists():
+        try:
+            with open(json_path) as f:
+                data = json.load(f)
+            for b in data.get("benchmarks", []):
+                rows.append({
+                    "benchmark": b.get("benchmark", ""),
+                    "throughput": b.get("throughput_tflop_s_per_gpu") or "N/A",
+                    "elapsed": b.get("elapsed_ms_per_iter") or "N/A",
+                    "tokens": b.get("tokens_per_gpu_s") or "N/A",
+                    "mem": b.get("mem_gb") or "N/A",
+                })
+        except Exception as e:
+            print(f"Warning: Could not parse JSON: {e}", file=os.sys.stderr)
+
+    if not rows and Path(results_path).exists():
+        with open(results_path) as f:
+            for line in f:
+                m = re.match(r"^([^|]+)\|([^|]*)\|([^|]*)\|([^|]*)\|([^|]*)$", line.strip())
+                first = m.group(1).strip()
+                if m and first != "Benchmark" and not first.startswith("-"):
+                    rows.append({
+                        "benchmark": m.group(1).strip(),
+                        "throughput": m.group(2).strip() or "N/A",
+                        "elapsed": m.group(3).strip() or "N/A",
+                        "tokens": m.group(4).strip() or "N/A",
+                        "mem": m.group(5).strip() or "N/A",
+                    })
+    return rows
+
+
+def to_markdown_table(rows):
+    if not rows:
+        return "No benchmark results available."
+    header = "| Benchmark | Throughput (TFLOP/s/GPU) | Elapsed (ms/iter) | Tokens/GPU/s | Mem (GB) |"
+    sep = "|-----------|--------------------------|-------------------|--------------|----------|"
+    data_rows = [
+        "| "
+        + " | ".join(
+            [
+                r["benchmark"],
+                _fmt_num(r["throughput"], 5),
+                _fmt_num(r["elapsed"], 5),
+                _fmt_num(r["tokens"], 5),
+                _fmt_num(r["mem"], 4),
+            ]
+        )
+        + " |"
+        for r in rows
+    ]
+    return "\n".join([header, sep] + data_rows)
+
+
+def main():
+    try:
+        rows = parse_results()
+    except Exception as e:
+        print(f"Warning: Could not parse benchmark results: {e}", file=os.sys.stderr)
+        rows = []
+    table = to_markdown_table(rows)
+    report = f"""## Megatron-LM Benchmark Performance Report
+
+{table}
+
+*Generated at {datetime.now(timezone.utc).strftime('%Y-%m-%dT%H:%M:%SZ')}*"""
+    out_path = os.environ.get("BENCHMARK_MARKDOWN_OUT", "output/benchmark_summary.md")
+    Path(out_path).parent.mkdir(parents=True, exist_ok=True)
+    Path(out_path).write_text(report)
+    print(report)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/.github/workflows/megatron-benchmarks.yml b/.github/workflows/megatron-benchmarks.yml
new file mode 100644
index 00000000000..d1e4d5702ca
--- /dev/null
+++ b/.github/workflows/megatron-benchmarks.yml
@@ -0,0 +1,296 @@
+# Manual benchmarks / profiling only — does not run unit tests.
+# Single runner job: build image locally (no final image push / no image tar artifacts).
+# Read-only registry layer cache (no cache-to): TE_COMMIT changes won't write new cache blobs from this workflow.
+# Actions → "Megatron-LM benchmarks (manual)" → Run workflow
+
+name: Megatron-LM benchmarks (manual)
+
+on:
+  workflow_dispatch:
+    inputs:
+      gpu_arch:
+        description: 'GPU architecture for the Docker build (e.g. gfx942) if not detected'
+        required: false
+        default: 'gfx942'
+      te_branch:
+        description: 'ROCm/TransformerEngine.git branch used to resolve TE_COMMIT build-arg'
+        required: false
+        default: 'release_v2.10_rocm'
+        type: string
+      benchmark_preset:
+        description: 'Named benchmark matrix (see run_benchmarks.sh). LLAMA FSDP = llama_fsdp (70B FSDP+recompute).'
+        required: false
+        default: 'all'
+        type: choice
+        options:
+          - all
+          - llama
+          - deepseek
+          - llama_fsdp
+          - llama_8b
+          - llama_70b
+          - deepseek_v2
+          - deepseek_v3
+      total_iters:
+        description: 'Training iterations per benchmark row (TOTAL_ITERS / TRAIN_ITERS)'
+        required: false
+        default: '12'
+        type: string
+      generate_benchmark_charts:
+        description: 'Emit memory_footprint + perf_breakdown PNG/JSON under output/'
+        required: false
+        default: false
+        type: boolean
+      run_te_profile:
+        description: 'After benchmarks, run TE E2E profiler (run_profile_ci.sh)'
+        required: false
+        default: false
+        type: boolean
+
+concurrency:
+  group: megatron-benchmarks-${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+jobs:
+  benchmark:
+    name: Build image and run benchmarks
+    runs-on: linux-megatron-mi325-8
+    timeout-minutes: 1000
+    continue-on-error: false
+    permissions:
+      contents: read
+      checks: write
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v6
+        with:
+          fetch-depth: 0
+
+      - name: Print environment
+        run: |
+          echo "::group::Environment"
+          env | sort
+          echo "::endgroup::"
+
+      - name: ROCm diagnostics (host)
+        run: |
+          if command -v rocm-smi >/dev/null 2>&1; then
+            rocm-smi || true
+          else
+            echo "rocm-smi not found on runner"
+          fi
+
+      - name: Determine GPU architecture via rocminfo
+        id: gpu-arch
+        run: |
+          set -e
+          DEFAULT_ARCH="${{ inputs.gpu_arch }}"
+          if command -v rocminfo >/dev/null 2>&1; then
+            ARCH=$(rocminfo | grep -m 1 -oP 'gfx[0-9a-fA-F]+') || true
+          else
+            echo "rocminfo not found on runner, falling back to default arch"
+            ARCH=""
+          fi
+
+          if [ -z "$ARCH" ]; then
+            echo "Could not detect GPU arch automatically, falling back to $DEFAULT_ARCH"
+            ARCH="$DEFAULT_ARCH"
+          fi
+
+          echo "Using GPU arch: $ARCH"
+          echo "arch=$ARCH" >> "$GITHUB_OUTPUT"
+
+      - name: Resolve TransformerEngine commit for branch
+        id: te-ref
+        env:
+          TE_BRANCH: ${{ inputs.te_branch }}
+        run: |
+          set -e
+          BRANCH="${TE_BRANCH:-release_v2.10_rocm}"
+          SHA=$(git ls-remote https://github.com/ROCm/TransformerEngine.git "refs/heads/$BRANCH" | cut -f1)
+          if [ -z "$SHA" ]; then
+            echo "::error::Failed to resolve TransformerEngine commit for branch $BRANCH"
+            exit 1
+          fi
+          echo "Using TransformerEngine branch $BRANCH commit: $SHA"
+          echo "sha=$SHA" >> "$GITHUB_OUTPUT"
+
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ secrets.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.DOCKERHUB_TOKEN }}
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+
+      - name: Build Docker image (local only, read-only registry cache)
+        uses: docker/build-push-action@v5
+        with:
+          context: .
+          file: ./Dockerfile_rocm.ci
+          push: false
+          load: true
+          tags: |
+            rocm/megatron-lm-private:${{ github.sha }}
+          build-args: |
+            PYTORCH_ROCM_ARCH_OVERRIDE=${{ steps.gpu-arch.outputs.arch }}
+            TE_COMMIT=${{ steps.te-ref.outputs.sha }}
+          cache-from: |
+            type=registry,ref=rocm/megatron-lm-private:cache
+
+      - name: Run benchmarks and generate performance report
+        id: benchmarks
+        continue-on-error: true
+        env:
+          GENERATE_BENCHMARK_CHARTS: ${{ inputs.generate_benchmark_charts && '1' || '0' }}
+          BENCHMARK_PRESET: ${{ inputs.benchmark_preset }}
+          TOTAL_ITERS: ${{ inputs.total_iters }}
+          TRAIN_ITERS: ${{ inputs.total_iters }}
+        run: |
+          set -euxo pipefail
+          mkdir -p output
+          docker run --rm \
+            --network=host \
+            -u root \
+            --device=/dev/kfd \
+            --device=/dev/dri \
+            --group-add "$(getent group video | cut -d: -f3)" \
+            --cap-add=SYS_PTRACE \
+            --security-opt seccomp=unconfined \
+            --security-opt apparmor=unconfined \
+            --shm-size=128G \
+            -e GENERATE_BENCHMARK_CHARTS="${GENERATE_BENCHMARK_CHARTS:-0}" \
+            -e BENCHMARK_PRESET="${BENCHMARK_PRESET}" \
+            -e TOTAL_ITERS="${TOTAL_ITERS:-12}" \
+            -e TRAIN_ITERS="${TRAIN_ITERS:-12}" \
+            -v "${{ github.workspace }}:/workspace/Megatron-LM" \
+            --workdir /workspace/Megatron-LM \
+            --entrypoint /bin/bash \
+            rocm/megatron-lm-private:${{ github.sha }} \
+            -c "bash run_benchmarks.sh" 2>&1 | tee output/benchmark_report.txt
+
+      - name: Display benchmark report
+        if: always()
+        run: |
+          echo "::group::Megatron-LM Benchmark Performance Report"
+          if [ -f output/benchmark_report.txt ]; then
+            cat output/benchmark_report.txt
+          else
+            echo "Benchmark report not found (benchmarks may have been skipped or failed)"
+          fi
+          echo "::endgroup::"
+
+      - name: Generate benchmark report markdown
+        if: always()
+        run: |
+          python3 .github/scripts/benchmark-report-to-markdown.py
+
+      - name: Publish benchmark report to Job Summary
+        if: always()
+        run: |
+          if [ -f output/benchmark_summary.md ]; then
+            cat output/benchmark_summary.md >> $GITHUB_STEP_SUMMARY
+          fi
+
+      - name: Publish benchmark report as Check Run
+        if: always()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            let summary = 'No benchmark results available.';
+            if (fs.existsSync('output/benchmark_summary.md')) {
+              summary = fs.readFileSync('output/benchmark_summary.md', 'utf8');
+            }
+            try {
+              await github.rest.checks.create({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                name: 'Megatron-LM Benchmark Report (manual)',
+                head_sha: context.sha,
+                status: 'completed',
+                conclusion: 'success',
+                output: {
+                  title: 'Benchmark Performance Report',
+                  summary: summary
+                }
+              });
+              core.info('Benchmark report published as Check Run');
+            } catch (error) {
+              core.warning('Could not create Check Run (may lack permission on forked PRs): ' + error.message);
+            }
+
+      - name: Upload benchmark artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: megatron-lm-benchmark-reports-manual
+          path: |
+            output/benchmark_report.txt
+            output/benchmark_report.json
+            output/benchmark_results.tmp
+            output/benchmark_summary.md
+            output/bench_*.log
+            output/memory_footprint.json
+            output/memory_footprint.png
+            output/perf_breakdown.json
+            output/perf_breakdown.png
+            output/perf_breakdown_table.md
+            output/perf_breakdown_non_compute.png
+            output/profiler_trace_summary.txt
+            output/nccl_summary.csv
+            output/tracelens_bench/**
+            output/tracelens/**
+            output/*.xlsx
+          if-no-files-found: ignore
+          retention-days: 7
+
+      - name: Run TE profile CI script
+        if: ${{ inputs.run_te_profile }}
+        continue-on-error: true
+        run: |
+          set -euxo pipefail
+          mkdir -p output
+          docker run --rm \
+            --network=host \
+            -u root \
+            --device=/dev/kfd \
+            --device=/dev/dri \
+            --group-add "$(getent group video | cut -d: -f3)" \
+            --cap-add=SYS_PTRACE \
+            --security-opt seccomp=unconfined \
+            --security-opt apparmor=unconfined \
+            --shm-size=128G \
+            -v "${{ github.workspace }}:/workspace/Megatron-LM" \
+            --workdir /workspace/Megatron-LM \
+            --entrypoint /bin/bash \
+            rocm/megatron-lm-private:${{ github.sha }} \
+            -c "bash run_profile_ci.sh" 2>&1 | tee output/te_profile_ci.log
+
+      - name: Upload TE profile artifacts
+        if: ${{ always() && inputs.run_te_profile }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: megatron-lm-te-profile-manual
+          path: |
+            output/te_profile_run/**
+            output/te_profile_ci.log
+          if-no-files-found: ignore
+          retention-days: 7
+
+      - name: Add TE profile summary
+        if: ${{ always() && inputs.run_te_profile }}
+        run: |
+          if [ -f output/te_profile_ci.log ]; then
+            echo "## TE E2E profile run" >> "$GITHUB_STEP_SUMMARY"
+            echo '```' >> "$GITHUB_STEP_SUMMARY"
+            tail -n 80 output/te_profile_ci.log >> "$GITHUB_STEP_SUMMARY" || true
+            echo '```' >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Remove local Docker image
+        if: always()
+        run: |
+          docker image rm rocm/megatron-lm-private:${{ github.sha }} || true
diff --git a/.github/workflows/megatron-ci.yml b/.github/workflows/megatron-ci.yml
index 94e10c33125..d1e4d5702ca 100644
--- a/.github/workflows/megatron-ci.yml
+++ b/.github/workflows/megatron-ci.yml
@@ -1,28 +1,65 @@
-name: Megatron-LM CI
+# Manual benchmarks / profiling only — does not run unit tests.
+# Single runner job: build image locally (no final image push / no image tar artifacts).
+# Read-only registry layer cache (no cache-to): TE_COMMIT changes won't write new cache blobs from this workflow.
+# Actions → "Megatron-LM benchmarks (manual)" → Run workflow
+
+name: Megatron-LM benchmarks (manual)
 
 on:
-  push:
-    branches:
-      - rocm_dev
-  pull_request:
-    branches:
-      - '**'
   workflow_dispatch:
     inputs:
       gpu_arch:
-        description: 'GPU architecture to use for build/tests (e.g. gfx942)'
+        description: 'GPU architecture for the Docker build (e.g. gfx942) if not detected'
         required: false
         default: 'gfx942'
+      te_branch:
+        description: 'ROCm/TransformerEngine.git branch used to resolve TE_COMMIT build-arg'
+        required: false
+        default: 'release_v2.10_rocm'
+        type: string
+      benchmark_preset:
+        description: 'Named benchmark matrix (see run_benchmarks.sh). LLAMA FSDP = llama_fsdp (70B FSDP+recompute).'
+        required: false
+        default: 'all'
+        type: choice
+        options:
+          - all
+          - llama
+          - deepseek
+          - llama_fsdp
+          - llama_8b
+          - llama_70b
+          - deepseek_v2
+          - deepseek_v3
+      total_iters:
+        description: 'Training iterations per benchmark row (TOTAL_ITERS / TRAIN_ITERS)'
+        required: false
+        default: '12'
+        type: string
+      generate_benchmark_charts:
+        description: 'Emit memory_footprint + perf_breakdown PNG/JSON under output/'
+        required: false
+        default: false
+        type: boolean
+      run_te_profile:
+        description: 'After benchmarks, run TE E2E profiler (run_profile_ci.sh)'
+        required: false
+        default: false
+        type: boolean
 
 concurrency:
-  group: ${{ github.workflow }}-${{ github.ref }}
+  group: megatron-benchmarks-${{ github.workflow }}-${{ github.ref }}
   cancel-in-progress: true
 
 jobs:
-  build_and_test:
-    name: Build and Test on GPU
-    timeout-minutes: 720
+  benchmark:
+    name: Build image and run benchmarks
     runs-on: linux-megatron-mi325-8
+    timeout-minutes: 1000
+    continue-on-error: false
+    permissions:
+      contents: read
+      checks: write
 
     steps:
       - name: Checkout repository
@@ -48,7 +85,7 @@ jobs:
         id: gpu-arch
         run: |
           set -e
-          DEFAULT_ARCH="${{ github.event.inputs.gpu_arch || 'gfx942' }}"
+          DEFAULT_ARCH="${{ inputs.gpu_arch }}"
           if command -v rocminfo >/dev/null 2>&1; then
             ARCH=$(rocminfo | grep -m 1 -oP 'gfx[0-9a-fA-F]+') || true
           else
@@ -66,15 +103,17 @@ jobs:
 
       - name: Resolve TransformerEngine commit for branch
         id: te-ref
+        env:
+          TE_BRANCH: ${{ inputs.te_branch }}
         run: |
           set -e
-          BRANCH=release_v2.10_rocm
+          BRANCH="${TE_BRANCH:-release_v2.10_rocm}"
           SHA=$(git ls-remote https://github.com/ROCm/TransformerEngine.git "refs/heads/$BRANCH" | cut -f1)
           if [ -z "$SHA" ]; then
             echo "::error::Failed to resolve TransformerEngine commit for branch $BRANCH"
             exit 1
           fi
-          echo "Using TransformerEngine commit: $SHA"
+          echo "Using TransformerEngine branch $BRANCH commit: $SHA"
           echo "sha=$SHA" >> "$GITHUB_OUTPUT"
 
       - name: Login to Docker Hub
@@ -86,15 +125,13 @@ jobs:
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
 
-      - name: Build and push Docker image with registry cache
+      - name: Build Docker image (local only, read-only registry cache)
         uses: docker/build-push-action@v5
         with:
           context: .
           file: ./Dockerfile_rocm.ci
-          push: true
-          # Also write image to disk so `docker run` does not need `docker pull`.
-          # On some runners dockerd cannot reach auth.docker.io (timeout); BuildKit push may still work.
-          outputs: type=docker,dest=${{ runner.temp }}/megatron-ci-image.tar
+          push: false
+          load: true
           tags: |
             rocm/megatron-lm-private:${{ github.sha }}
           build-args: |
@@ -102,15 +139,18 @@ jobs:
             TE_COMMIT=${{ steps.te-ref.outputs.sha }}
           cache-from: |
             type=registry,ref=rocm/megatron-lm-private:cache
-          cache-to: |
-            type=registry,ref=rocm/megatron-lm-private:cache,mode=max
 
-      - name: Load image for docker run
-        run: docker load -i "${{ runner.temp }}/megatron-ci-image.tar"
-
-      - name: Run unit tests in container
+      - name: Run benchmarks and generate performance report
+        id: benchmarks
+        continue-on-error: true
+        env:
+          GENERATE_BENCHMARK_CHARTS: ${{ inputs.generate_benchmark_charts && '1' || '0' }}
+          BENCHMARK_PRESET: ${{ inputs.benchmark_preset }}
+          TOTAL_ITERS: ${{ inputs.total_iters }}
+          TRAIN_ITERS: ${{ inputs.total_iters }}
         run: |
           set -euxo pipefail
+          mkdir -p output
           docker run --rm \
             --network=host \
             -u root \
@@ -121,45 +161,136 @@ jobs:
             --security-opt seccomp=unconfined \
             --security-opt apparmor=unconfined \
             --shm-size=128G \
+            -e GENERATE_BENCHMARK_CHARTS="${GENERATE_BENCHMARK_CHARTS:-0}" \
+            -e BENCHMARK_PRESET="${BENCHMARK_PRESET}" \
+            -e TOTAL_ITERS="${TOTAL_ITERS:-12}" \
+            -e TRAIN_ITERS="${TRAIN_ITERS:-12}" \
             -v "${{ github.workspace }}:/workspace/Megatron-LM" \
             --workdir /workspace/Megatron-LM \
-            --name megatron-lm-container \
-            --entrypoint /workspace/Megatron-LM/run_unit_tests.sh \
-            rocm/megatron-lm-private:${{ github.sha }}
+            --entrypoint /bin/bash \
+            rocm/megatron-lm-private:${{ github.sha }} \
+            -c "bash run_benchmarks.sh" 2>&1 | tee output/benchmark_report.txt
 
-      - name: Publish test report
+      - name: Display benchmark report
         if: always()
-        uses: dorny/test-reporter@v1
-        with:
-          name: Megatron-LM unit tests report
-          path: output/junit_report_*.xml
-          reporter: java-junit
-          fail-on-error: false
+        run: |
+          echo "::group::Megatron-LM Benchmark Performance Report"
+          if [ -f output/benchmark_report.txt ]; then
+            cat output/benchmark_report.txt
+          else
+            echo "Benchmark report not found (benchmarks may have been skipped or failed)"
+          fi
+          echo "::endgroup::"
+
+      - name: Generate benchmark report markdown
+        if: always()
+        run: |
+          python3 .github/scripts/benchmark-report-to-markdown.py
 
-      - name: List test reports
+      - name: Publish benchmark report to Job Summary
         if: always()
         run: |
-          echo "Workspace contents:"
-          ls -R
-          if [ -d output ]; then
-            echo "Contents of output/:"
-            ls -R output || true
+          if [ -f output/benchmark_summary.md ]; then
+            cat output/benchmark_summary.md >> $GITHUB_STEP_SUMMARY
           fi
 
-      - name: Upload test artifacts
+      - name: Publish benchmark report as Check Run
         if: always()
+        uses: actions/github-script@v7
+        with:
+          script: |
+            const fs = require('fs');
+            let summary = 'No benchmark results available.';
+            if (fs.existsSync('output/benchmark_summary.md')) {
+              summary = fs.readFileSync('output/benchmark_summary.md', 'utf8');
+            }
+            try {
+              await github.rest.checks.create({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                name: 'Megatron-LM Benchmark Report (manual)',
+                head_sha: context.sha,
+                status: 'completed',
+                conclusion: 'success',
+                output: {
+                  title: 'Benchmark Performance Report',
+                  summary: summary
+                }
+              });
+              core.info('Benchmark report published as Check Run');
+            } catch (error) {
+              core.warning('Could not create Check Run (may lack permission on forked PRs): ' + error.message);
+            }
+
+      - name: Upload benchmark artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: megatron-lm-benchmark-reports-manual
+          path: |
+            output/benchmark_report.txt
+            output/benchmark_report.json
+            output/benchmark_results.tmp
+            output/benchmark_summary.md
+            output/bench_*.log
+            output/memory_footprint.json
+            output/memory_footprint.png
+            output/perf_breakdown.json
+            output/perf_breakdown.png
+            output/perf_breakdown_table.md
+            output/perf_breakdown_non_compute.png
+            output/profiler_trace_summary.txt
+            output/nccl_summary.csv
+            output/tracelens_bench/**
+            output/tracelens/**
+            output/*.xlsx
+          if-no-files-found: ignore
+          retention-days: 7
+
+      - name: Run TE profile CI script
+        if: ${{ inputs.run_te_profile }}
+        continue-on-error: true
+        run: |
+          set -euxo pipefail
+          mkdir -p output
+          docker run --rm \
+            --network=host \
+            -u root \
+            --device=/dev/kfd \
+            --device=/dev/dri \
+            --group-add "$(getent group video | cut -d: -f3)" \
+            --cap-add=SYS_PTRACE \
+            --security-opt seccomp=unconfined \
+            --security-opt apparmor=unconfined \
+            --shm-size=128G \
+            -v "${{ github.workspace }}:/workspace/Megatron-LM" \
+            --workdir /workspace/Megatron-LM \
+            --entrypoint /bin/bash \
+            rocm/megatron-lm-private:${{ github.sha }} \
+            -c "bash run_profile_ci.sh" 2>&1 | tee output/te_profile_ci.log
+
+      - name: Upload TE profile artifacts
+        if: ${{ always() && inputs.run_te_profile }}
         uses: actions/upload-artifact@v4
         with:
-          name: megatron-lm-test-reports
+          name: megatron-lm-te-profile-manual
           path: |
-            output/unified_test_report.csv
-            output/junit_report_*.xml
+            output/te_profile_run/**
+            output/te_profile_ci.log
           if-no-files-found: ignore
           retention-days: 7
 
-      - name: Cleanup Docker image
+      - name: Add TE profile summary
+        if: ${{ always() && inputs.run_te_profile }}
+        run: |
+          if [ -f output/te_profile_ci.log ]; then
+            echo "## TE E2E profile run" >> "$GITHUB_STEP_SUMMARY"
+            echo '```' >> "$GITHUB_STEP_SUMMARY"
+            tail -n 80 output/te_profile_ci.log >> "$GITHUB_STEP_SUMMARY" || true
+            echo '```' >> "$GITHUB_STEP_SUMMARY"
+          fi
+
+      - name: Remove local Docker image
         if: always()
         run: |
           docker image rm rocm/megatron-lm-private:${{ github.sha }} || true
-          rm -f "${{ runner.temp }}/megatron-ci-image.tar"
-
diff --git a/Dockerfile_rocm.ci b/Dockerfile_rocm.ci
index e0af8724165..d88059338f5 100755
--- a/Dockerfile_rocm.ci
+++ b/Dockerfile_rocm.ci
@@ -103,6 +103,10 @@ RUN git clone https://github.com/caaatch22/grouped_gemm.git &&\
     git submodule update --init --recursive &&\
     pip install --no-build-isolation .
 
+# TraceLens: Excel perf reports from PyTorch profiler JSON (openpyxl for .xlsx)
+ARG TRACELENS_REF=main
+RUN pip install --no-cache-dir openpyxl "git+https://github.com/AMD-AGI/TraceLens.git@${TRACELENS_REF}"
+
 WORKDIR $WORKSPACE_DIR
 COPY . Megatron-LM
 WORKDIR $WORKSPACE_DIR/Megatron-LM
diff --git a/examples/deepseek_v2/train_deepseekv2.sh b/examples/deepseek_v2/train_deepseekv2.sh
index d7bc71f32c8..94b5eac8d4a 100644
--- a/examples/deepseek_v2/train_deepseekv2.sh
+++ b/examples/deepseek_v2/train_deepseekv2.sh
@@ -4,7 +4,7 @@
 #
 # See LICENSE for license information.
 #################################################################################
-set -e
+# set -e
 
 TOKENIZER_MODEL="deepseek-ai/DeepSeek-V2-Lite"
 
diff --git a/examples/deepseek_v3/train_deepseekv3.sh b/examples/deepseek_v3/train_deepseekv3.sh
index 491be4b745c..eba11832eda 100644
--- a/examples/deepseek_v3/train_deepseekv3.sh
+++ b/examples/deepseek_v3/train_deepseekv3.sh
@@ -5,7 +5,7 @@
 # See LICENSE for license information.
 #################################################################################
 
-set -e
+# set -e
 
 # exp
 EXPERIMENT="deepseek_v3"
diff --git a/examples/llama/train_llama3.sh b/examples/llama/train_llama3.sh
index a13c68c1357..01cb38b76f3 100755
--- a/examples/llama/train_llama3.sh
+++ b/examples/llama/train_llama3.sh
@@ -42,7 +42,15 @@ EXP_NAME="${EXP_NAME:-perf}"
 TEE_OUTPUT="${TEE_OUTPUT:-1}"
 USE_FLASH_ATTN="${USE_FLASH_ATTN:-0}"
 NO_TRAINING="${NO_TRAINING:-0}" # NO_TRAINING=1: for computing metrics only
-ENABLE_PROFILING="${ENABLE_PROFILING:-0}" #enable pytorch profiling
+ENABLE_PROFILING="${ENABLE_PROFILING:-0}" # enable pytorch profiling
+# Optional: PyTorch profiler schedule uses active = profile_step_end - profile_step_start.
+# Default one *active* step keeps Chrome trace JSON small; widen with PROFILE_STEP_END if needed.
+PROFILE_STEP_START="${PROFILE_STEP_START:-2}"
+PROFILE_STEP_END="${PROFILE_STEP_END:-}"  # empty => one active step: start+1 (see below)
+# Default: Transformer Engine for BF16 (TE_FP8=0). Set USE_TE=0 for local Megatron transformer.
+USE_TE="${USE_TE:-1}"
+# Set to 1 to pass --timing-log-level 2 for richer Megatron timer breakdown in logs (perf charts)
+LOG_DETAILED_TIMERS="${LOG_DETAILED_TIMERS:-0}"
 echo "NO_TRAINING=$NO_TRAINING"
 
 CWD=`pwd`
@@ -295,7 +303,22 @@ else
 fi
 
 if [ "$ENABLE_PROFILING" -eq 1 ]; then
-    EXTRA_ARGS="$EXTRA_ARGS --profile --use-pytorch-profiler --tensorboard-dir $LOG_DIR"
+    if [ -z "$PROFILE_STEP_END" ]; then
+        PROFILE_STEP_END=$((PROFILE_STEP_START + 1))
+    fi
+    # Megatron stops the profiler when iteration == PROFILE_STEP_END; ensure train-iters is enough.
+    NEED_ITERS=$((PROFILE_STEP_END + 1))
+    if [ "$TOTAL_ITERS" -lt "$NEED_ITERS" ]; then
+        echo "ENABLE_PROFILING=1: bumping TOTAL_ITERS from $TOTAL_ITERS to $NEED_ITERS (PROFILE_STEP_END=$PROFILE_STEP_END)"
+        TOTAL_ITERS=$NEED_ITERS
+    fi
+    echo "Profiler window: step_start=$PROFILE_STEP_START step_end=$PROFILE_STEP_END (active_steps=$((PROFILE_STEP_END - PROFILE_STEP_START)), larger window => bigger trace JSON)"
+    EXTRA_ARGS="$EXTRA_ARGS --profile --use-pytorch-profiler --tensorboard-dir $LOG_DIR \
+        --profile-step-start $PROFILE_STEP_START --profile-step-end $PROFILE_STEP_END"
+fi
+
+if [ "$LOG_DETAILED_TIMERS" -eq 1 ]; then
+    EXTRA_ARGS="$EXTRA_ARGS --timing-log-level 2"
 fi
 
 if [ "$USE_FLASH_ATTN" -eq 1 ]; then
@@ -316,10 +339,12 @@ if [ "$MCORE" -eq 1 ]; then
     EXTRA_ARGS="$EXTRA_ARGS --use-mcore-models"
 fi
 
-if [ "$TE_FP8" -eq 1 ]; then
+if [ "$TE_FP8" -eq 1 ] || [ "$USE_TE" -eq 1 ]; then
     EXTRA_ARGS="$EXTRA_ARGS --transformer-impl=transformer_engine \
 "
+fi
 
+if [ "$TE_FP8" -eq 1 ]; then
     if [ "$TE_FP8_RECIPE" == "delayed" ]; then
         EXTRA_ARGS="$EXTRA_ARGS --fp8-recipe=delayed \
             --fp8-format=hybrid \
@@ -343,7 +368,9 @@ if [ "$TE_FP8" -eq 1 ]; then
         echo "$TE_FP8_RECIPE is not supported"
         exit
     fi
+fi
 
+if [ "$TE_FP8" -eq 1 ]; then
     if [ "$FP8_PARAM_GATHER" -eq 1 ]; then
         if [ "$TE_FP8_RECIPE" == "mxfp8" ]  && [ "$FSDP" -eq 1 ]; then
             EXTRA_ARGS="$EXTRA_ARGS --fp8-param-gather"
diff --git a/megatron/training/theoretical_memory_usage.py b/megatron/training/theoretical_memory_usage.py
index 8737015dfa4..dcf97046593 100644
--- a/megatron/training/theoretical_memory_usage.py
+++ b/megatron/training/theoretical_memory_usage.py
@@ -4,6 +4,8 @@
 
 
 import math
+from types import SimpleNamespace
+
 from .utils import print_rank_0
 
 NUM_BYTES_IN_MEGABYTE = 1024 * 1024
@@ -186,6 +188,160 @@ def compute_weight_and_optimizer_memory(args, verbose=False):
     return weight_and_optimizer_memory
 
 
+def compute_theoretical_memory_breakdown_megabytes(
+    args, num_microbatches=None, verbose=False, use_selective_sp_activation_path=None
+):
+    """Per-GPU theoretical memory (most loaded shard) split for footprint charts.
+
+    Splits weight+optimizer total into param (bf16), grad (bf16), and optimizer
+    using the same total as :func:`compute_weight_and_optimizer_memory` (param+grad
+    = 4 bytes per parameter on shard; remainder attributed to optimizer states).
+
+    Returns:
+        dict with keys: param_megabytes, grad_megabytes, optimizer_megabytes,
+        activation_megabytes, total_megabytes, and *_gigabytes float fields.
+    """
+    if args.is_hybrid_model:
+        print_rank_0("Theoretical memory breakdown not yet supported for hybrid Mamba-Transformer models.")
+        return None
+
+    n = _resolve_num_parameters_on_most_loaded_shard(args)
+    num_bytes_per_param = 18 if not args.use_distributed_optimizer else 6 + (12 / args.data_parallel_size)
+    w_opt_bytes = n * num_bytes_per_param
+    param_bytes = 2 * n
+    grad_bytes = 2 * n
+    optim_bytes = max(0.0, w_opt_bytes - param_bytes - grad_bytes)
+    act_path = use_selective_sp_activation_path
+    if act_path is None:
+        act_path = args.sequence_parallel and args.recompute_granularity == "selective"
+    if act_path:
+        activation_bytes = compute_activation_memory(args, num_microbatches=num_microbatches, verbose=verbose)
+    else:
+        activation_bytes = compute_activation_memory_without_sp(
+            args, num_microbatches=num_microbatches, verbose=verbose
+        )
+    to_mb = lambda b: b / NUM_BYTES_IN_MEGABYTE
+    out = {
+        "param_megabytes": to_mb(param_bytes),
+        "grad_megabytes": to_mb(grad_bytes),
+        "optimizer_megabytes": to_mb(optim_bytes),
+        "activation_megabytes": to_mb(activation_bytes),
+        "total_megabytes": to_mb(w_opt_bytes + activation_bytes),
+        "param_gigabytes": to_mb(param_bytes) / 1024.0,
+        "grad_gigabytes": to_mb(grad_bytes) / 1024.0,
+        "optimizer_gigabytes": to_mb(optim_bytes) / 1024.0,
+        "activation_gigabytes": to_mb(activation_bytes) / 1024.0,
+        "total_gigabytes": to_mb(w_opt_bytes + activation_bytes) / 1024.0,
+    }
+    return out
+
+
+def _resolve_num_parameters_on_most_loaded_shard(args):
+    """Duplicate parameter-count-on-shard logic from compute_weight_and_optimizer_memory (dense/MoE)."""
+    query_projection_size = args.kv_channels * args.num_attention_heads
+    query_projection_to_hidden_size_ratio = query_projection_size / args.hidden_size
+    if not args.group_query_attention:
+        args.num_query_groups = args.num_attention_heads
+    num_experts = 1 if args.num_experts is None else args.num_experts
+    gated_linear_multiplier = 3 / 2 if args.swiglu else 1
+    shared_expert_ffn_hidden_size = (
+        0
+        if args.moe_shared_expert_intermediate_size is None
+        else args.moe_shared_expert_intermediate_size
+    )
+    if args.num_experts is not None:
+        if isinstance(args.moe_layer_freq, int):
+            moe_layer_pattern = [
+                1 if (i % args.moe_layer_freq == 0) else 0 for i in range(args.num_layers)
+            ]
+        elif isinstance(args.moe_layer_freq, list):
+            moe_layer_pattern = args.moe_layer_freq
+            assert len(moe_layer_pattern) == args.num_layers, (
+                f"Invalid length of moe_layer_pattern: {len(moe_layer_pattern)}, "
+                f"expected {args.num_layers}, "
+                f"current moe layer pattern: {args.moe_layer_freq}"
+            )
+        else:
+            raise ValueError(f"Invalid moe_layer_freq type: {type(args.moe_layer_freq)}")
+        num_dense_layers = args.num_layers - sum(moe_layer_pattern)
+        num_moe_layers = sum(moe_layer_pattern)
+        moe_ffn_hidden_size = args.moe_ffn_hidden_size
+    else:
+        moe_layer_pattern = [0] * args.num_layers
+        num_dense_layers = args.num_layers
+        num_moe_layers = 0
+        moe_ffn_hidden_size = 0
+    if args.mtp_num_layers is not None:
+        mtp_layer_is_moe = moe_layer_pattern[-1]
+        mtp_num_moe_layers = mtp_layer_is_moe * args.mtp_num_layers
+        mtp_num_dense_layers = (1 - mtp_layer_is_moe) * args.mtp_num_layers
+    else:
+        mtp_num_moe_layers = 0
+        mtp_num_dense_layers = 0
+    if args.multi_latent_attention:
+        assert not args.group_query_attention
+        if args.q_lora_rank is None:
+            q_term = args.hidden_size * args.num_attention_heads * (args.qk_head_dim + args.qk_pos_emb_head_dim)
+        else:
+            q_term = args.q_lora_rank * (
+                args.hidden_size
+                + args.num_attention_heads * (args.qk_head_dim + args.qk_pos_emb_head_dim)
+                + 1
+            )
+        self_attn_term = (
+            q_term
+            + args.kv_lora_rank
+            * (args.hidden_size + args.num_attention_heads * (args.qk_head_dim + args.v_head_dim) + 1)
+            + args.hidden_size * args.qk_pos_emb_head_dim
+            + (args.num_attention_heads * args.v_head_dim) * args.hidden_size
+        )
+    else:
+        self_attn_term = (
+            2
+            * args.hidden_size
+            * args.hidden_size
+            * (
+                (1 + (args.num_query_groups / args.num_attention_heads))
+                * query_projection_to_hidden_size_ratio
+            )
+        )
+    num_parameters_in_transformer_layer_dense = (
+        2
+        * args.hidden_size
+        * ((args.ffn_hidden_size * gated_linear_multiplier) + 2)
+        + self_attn_term
+    )
+    num_parameters_in_transformer_layer_moe = (
+        2
+        * args.hidden_size
+        * ((moe_ffn_hidden_size * num_experts * gated_linear_multiplier) + (shared_expert_ffn_hidden_size * gated_linear_multiplier) + 2)
+        + self_attn_term
+    )
+    embedding_size = args.hidden_size * args.padded_vocab_size
+    final_layernorm = 2 * args.hidden_size
+    if args.untie_embeddings_and_output_weights:
+        num_parameters_in_embedding_layers = 2 * embedding_size
+    else:
+        num_parameters_in_embedding_layers = embedding_size
+    num_parameters_in_transformer_block = (
+        num_parameters_in_transformer_layer_dense * num_dense_layers
+        + num_parameters_in_transformer_layer_moe * num_moe_layers
+        + final_layernorm
+    )
+    num_parameters_in_mtp_block = (
+        num_parameters_in_transformer_layer_dense * mtp_num_dense_layers
+        + num_parameters_in_transformer_layer_moe * mtp_num_moe_layers
+    )
+    num_parameters_on_most_loaded_model_shard = (
+        (num_parameters_in_transformer_block / args.pipeline_model_parallel_size)
+        + num_parameters_in_mtp_block
+        + embedding_size
+    ) / args.tensor_model_parallel_size
+    if args.untie_embeddings_and_output_weights and args.pipeline_model_parallel_size == 1:
+        num_parameters_on_most_loaded_model_shard += embedding_size / args.tensor_model_parallel_size
+    return num_parameters_on_most_loaded_model_shard
+
+
 def compute_activation_memory(args, num_microbatches, verbose=False):
     # Using formula in Table 2 of https://arxiv.org/pdf/2205.05198.pdf.
     # We are trying to compute the maximum activation footprint, so all calculations in this
@@ -365,3 +521,82 @@ def report_theoretical_memory(args, num_microbatches=None, verbose=False):
     )
 
     return weight_and_optimizer_memory, activation_memory, total_memory
+
+
+def build_llama_dense_args_for_theoretical_memory(
+    *,
+    model_size: int,
+    tensor_parallel_size: int,
+    pipeline_model_parallel_size: int,
+    context_parallel_size: int,
+    seq_length: int,
+    micro_batch_size: int,
+    global_batch_size: int,
+    world_size: int,
+    use_distributed_optimizer: bool = True,
+    sequence_parallel: bool = True,
+):
+    """Build a minimal Namespace for Llama 3 dense (8B/70B-style) theoretical memory APIs.
+
+    Used by benchmark tooling when no full Megatron argparse Namespace exists.
+    Memory math is unchanged: callers pass the result to
+    :func:`compute_theoretical_memory_breakdown_megabytes` or
+    :func:`report_theoretical_memory`.
+
+    ``world_size`` should match the training job (nodes * GPUs per node).
+    """
+    if model_size == 8:
+        hidden_size = 4096
+        ffn_hidden_size = 14336
+        num_layers = 32
+        num_attention_heads = 32
+        num_query_groups = 8
+    elif model_size == 70:
+        hidden_size = 8192
+        ffn_hidden_size = 28672
+        num_layers = 80
+        num_attention_heads = 64
+        num_query_groups = 8
+    else:
+        raise ValueError(f"Unsupported model_size={model_size} for Llama preset")
+
+    kv_channels = hidden_size // num_attention_heads
+    cp = max(1, context_parallel_size)
+    tp = max(1, tensor_parallel_size)
+    pp = max(1, pipeline_model_parallel_size)
+    dp = max(1, world_size // (tp * pp * cp))
+
+    return SimpleNamespace(
+        is_hybrid_model=False,
+        num_layers=num_layers,
+        hidden_size=hidden_size,
+        ffn_hidden_size=ffn_hidden_size,
+        num_attention_heads=num_attention_heads,
+        num_query_groups=num_query_groups,
+        group_query_attention=True,
+        kv_channels=kv_channels,
+        seq_length=seq_length,
+        micro_batch_size=micro_batch_size,
+        tensor_model_parallel_size=tp,
+        pipeline_model_parallel_size=pp,
+        context_parallel_size=cp,
+        padded_vocab_size=128256,
+        untie_embeddings_and_output_weights=True,
+        swiglu=True,
+        use_distributed_optimizer=use_distributed_optimizer,
+        data_parallel_size=dp,
+        sequence_parallel=sequence_parallel,
+        recompute_granularity=None,
+        multi_latent_attention=False,
+        num_experts=None,
+        moe_layer_freq=None,
+        moe_ffn_hidden_size=None,
+        moe_shared_expert_intermediate_size=None,
+        mtp_num_layers=None,
+        virtual_pipeline_model_parallel_size=None,
+        q_lora_rank=None,
+        qk_head_dim=None,
+        qk_pos_emb_head_dim=None,
+        kv_lora_rank=None,
+        v_head_dim=None,
+    )
diff --git a/run_benchmarks.sh b/run_benchmarks.sh
new file mode 100755
index 00000000000..0bfaae29bc9
--- /dev/null
+++ b/run_benchmarks.sh
@@ -0,0 +1,266 @@
+#!/bin/bash
+###############################################################################
+# Run Megatron-LM benchmarks and generate a performance report.
+# Single entry for CI/local: Llama uses Transformer Engine by default
+# (examples/llama/train_llama3.sh, USE_TE=1); use USE_TE=0 on a row for local baseline.
+###############################################################################
+set -e
+
+BENCHMARK_REPORT="${BENCHMARK_REPORT:-output/benchmark_report.txt}"
+BENCHMARK_JSON="${BENCHMARK_JSON:-output/benchmark_report.json}"
+RESULTS_FILE="${RESULTS_FILE:-output/benchmark_results.tmp}"
+mkdir -p output
+
+# Run benchmark and append metrics to RESULTS_FILE (caller sets RESULTS_FILE)
+run_and_collect() {
+    local name="$1"
+    shift
+    local log_file="output/bench_${name}.log"
+    echo "=========================================="
+    echo "Running benchmark: $name"
+    echo "Command: $*"
+    echo "=========================================="
+
+    if eval "$@" 2>&1 | tee "$log_file"; then
+        echo "[PASS] $name"
+    else
+        echo "[FAIL] $name (exit code: $?)"
+    fi
+
+    # Parse metrics from log file
+    local throughput elapsed_ms tokens_per_gpu_s mem_gb
+    throughput=$(grep -E "throughput per GPU:" "$log_file" 2>/dev/null | tail -1 | sed -E 's/.*: ([0-9\.]+).*/\1/' || echo "N/A")
+    elapsed_ms=$(grep -E "elapsed time per iteration:" "$log_file" 2>/dev/null | tail -1 | sed -E 's/.*: ([0-9\.]+).*/\1/' || echo "N/A")
+    tokens_per_gpu_s=$(grep -E "tokens/GPU/s:" "$log_file" 2>/dev/null | tail -1 | sed -E 's/.*: ([0-9\.]+).*/\1/' || echo "N/A")
+    mem_gb=$(grep -E "mem usages:" "$log_file" 2>/dev/null | tail -1 | sed -E 's/.*: ([0-9\.]+).*/\1/' || echo "N/A")
+
+    echo "$name|$throughput|$elapsed_ms|$tokens_per_gpu_s|$mem_gb" >> "$RESULTS_FILE"
+    echo ""
+}
+
+# Main
+# BENCHMARK_PRESET (optional): fine-grained row selection; when set, overrides BENCHMARK_SUITE.
+#   all | full          → full matrix
+#   llama | llama_all   → all Llama rows
+#   deepseek            → DeepSeek v2 + v3
+#   llama_fsdp          → Llama 70B PyTorch FSDP + recompute only
+#   llama_8b            → Llama 8B FP8 + 8B BF16
+#   llama_70b           → Llama 70B rows (excludes 8B): TE BF16, FSDP, TP8, TP4/PP2, local
+#   deepseek_v2 | deepseek_v3 → single DeepSeek run
+#
+# BENCHMARK_SUITE (optional, used only when BENCHMARK_PRESET is unset):
+#   unset, empty, "full", or "all" → full matrix (default).
+#   "llama" → Llama rows only. "deepseek" → DeepSeek v2/v3 only. "llama,deepseek" → both groups.
+echo "=============================================="
+echo "     Megatron-LM Benchmark Performance Report"
+echo "     $(date -u +"%Y-%m-%d %H:%M:%S UTC")"
+echo "=============================================="
+echo ""
+
+L8_FP8=0
+L8_BF16=0
+L70_TE=0
+L70_FSDP=0
+L70_TP8=0
+L70_PP=0
+L70_LOC=0
+D2=0
+D3=0
+
+_PRESET=$(echo "${BENCHMARK_PRESET:-}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')
+
+if [ -n "${_PRESET}" ]; then
+  case "${_PRESET}" in
+    all|full)
+      L8_FP8=1 L8_BF16=1 L70_TE=1 L70_FSDP=1 L70_TP8=1 L70_PP=1 L70_LOC=1 D2=1 D3=1
+      ;;
+    llama|llama_all)
+      L8_FP8=1 L8_BF16=1 L70_TE=1 L70_FSDP=1 L70_TP8=1 L70_PP=1 L70_LOC=1
+      ;;
+    deepseek)
+      D2=1 D3=1
+      ;;
+    llama_fsdp)
+      L70_FSDP=1
+      ;;
+    llama_8b)
+      L8_FP8=1 L8_BF16=1
+      ;;
+    llama_70b)
+      L70_TE=1 L70_FSDP=1 L70_TP8=1 L70_PP=1 L70_LOC=1
+      ;;
+    deepseek_v2)
+      D2=1
+      ;;
+    deepseek_v3)
+      D3=1
+      ;;
+    *)
+      echo "Unknown BENCHMARK_PRESET '${BENCHMARK_PRESET}' (see run_benchmarks.sh header for valid values)"
+      exit 1
+      ;;
+  esac
+  echo "BENCHMARK_PRESET=${BENCHMARK_PRESET} (row flags: 8b_fp8=$L8_FP8 8b_bf16=$L8_BF16 70_te=$L70_TE 70_fsdp=$L70_FSDP 70_tp8=$L70_TP8 70_pp=$L70_PP 70_loc=$L70_LOC deepseek_v2=$D2 deepseek_v3=$D3)"
+else
+  _SUITE_RAW="${BENCHMARK_SUITE:-}"
+  _SUITE=$(echo "${_SUITE_RAW}" | tr '[:upper:]' '[:lower:]' | tr -d '[:space:]')
+  RUN_LLAMA=1
+  RUN_DEEPSEEK=1
+  if [ -n "${_SUITE}" ] && [ "${_SUITE}" != "full" ] && [ "${_SUITE}" != "all" ]; then
+    RUN_LLAMA=0
+    RUN_DEEPSEEK=0
+    _LINE=$(echo "${BENCHMARK_SUITE}" | tr -d '[:space:]' | tr '[:upper:]' '[:lower:]')
+    IFS=',' read -r -a _PARTS <<< "$_LINE"
+    for _p in "${_PARTS[@]}"; do
+      case "$_p" in
+        llama) RUN_LLAMA=1 ;;
+        deepseek) RUN_DEEPSEEK=1 ;;
+        *)
+          echo "Unknown BENCHMARK_SUITE segment '${_p}' (use full, llama, deepseek, or llama,deepseek)"
+          exit 1
+          ;;
+      esac
+    done
+  fi
+  if [ "$RUN_LLAMA" -eq 1 ]; then
+    L8_FP8=1 L8_BF16=1 L70_TE=1 L70_FSDP=1 L70_TP8=1 L70_PP=1 L70_LOC=1
+  fi
+  if [ "$RUN_DEEPSEEK" -eq 1 ]; then
+    D2=1 D3=1
+  fi
+  echo "BENCHMARK_SUITE=${_SUITE_RAW:-<empty>=full} RUN_LLAMA=$RUN_LLAMA RUN_DEEPSEEK=$RUN_DEEPSEEK"
+fi
+echo ""
+
+# Use TOTAL_ITERS for short runs (Llama defaults to 12)
+export TOTAL_ITERS="${TOTAL_ITERS:-12}"
+export TRAIN_ITERS="${TRAIN_ITERS:-12}"
+export TEE_OUTPUT=1
+# Richer timer lines for tools/gen_perf_breakdown_charts.py (optional)
+if [ "${GENERATE_BENCHMARK_CHARTS:-0}" = "1" ]; then
+  export LOG_DETAILED_TIMERS=1
+fi
+
+# Initialize results file with header
+echo "Benchmark|Throughput (TFLOP/s/GPU)|Elapsed (ms/iter)|Tokens/GPU/s|Mem (GB)" > "$RESULTS_FILE"
+echo "---------|------------------------|-----------------|-------------|--------" >> "$RESULTS_FILE"
+
+# Llama3 benchmarks (train_llama3.sh defaults to USE_TE=1 / --transformer-impl=transformer_engine for BF16)
+if [ "$L8_FP8" -eq 1 ]; then
+run_and_collect "llama3_8B_TP1_CP1_FP8" \
+    "MBS=1 BS=128 SEQ_LENGTH=8192 TP=1 CP=1 MODEL_SIZE=8 TE_FP8=1 bash examples/llama/train_llama3.sh" || true
+fi
+if [ "$L8_BF16" -eq 1 ]; then
+run_and_collect "llama3_8B_TP1_CP1_BF16" \
+    "MBS=1 BS=128 SEQ_LENGTH=8192 TP=1 CP=1 MODEL_SIZE=8 TE_FP8=0 bash examples/llama/train_llama3.sh" || true
+fi
+if [ "$L70_TE" -eq 1 ]; then
+run_and_collect "llama3_70B_TP8_TE_BF16" \
+    "MBS=1 BS=8 SEQ_LENGTH=8192 TP=8 TE_FP8=0 bash examples/llama/train_llama3.sh" || true
+fi
+if [ "$L70_FSDP" -eq 1 ]; then
+run_and_collect "llama3_70B_PYTORCH_FSDP_RECOMPUTE" \
+    "MBS=1 BS=8 FSDP=1 TP=1 TE_FP8=0 SEQ_LENGTH=8192 RECOMPUTE=1 bash examples/llama/train_llama3.sh" || true
+fi
+if [ "$L70_TP8" -eq 1 ]; then
+run_and_collect "llama3_70B_TP8" \
+    "MBS=1 BS=8 TP=8 TE_FP8=0 SEQ_LENGTH=8192 bash examples/llama/train_llama3.sh" || true
+fi
+if [ "$L70_PP" -eq 1 ]; then
+run_and_collect "llama3_70B_TP4_PP2" \
+    "MBS=1 BS=8 TP=4 PP=2 TE_FP8=0 SEQ_LENGTH=8192 bash examples/llama/train_llama3.sh" || true
+fi
+if [ "$L70_LOC" -eq 1 ]; then
+# Optional non-TE baseline (local transformer impl) for A/B vs Transformer Engine
+run_and_collect "llama3_70B_TP8_local" \
+    "USE_TE=0 TE_FP8=0 MBS=1 BS=8 SEQ_LENGTH=8192 TP=8 bash examples/llama/train_llama3.sh" || true
+fi
+
+# DeepSeek benchmarks (use correct paths: deepseek_v2, deepseek_v3)
+if [ "$D2" -eq 1 ]; then
+run_and_collect "deepseek_v2" \
+    "bash examples/deepseek_v2/train_deepseekv2.sh" || true
+fi
+if [ "$D3" -eq 1 ]; then
+run_and_collect "deepseek_v3" \
+    "bash examples/deepseek_v3/train_deepseekv3.sh" || true
+fi
+
+# Print results table
+echo "=============================================="
+echo "              Performance Summary"
+echo "=============================================="
+cat "$RESULTS_FILE"
+echo ""
+echo "Benchmark logs saved to output/bench_*.log"
+echo "=============================================="
+
+# Generate JSON report
+python3 - <<PYEOF 2>/dev/null || true
+import json
+import re
+import os
+
+results_path = os.environ.get('RESULTS_FILE', 'output/benchmark_results.tmp')
+json_path = os.environ.get('BENCHMARK_JSON', 'output/benchmark_report.json')
+
+results = []
+try:
+    with open(results_path) as f:
+        for line in f:
+            m = re.match(r'^([^|]+)\|([^|]*)\|([^|]*)\|([^|]*)\|([^|]*)$', line.strip())
+            if m and m.group(1) not in ('Benchmark', '--------'):
+                results.append({
+                    'benchmark': m.group(1).strip(),
+                    'throughput_tflop_s_per_gpu': m.group(2).strip() if m.group(2) != 'N/A' else None,
+                    'elapsed_ms_per_iter': m.group(3).strip() if m.group(3) != 'N/A' else None,
+                    'tokens_per_gpu_s': m.group(4).strip() if m.group(4) != 'N/A' else None,
+                    'mem_gb': m.group(5).strip() if m.group(5) != 'N/A' else None,
+                })
+    with open(json_path, 'w') as f:
+        json.dump({'benchmarks': results}, f, indent=2)
+    print(f"JSON report saved to {json_path}")
+except Exception as e:
+    print(f"Could not generate JSON: {e}")
+PYEOF
+
+# Optional: theoretical memory + perf breakdown charts for CI artifacts
+if [ "${GENERATE_BENCHMARK_CHARTS:-0}" = "1" ]; then
+  echo "Generating memory footprint and perf breakdown charts..."
+  python3 tools/gen_memory_footprint_charts.py --output-json output/memory_footprint.json --output-png output/memory_footprint.png || true
+  shopt -s nullglob
+  mapfile -t BENCH_LOGS < <(ls -1 output/bench_*.log 2>/dev/null || true)
+  if [ "${#BENCH_LOGS[@]}" -gt 0 ]; then
+    python3 tools/gen_perf_breakdown_charts.py "${BENCH_LOGS[@]}" \
+      --output-json output/perf_breakdown.json --output-png output/perf_breakdown.png || true
+  fi
+  shopt -u nullglob
+fi
+
+# When a PyTorch Chrome trace exists under output/, generate TraceLens Excel + text summary (+ NCCL CSV if multi-rank traces).
+TRACE_JSON=""
+while IFS= read -r f; do
+  TRACE_JSON="$f"
+  break
+done < <(find output -type f -path '*/tb/*.json' 2>/dev/null || true)
+if [ -z "$TRACE_JSON" ]; then
+  while IFS= read -r f; do
+    case "$f" in
+      *benchmark_report.json) continue ;;
+    esac
+    if head -c 800 "$f" 2>/dev/null | grep -q '"traceEvents"'; then
+      TRACE_JSON="$f"
+      break
+    fi
+  done < <(find output -type f -name '*.json' 2>/dev/null || true)
+fi
+if [ -n "$TRACE_JSON" ]; then
+  echo "Chrome profiler trace found: $TRACE_JSON — running TraceLens + summarize_profiler_trace (+ optional NcclAnalyser)"
+  export TRACE_JSON
+  export TRACELENS_OUT_DIR="${TRACELENS_OUT_DIR:-output/tracelens_bench}"
+  bash tools/run_tracelens_report.sh || echo "TraceLens step failed (see ${TRACELENS_OUT_DIR:-output/tracelens_bench}/tracelens.log)"
+  python3 tools/summarize_profiler_trace.py "$TRACE_JSON" -o output/profiler_trace_summary.txt || true
+  python3 tools/run_nccl_analyser_if_multirank.py --search-dir output --output-csv output/nccl_summary.csv || true
+else
+  echo "No Chrome trace JSON under output/ (enable profiling / tensorboard trace export to get TraceLens artifacts)."
+fi
diff --git a/run_profile_ci.sh b/run_profile_ci.sh
new file mode 100755
index 00000000000..1d64536dd9d
--- /dev/null
+++ b/run_profile_ci.sh
@@ -0,0 +1,55 @@
+#!/bin/bash
+###############################################################################
+# Short profiled TE E2E run for CI: PyTorch profiler + TraceLens Excel + text summary.
+# Usage: bash run_profile_ci.sh
+# Env:
+#   TRACE_DIR - output root (default: output/te_profile_run)
+# Full NCCL skew/bw tables need one trace per rank; otherwise see gpu_timeline in Excel.
+###############################################################################
+set -euo pipefail
+
+cd "$(dirname "$0")"
+ROOT="$(pwd)"
+TRACE_DIR="${TRACE_DIR:-${ROOT}/output/te_profile_run}"
+mkdir -p "$TRACE_DIR"
+
+export ENABLE_PROFILING=1
+# train_llama3.sh defaults USE_TE=1; override with USE_TE=0 if needed
+export TE_FP8="${TE_FP8:-0}"
+# Short run: profiling defaults to a single *active* step (small trace). Override with PROFILE_TRAIN_ITERS.
+export TOTAL_ITERS="${PROFILE_TRAIN_ITERS:-6}"
+export LOG_DETAILED_TIMERS=1
+export MBS=1
+export BS=64
+export SEQ_LENGTH=4096
+export TP=1
+export CP=1
+export MODEL_SIZE=8
+export LOG_DIR="${TRACE_DIR}/tb"
+
+echo "Running profiled training; logs under $TRACE_DIR"
+bash examples/llama/train_llama3.sh 2>&1 | tee "${TRACE_DIR}/train.log"
+
+# Find most recent json trace under LOG_DIR (PyTorch tensorboard_trace_handler)
+TRACE_JSON=$(find "$LOG_DIR" -name "*.json" -type f 2>/dev/null | head -1 || true)
+if [ -n "$TRACE_JSON" ]; then
+  echo "Found profiler trace: $TRACE_JSON"
+  echo "$TRACE_JSON" > "${TRACE_DIR}/trace_path.txt"
+  export TRACE_JSON
+  export TRACELENS_OUT_DIR="${TRACE_DIR}/tracelens"
+  bash tools/run_tracelens_report.sh || echo "TraceLens step failed (install AMD-AGI/TraceLens + openpyxl)"
+  python3 tools/summarize_profiler_trace.py "$TRACE_JSON" -o "${TRACE_DIR}/profiler_trace_summary.txt" || true
+  python3 tools/run_nccl_analyser_if_multirank.py --search-dir "$TRACE_DIR" --output-csv "${TRACE_DIR}/nccl_summary.csv" || true
+else
+  echo "No exported Chrome trace JSON found under $LOG_DIR; TensorBoard may use a different layout."
+fi
+
+# Perf breakdown from Megatron timer lines in the training log
+if command -v python3 >/dev/null; then
+  python3 tools/gen_perf_breakdown_charts.py "${TRACE_DIR}/train.log" \
+    --output-json "${TRACE_DIR}/perf_breakdown.json" \
+    --output-png "${TRACE_DIR}/perf_breakdown.png" \
+    --label profile_te_llama3_8b || true
+fi
+
+echo "Profile artifacts in $TRACE_DIR"
diff --git a/tools/gen_memory_footprint_charts.py b/tools/gen_memory_footprint_charts.py
new file mode 100755
index 00000000000..108b283844a
--- /dev/null
+++ b/tools/gen_memory_footprint_charts.py
@@ -0,0 +1,138 @@
+#!/usr/bin/env python3
+# Copyright (c) 2026, Advanced Micro Devices, Inc.
+"""Theoretical per-GPU memory stacked bars (params / grads / optimizer / activations)."""
+
+from __future__ import annotations
+
+import argparse
+import json
+import sys
+from pathlib import Path
+
+# Repo root on path
+_REPO = Path(__file__).resolve().parents[1]
+if str(_REPO) not in sys.path:
+    sys.path.insert(0, str(_REPO))
+
+from megatron.training.theoretical_memory_usage import (  # noqa: E402
+    build_llama_dense_args_for_theoretical_memory,
+    compute_theoretical_memory_breakdown_megabytes,
+)
+
+
+def _series_from_benchmark_matrix() -> list:
+    """Preset sweep roughly matching run_benchmarks.sh Llama rows."""
+    rows = []
+    # name, model_size, tp, pp, cp, seq, mbs, bs, world_hint (TP*PP*CP implied by tp*pp for single node)
+    specs = [
+        ("llama3_8B_TP1_CP1_FP8", 8, 1, 1, 1, 8192, 1, 128, 1),
+        ("llama3_8B_TP1_CP1_BF16", 8, 1, 1, 1, 8192, 1, 128, 1),
+        ("llama3_70B_TP8", 70, 8, 1, 1, 8192, 1, 8, 8),
+        ("llama3_70B_TP4_PP2", 70, 4, 2, 1, 8192, 1, 8, 8),
+        ("llama3_70B_TP8_local", 70, 8, 1, 1, 8192, 1, 8, 8),
+    ]
+    for name, ms, tp, pp, cp, sl, mbs, bs, ws in specs:
+        args = build_llama_dense_args_for_theoretical_memory(
+            model_size=ms,
+            tensor_parallel_size=tp,
+            pipeline_model_parallel_size=pp,
+            context_parallel_size=cp,
+            seq_length=sl,
+            micro_batch_size=mbs,
+            global_batch_size=bs,
+            world_size=ws,
+        )
+        br = compute_theoretical_memory_breakdown_megabytes(args, num_microbatches=None, verbose=False)
+        if br is None:
+            continue
+        rows.append(
+            {
+                "label": name,
+                "spec": {
+                    "model_size": ms,
+                    "tp": tp,
+                    "pp": pp,
+                    "cp": cp,
+                    "seq_length": sl,
+                    "micro_batch_size": mbs,
+                    "world_size": ws,
+                },
+                "gigabytes": {k: round(v, 3) for k, v in br.items() if k.endswith("_gigabytes")},
+            }
+        )
+    return rows
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("--output-json", default="output/memory_footprint.json")
+    ap.add_argument("--output-png", default="output/memory_footprint.png")
+    ap.add_argument(
+        "--input-json",
+        help="Optional JSON: { 'series': [ { 'label', 'model_size', 'tp', 'pp', 'cp', 'seq_length', 'micro_batch_size', 'global_batch_size', 'world_size' } ] }",
+    )
+    args = ap.parse_args()
+
+    if args.input_json:
+        data = json.loads(Path(args.input_json).read_text())
+        series = []
+        for ent in data.get("series", []):
+            a = build_llama_dense_args_for_theoretical_memory(
+                model_size=int(ent["model_size"]),
+                tensor_parallel_size=int(ent.get("tp", 1)),
+                pipeline_model_parallel_size=int(ent.get("pp", 1)),
+                context_parallel_size=int(ent.get("cp", 1)),
+                seq_length=int(ent["seq_length"]),
+                micro_batch_size=int(ent["micro_batch_size"]),
+                global_batch_size=int(ent.get("global_batch_size", 8)),
+                world_size=int(ent["world_size"]),
+            )
+            br = compute_theoretical_memory_breakdown_megabytes(a, num_microbatches=None)
+            series.append(
+                {
+                    "label": ent["label"],
+                    "spec": ent,
+                    "gigabytes": {k: round(v, 3) for k, v in br.items() if k.endswith("_gigabytes")},
+                }
+            )
+    else:
+        series = _series_from_benchmark_matrix()
+
+    out_dir = Path(args.output_json).parent
+    out_dir.mkdir(parents=True, exist_ok=True)
+    Path(args.output_json).write_text(json.dumps({"series": series}, indent=2))
+
+    try:
+        import matplotlib
+
+        matplotlib.use("Agg")
+        import matplotlib.pyplot as plt
+        import numpy as np
+    except ImportError:
+        print("matplotlib not installed; JSON only.")
+        return
+
+    labels = [s["label"] for s in series]
+    pg = [s["gigabytes"].get("param_gigabytes", 0) for s in series]
+    gg = [s["gigabytes"].get("grad_gigabytes", 0) for s in series]
+    og = [s["gigabytes"].get("optimizer_gigabytes", 0) for s in series]
+    ag = [s["gigabytes"].get("activation_gigabytes", 0) for s in series]
+    x = np.arange(len(labels))
+    fig, ax = plt.subplots(figsize=(max(8, len(labels) * 1.1), 5))
+    ax.bar(x, pg, label="param (bf16)")
+    ax.bar(x, gg, bottom=np.array(pg), label="main grad (bf16)")
+    ax.bar(x, og, bottom=np.array(pg) + np.array(gg), label="optimizer states")
+    ax.bar(x, ag, bottom=np.array(pg) + np.array(gg) + np.array(og), label="activation (bf16)")
+    ax.set_xticks(x)
+    ax.set_xticklabels(labels, rotation=25, ha="right")
+    ax.set_ylabel("Memory (GB, theoretical per GPU)")
+    ax.set_title("Theoretical memory footprint breakdown")
+    ax.legend(loc="upper center", bbox_to_anchor=(0.5, -0.2), ncol=2)
+    fig.tight_layout()
+    fig.savefig(args.output_png, dpi=120)
+    plt.close(fig)
+    print(f"Wrote {args.output_json} and {args.output_png}")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/gen_perf_breakdown_charts.py b/tools/gen_perf_breakdown_charts.py
new file mode 100755
index 00000000000..ab19c9060e7
--- /dev/null
+++ b/tools/gen_perf_breakdown_charts.py
@@ -0,0 +1,383 @@
+#!/usr/bin/env python3
+# Copyright (c) 2026, Advanced Micro Devices, Inc.
+"""Parse Megatron training logs for timer blocks; emit stacked % bar chart + JSON.
+
+Timer ratios are serial shares of summed categorized times (not GPU overlap).
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+import math
+import os
+import re
+from pathlib import Path
+from typing import Any, Dict, List, Tuple
+
+# Megatron log lines (see megatron/core/timers.py)
+_RE_MAX = re.compile(
+    r"^\s+([\w\-]+)\s*\.+:\s+([\d.]+)\s*$"
+)
+_RE_MINMAX = re.compile(
+    r"^\s+([\w\-]+)\s*\.+:\s+\([\d.]+,\s*([\d.]+)\)\s*$"
+)
+_RE_ELAPSED = re.compile(
+    r"elapsed time per iteration \(ms\):\s*([\d.]+)", re.I
+)
+
+# Bucket mapping (extend for MoE / custom timers)
+COMPUTE_KEYS = {"forward-compute", "backward-compute"}
+SEND_RECV_KEYS = {
+    "forward-send",
+    "backward-send",
+    "forward-recv",
+    "backward-recv",
+    "forward-send-forward-recv",
+    "forward-send-backward-recv",
+    "backward-send-forward-recv",
+    "backward-send-backward-recv",
+    "forward-backward-send-forward-backward-recv",
+}
+COLLECTIVE_KEYS = {
+    "layernorm-grads-all-reduce",
+    "embedding-grads-all-reduce",
+    "all-grads-sync",
+    "params-all-gather",
+}
+OPT_KEYS = {
+    "optimizer",
+    "optimizer-copy-to-main-grad",
+    "optimizer-unscale-and-check-inf",
+    "optimizer-clip-main-grad",
+    "optimizer-count-zeros",
+    "optimizer-inner-step",
+    "optimizer-copy-main-to-model-params",
+}
+DATA_KEYS = {"batch-generator"}
+# Remainder of forward-backward pipeline comms
+OTHER_FB_KEYS = {"forward-backward"}
+
+
+def _parse_timer_blocks(text: str) -> List[Dict[str, float]]:
+    """Extract timer dicts from each 'max time across ranks' section."""
+    blocks: List[Dict[str, float]] = []
+    lines = text.splitlines()
+    i = 0
+    while i < len(lines):
+        line = lines[i]
+        if "max time across ranks (ms):" in line or "(min, max) time across ranks (ms):" in line:
+            i += 1
+            timers: Dict[str, float] = {}
+            while i < len(lines):
+                ln = lines[i]
+                m = _RE_MAX.match(ln) or _RE_MINMAX.match(ln)
+                if m:
+                    timers[m.group(1).strip()] = float(m.group(2))
+                    i += 1
+                    continue
+                break
+            if timers:
+                blocks.append(timers)
+            continue
+        i += 1
+    return blocks
+
+
+def _aggregate_timers(blocks: List[Dict[str, float]]) -> Dict[str, float]:
+    if not blocks:
+        return {}
+    keys = set()
+    for b in blocks:
+        keys |= set(b.keys())
+    out: Dict[str, float] = {}
+    for k in keys:
+        vals = [b[k] for b in blocks if k in b]
+        if vals:
+            out[k] = sum(vals) / len(vals)
+    return out
+
+
+def _bucket_ms(timers: Dict[str, float]) -> Dict[str, float]:
+    buckets = {
+        "compute_ms": 0.0,
+        "comm_send_recv_ms": 0.0,
+        "comm_collective_ms": 0.0,
+        "optimizer_ms": 0.0,
+        "data_ms": 0.0,
+        "forward_backward_total_ms": 0.0,
+        "bubble_other_ms": 0.0,
+    }
+    accounted = set()
+    for k, v in timers.items():
+        if k in COMPUTE_KEYS:
+            buckets["compute_ms"] += v
+            accounted.add(k)
+        elif k in SEND_RECV_KEYS:
+            buckets["comm_send_recv_ms"] += v
+            accounted.add(k)
+        elif k in COLLECTIVE_KEYS:
+            buckets["comm_collective_ms"] += v
+            accounted.add(k)
+        elif k in OPT_KEYS:
+            buckets["optimizer_ms"] += v
+            accounted.add(k)
+        elif k in DATA_KEYS:
+            buckets["data_ms"] += v
+            accounted.add(k)
+        elif k in OTHER_FB_KEYS:
+            buckets["forward_backward_total_ms"] += v
+            accounted.add(k)
+    fb = buckets["forward_backward_total_ms"]
+    inner = (
+        buckets["compute_ms"]
+        + buckets["comm_send_recv_ms"]
+        + buckets["comm_collective_ms"]
+    )
+    if fb > 0 and inner <= fb + 1e-6:
+        buckets["bubble_other_ms"] = max(0.0, fb - inner)
+    else:
+        unacc = sum(v for k, v in timers.items() if k not in accounted)
+        buckets["bubble_other_ms"] = max(0.0, unacc)
+    return buckets
+
+
+def _write_breakdown_markdown(series: List[Dict[str, Any]], md_path: str) -> None:
+    """Human-readable table: ms and % per Megatron timer bucket per benchmark row."""
+    lines = [
+        "## Megatron timer bucket breakdown (serial ratio)",
+        "",
+        "*Not GPU kernel overlap; use TraceLens `gpu_timeline` / Excel for device-level comm and compute.*",
+        "",
+    ]
+    for s in series:
+        label = s["label"]
+        p = s["percentages"]
+        b = s["buckets_ms"]
+        tot = p.get("total_ms", 0.0)
+        lines.append(f"### {label}")
+        lines.append("")
+        lines.append(
+            "| Bucket | ms | % of step |"
+        )
+        lines.append("|--------|-----|-----------|")
+        order = [
+            ("compute", "compute_ms"),
+            ("comm send/recv", "comm_send_recv_ms"),
+            ("comm collective", "comm_collective_ms"),
+            ("optimizer", "optimizer_ms"),
+            ("data", "data_ms"),
+            ("bubble / other", "bubble_other_ms"),
+        ]
+        pct_keys = [
+            "compute_pct",
+            "comm_send_recv_pct",
+            "comm_collective_pct",
+            "optimizer_pct",
+            "data_pct",
+            "bubble_other_pct",
+        ]
+        for (name, bk), pk in zip(order, pct_keys):
+            ms = b.get(bk, 0.0)
+            pc = p.get(pk, 0.0)
+            lines.append(f"| {name} | {ms:.2f} | {pc:.2f}% |")
+        lines.append(f"| **total (summed buckets)** | **{tot:.2f}** | **100%** |")
+        lines.append("")
+    Path(md_path).write_text("\n".join(lines))
+
+
+def _pct_row(buckets: Dict[str, float]) -> Dict[str, Any]:
+    parts = {
+        "compute": buckets.get("compute_ms", 0.0),
+        "comm_send_recv": buckets.get("comm_send_recv_ms", 0.0),
+        "comm_collective": buckets.get("comm_collective_ms", 0.0),
+        "optimizer": buckets.get("optimizer_ms", 0.0),
+        "data": buckets.get("data_ms", 0.0),
+        "bubble_other": buckets.get("bubble_other_ms", 0.0),
+    }
+    total = sum(parts.values())
+    if total <= 0:
+        return {**{k: 0.0 for k in parts}, "total_ms": 0.0, "note": "no_timer_data"}
+    pct = {f"{k}_pct": round(100.0 * v / total, 2) for k, v in parts.items()}
+    pct["total_ms"] = round(total, 2)
+    pct["accounting"] = "serial_timer_ratio"
+    return pct
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument(
+        "logs",
+        nargs="+",
+        help="Training log files (e.g. output/bench_*.log)",
+    )
+    ap.add_argument(
+        "--output-json",
+        default="output/perf_breakdown.json",
+        help="Write aggregated JSON",
+    )
+    ap.add_argument(
+        "--output-png",
+        default="output/perf_breakdown.png",
+        help="Write stacked bar chart",
+    )
+    ap.add_argument(
+        "--label",
+        action="append",
+        default=[],
+        help="Label per log file (same order as logs); default stem names",
+    )
+    ap.add_argument(
+        "--output-md",
+        default="",
+        help="Write markdown table of ms and %% per bucket (default: next to --output-json as perf_breakdown_table.md)",
+    )
+    args = ap.parse_args()
+    labels = args.label
+    if labels and len(labels) != len(args.logs):
+        raise SystemExit("--label count must match logs")
+    if not labels:
+        labels = [Path(p).stem for p in args.logs]
+
+    series: List[Dict[str, Any]] = []
+    for path, label in zip(args.logs, labels):
+        text = Path(path).read_text(errors="replace")
+        blocks = _parse_timer_blocks(text)
+        agg = _aggregate_timers(blocks)
+        buckets = _bucket_ms(agg)
+        row = _pct_row(buckets)
+        elapsed_m = None
+        m = _RE_ELAPSED.search(text)
+        if m:
+            elapsed_m = float(m.group(1))
+        series.append(
+            {
+                "label": label,
+                "log_path": path,
+                "timer_means_ms": {k: round(v, 3) for k, v in agg.items()},
+                "buckets_ms": {k: round(v, 3) for k, v in buckets.items()},
+                "percentages": row,
+                "elapsed_time_per_iter_ms": elapsed_m,
+            }
+        )
+
+    out_dir = Path(args.output_json).parent
+    out_dir.mkdir(parents=True, exist_ok=True)
+    any_timers = any(s.get("timer_means_ms") for s in series)
+    note = None if any_timers else "no_megatron_timer_blocks_found_in_logs"
+    payload = {
+        "series": series,
+        "note": note,
+        "interpretation": (
+            "Megatron log timer categories; ratios are serial shares of summed bucket times, "
+            "not GPU overlap or kernel time. For GPU/kernel/comm detail use TraceLens Excel from the profiler trace."
+        ),
+    }
+    Path(args.output_json).write_text(json.dumps(payload, indent=2))
+
+    md_path = args.output_md or str(Path(args.output_json).with_name("perf_breakdown_table.md"))
+    if any_timers:
+        _write_breakdown_markdown(series, md_path)
+
+    try:
+        import matplotlib
+
+        matplotlib.use("Agg")
+        import matplotlib.pyplot as plt
+        import numpy as np
+    except ImportError:
+        print("matplotlib not installed; JSON written only.")
+        return
+
+    if not any_timers:
+        print("No timer blocks found; JSON written, skipping PNG.")
+        return
+
+    keys_order = [
+        "compute_pct",
+        "comm_send_recv_pct",
+        "comm_collective_pct",
+        "optimizer_pct",
+        "data_pct",
+        "bubble_other_pct",
+    ]
+    labels_short = [s["label"] for s in series]
+    data = []
+    for s in series:
+        p = s["percentages"]
+        data.append([p.get(k, 0.0) for k in keys_order])
+    arr = np.array(data).T if data else np.zeros((len(keys_order), 0))
+    x = np.arange(len(labels_short))
+    fig, ax = plt.subplots(figsize=(max(8, len(labels_short) * 1.2), 5))
+    bottom = np.zeros(len(labels_short))
+    colors = ["#1f4e79", "#c55a11", "#ffc000", "#92d050", "#00b0f0", "#5b9bd5"]
+    pretty = [
+        "compute",
+        "comm send/recv",
+        "comm collective",
+        "optimizer",
+        "data",
+        "bubble / other",
+    ]
+    _ms_for_layer = [
+        "compute_ms",
+        "comm_send_recv_ms",
+        "comm_collective_ms",
+        "optimizer_ms",
+        "data_ms",
+        "bubble_other_ms",
+    ]
+    for i, row in enumerate(arr):
+        ax.bar(x, row, bottom=bottom, label=pretty[i], color=colors[i % len(colors)])
+        for xi, seg_pct in enumerate(row):
+            if seg_pct < 1.5:
+                continue
+            ms_val = series[xi]["buckets_ms"].get(_ms_for_layer[i], 0.0)
+            ypos = bottom[xi] + seg_pct / 2.0
+            ax.text(
+                float(xi),
+                ypos,
+                f"{seg_pct:.1f}%\n({ms_val:.0f}ms)",
+                ha="center",
+                va="center",
+                fontsize=7,
+                color="white" if i < 2 else "black",
+            )
+        bottom += row
+    ax.set_xticks(x)
+    ax.set_xticklabels(labels_short, rotation=25, ha="right")
+    ax.set_ylabel("Ratio (%)")
+    ax.set_ylim(0, 100)
+    ax.legend(loc="upper center", bbox_to_anchor=(0.5, -0.18), ncol=3)
+    ax.set_title(
+        "Megatron timer buckets (serial ratio of summed categories; not GPU overlap)"
+    )
+    fig.tight_layout()
+    # Second figure: non-compute buckets only (easier to read when compute dominates)
+    fig2, ax2 = plt.subplots(figsize=(max(8, len(labels_short) * 1.2), 4))
+    nc_keys = keys_order[1:]  # skip compute_pct
+    nc_pretty = pretty[1:]
+    nc_colors = colors[1:]
+    nc_arr = arr[1:, :]
+    bottom2 = np.zeros(len(labels_short))
+    for i, row in enumerate(nc_arr):
+        ax2.bar(x, row, bottom=bottom2, label=nc_pretty[i], color=nc_colors[i % len(nc_colors)])
+        bottom2 += row
+    ax2.set_xticks(x)
+    ax2.set_xticklabels(labels_short, rotation=25, ha="right")
+    ax2.set_ylabel("Ratio (%) (of full step)")
+    ax2.set_ylim(0, min(100, float(np.max(bottom2)) * 1.25 + 1.0))
+    ax2.legend(loc="upper center", bbox_to_anchor=(0.5, -0.22), ncol=3)
+    ax2.set_title("Non-compute buckets only (same scale as top chart)")
+    fig2.tight_layout()
+    nc_png = Path(args.output_png).parent / (Path(args.output_png).stem + "_non_compute.png")
+    fig2.savefig(str(nc_png), dpi=120)
+    plt.close(fig2)
+    fig.savefig(args.output_png, dpi=120)
+    plt.close(fig)
+    extra = f", {md_path}, {nc_png}" if any_timers else ""
+    print(f"Wrote {args.output_json}, {args.output_png}{extra}")
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/run_nccl_analyser_if_multirank.py b/tools/run_nccl_analyser_if_multirank.py
new file mode 100644
index 00000000000..457c43dbbc4
--- /dev/null
+++ b/tools/run_nccl_analyser_if_multirank.py
@@ -0,0 +1,85 @@
+#!/usr/bin/env python3
+# Copyright (c) 2026, Advanced Micro Devices, Inc.
+"""If multiple rank trace JSONs exist, run TraceLens NcclAnalyser and write CSV.
+
+See https://github.com/AMD-AGI/TraceLens/blob/main/docs/NcclAnalyser.md
+
+Looks for rank*_trace.json or rank_*_trace.json under SEARCH_DIR (default: cwd).
+"""
+from __future__ import annotations
+
+import argparse
+import os
+import re
+import sys
+from pathlib import Path
+
+
+def _discover_traces(search_dir: Path) -> list[Path] | None:
+    patterns = [
+        re.compile(r"^rank(\d+)_trace\.json$", re.I),
+        re.compile(r"^rank(\d+).*\.json$", re.I),
+        re.compile(r"^trace_rank(\d+)\.json$", re.I),
+    ]
+    by_rank: dict[int, Path] = {}
+    for p in search_dir.rglob("*.json"):
+        name = p.name
+        for pat in patterns:
+            m = pat.match(name)
+            if m:
+                by_rank[int(m.group(1))] = p
+                break
+    if len(by_rank) < 2:
+        return None
+    ranks = sorted(by_rank.keys())
+    expected = list(range(len(ranks)))
+    if ranks != expected:
+        # still try world_size = len(by_rank) with sorted paths
+        paths = [by_rank[r] for r in ranks]
+    else:
+        paths = [by_rank[r] for r in ranks]
+    return paths
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser()
+    ap.add_argument(
+        "--search-dir",
+        type=Path,
+        default=Path(os.environ.get("NCCL_TRACE_SEARCH_DIR", ".")),
+        help="Directory to search for rank*.json traces",
+    )
+    ap.add_argument(
+        "--output-csv",
+        type=Path,
+        default=Path("output/nccl_summary.csv"),
+        help="Write NCCL summary CSV here",
+    )
+    args = ap.parse_args()
+
+    paths = _discover_traces(args.search_dir)
+    if not paths:
+        print(
+            "NcclAnalyser: fewer than 2 rank trace JSONs found; skipping (single-rank trace).",
+            file=sys.stderr,
+        )
+        return 0
+
+    try:
+        from TraceLens import NcclAnalyser
+    except ImportError:
+        print("TraceLens not installed; skipping NcclAnalyser.", file=sys.stderr)
+        return 0
+
+    world_size = len(paths)
+    str_paths = [str(p) for p in paths]
+    analyser = NcclAnalyser(str_paths, world_size)
+    df = analyser.build_df_summary_nccl_implicit_sync_cat(agg_metrics=["mean"])
+    args.output_csv.parent.mkdir(parents=True, exist_ok=True)
+    df.to_csv(args.output_csv, index=False)
+    print(f"NcclAnalyser summary written to {args.output_csv} (world_size={world_size})")
+    return 0
+
+
+if __name__ == "__main__":
+    raise SystemExit(main())
diff --git a/tools/run_tracelens_report.sh b/tools/run_tracelens_report.sh
new file mode 100755
index 00000000000..ecb9e319b34
--- /dev/null
+++ b/tools/run_tracelens_report.sh
@@ -0,0 +1,69 @@
+#!/bin/bash
+# Generate TraceLens Excel from a PyTorch profiler Chrome trace JSON.
+# Docs: https://github.com/AMD-AGI/TraceLens/blob/main/docs/generate_perf_report.md
+#
+# Required env:
+#   TRACE_JSON   Absolute path to profiler JSON (Chrome trace format).
+#
+# Optional env:
+#   TRACELENS_OUT_DIR       Output directory (default: output/tracelens)
+#   TRACELENS_GPU_ARCH_JSON Path to gpu_arch JSON for roofline / Pct Roofline
+#                            (default: tools/tracelens/gpu_arch_mi325.json next to this script)
+#   TRACELENS_TOPK_OPS      Default 80
+#   TRACELENS_TOPK_ROOFLINE Default 30
+#   TRACELENS_TOPK_SHORT    Default 50
+#   TRACELENS_EXTRA_ARGS    Extra CLI args (quoted string)
+
+set -euo pipefail
+TRACE_JSON="${TRACE_JSON:?Set TRACE_JSON to profiler json}"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+REPO_ROOT="$(cd "${SCRIPT_DIR}/.." && pwd)"
+OUT="${TRACELENS_OUT_DIR:-${REPO_ROOT}/output/tracelens}"
+mkdir -p "$OUT"
+
+DEFAULT_ARCH="${SCRIPT_DIR}/tracelens/gpu_arch_mi325.json"
+GPU_ARCH="${TRACELENS_GPU_ARCH_JSON:-$DEFAULT_ARCH}"
+TOPK_OPS="${TRACELENS_TOPK_OPS:-80}"
+TOPK_RF="${TRACELENS_TOPK_ROOFLINE:-30}"
+TOPK_SK="${TRACELENS_TOPK_SHORT:-50}"
+
+if ! command -v TraceLens_generate_perf_report_pytorch >/dev/null 2>&1; then
+  echo "TraceLens CLI not found. Install with:"
+  echo "  pip install 'git+https://github.com/AMD-AGI/TraceLens.git'"
+  exit 1
+fi
+
+BASE_OUT="$(basename "$TRACE_JSON" .json)_perf_report.xlsx"
+OUT_XLSX="${OUT}/${BASE_OUT}"
+
+ARGS=(
+  TraceLens_generate_perf_report_pytorch
+  --profile_json_path "$TRACE_JSON"
+  --output_xlsx_path "$OUT_XLSX"
+  --short_kernel_study
+  --topk_ops "$TOPK_OPS"
+  --topk_roofline_ops "$TOPK_RF"
+  --topk_short_kernels "$TOPK_SK"
+)
+
+if [ -f "$GPU_ARCH" ]; then
+  ARGS+=(--gpu_arch_json_path "$GPU_ARCH")
+else
+  echo "Warning: GPU arch JSON not found at $GPU_ARCH — roofline ceilings omitted."
+fi
+
+if [ -n "${TRACELENS_EXTRA_ARGS:-}" ]; then
+  # shellcheck disable=SC2206
+  EXTRA=( $TRACELENS_EXTRA_ARGS )
+  ARGS+=("${EXTRA[@]}")
+fi
+
+set +e
+( cd "$OUT" && "${ARGS[@]}" ) 2>&1 | tee "$OUT/tracelens.log"
+RC=$?
+set -e
+if [ "$RC" -ne 0 ]; then
+  echo "TraceLens_generate_perf_report_pytorch exited with $RC (see $OUT/tracelens.log)"
+  exit "$RC"
+fi
+echo "TraceLens Excel: $OUT_XLSX"
diff --git a/tools/summarize_profiler_trace.py b/tools/summarize_profiler_trace.py
new file mode 100755
index 00000000000..b97784b1b74
--- /dev/null
+++ b/tools/summarize_profiler_trace.py
@@ -0,0 +1,162 @@
+#!/usr/bin/env python3
+"""Lightweight summary of a PyTorch Chrome trace JSON (top GPU + CPU ops by duration).
+
+For full reports install TraceLens and use tools/run_tracelens_report.sh.
+"""
+
+from __future__ import annotations
+
+import argparse
+import json
+from collections import defaultdict
+from pathlib import Path
+
+
+def _dur_ms(ev: dict) -> float | None:
+    d = ev.get("dur")
+    if d is None:
+        return None
+    try:
+        return float(d) / 1000.0  # μs -> ms
+    except (TypeError, ValueError):
+        return None
+
+
+def _event_bucket(ev: dict) -> str | None:
+    """Route Complete ('X') events into cuda | cpu | cuda_runtime_cpu | skip."""
+    if ev.get("ph") != "X":
+        return None
+    cat = (ev.get("cat") or "").lower()
+    name = (ev.get("name") or "").lower()
+
+    # GPU kernels / device work (category varies by PyTorch / tool version)
+    if (
+        "cuda" in cat
+        or "kernel" in cat
+        or "gpu" in cat
+        or "gpu_mem" in cat
+        or "stream" in cat
+        or "Memcpy" in (ev.get("name") or "")
+    ):
+        return "cuda"
+
+    # CPU-side PyTorch / Python (profiler labels)
+    if (
+        "cpu_op" in cat
+        or cat in ("python_function", "user_annotation", "python")
+        or cat.startswith("cpu")
+        or "torch" in cat
+    ):
+        return "cpu"
+
+    # CUDA driver/runtime API time attributed to CPU thread (still useful for bottlenecks)
+    if "cuda_runtime" in cat or "cuda api" in cat or cat == "cuda_driver":
+        return "cuda_runtime_cpu"
+
+    return None
+
+
+def _aggregate(events: list) -> tuple[dict[str, float], dict[str, float], dict[str, float]]:
+    cuda_by_name: defaultdict[str, float] = defaultdict(float)
+    cpu_by_name: defaultdict[str, float] = defaultdict(float)
+    cuda_rt_by_name: defaultdict[str, float] = defaultdict(float)
+
+    for ev in events:
+        if not isinstance(ev, dict):
+            continue
+        ms = _dur_ms(ev)
+        if ms is None or ms <= 0:
+            continue
+        name = (ev.get("name") or "").strip()
+        if not name:
+            continue
+        bucket = _event_bucket(ev)
+        if bucket == "cuda":
+            cuda_by_name[name] += ms
+        elif bucket == "cpu":
+            cpu_by_name[name] += ms
+        elif bucket == "cuda_runtime_cpu":
+            cuda_rt_by_name[name] += ms
+
+    return cuda_by_name, cpu_by_name, cuda_rt_by_name
+
+
+def _fmt_section(title: str, totals: dict[str, float], top_n: int) -> list[str]:
+    lines = [title, ""]
+    top = sorted(totals.items(), key=lambda x: -x[1])[:top_n]
+    if not top:
+        lines.append("  (no events matched)")
+        lines.append("")
+        return lines
+    for name, ms in top:
+        lines.append(f"  {ms:10.3f}  {name}")
+    lines.append("")
+    return lines
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser()
+    ap.add_argument("trace_json", help="Chrome trace .json from PyTorch profiler")
+    ap.add_argument("-o", "--output", default="", help="Optional text summary path")
+    ap.add_argument(
+        "-n",
+        "--top",
+        type=int,
+        default=40,
+        help="Max lines per section (default 40)",
+    )
+    ap.add_argument(
+        "--no-cuda-runtime-cpu",
+        action="store_true",
+        help="Omit CUDA runtime (CPU-thread) section",
+    )
+    args = ap.parse_args()
+    path = Path(args.trace_json)
+    data = json.loads(path.read_text())
+    events = data.get("traceEvents") or data.get("events") or []
+
+    cuda_by_name, cpu_by_name, cuda_rt_by_name = _aggregate(events)
+
+    lines: list[str] = []
+    lines.extend(
+        _fmt_section(
+            "Top GPU / CUDA-named regions by inclusive duration (ms):",
+            cuda_by_name,
+            args.top,
+        )
+    )
+    lines.extend(
+        _fmt_section(
+            "Top CPU-side regions (cpu_op / Python / annotations, ms):",
+            cpu_by_name,
+            args.top,
+        )
+    )
+    if not args.no_cuda_runtime_cpu:
+        lines.extend(
+            _fmt_section(
+                "Top CUDA runtime API on CPU thread (ms, launch/sync overhead):",
+                cuda_rt_by_name,
+                args.top,
+            )
+        )
+
+    sum_cuda = sum(cuda_by_name.values())
+    sum_cpu = sum(cpu_by_name.values())
+    sum_rt = sum(cuda_rt_by_name.values())
+    lines.append("Bucket totals (sum of matched Complete events; overlapping GPU work is not deduplicated):")
+    lines.append("")
+    lines.append(f"  GPU / CUDA-named regions: {sum_cuda:,.3f} ms")
+    lines.append(f"  CPU-side regions:         {sum_cpu:,.3f} ms")
+    if not args.no_cuda_runtime_cpu:
+        lines.append(f"  CUDA runtime on CPU:      {sum_rt:,.3f} ms")
+    lines.append("")
+
+    text = "\n".join(lines).rstrip() + "\n"
+    print(text, end="")
+    if args.output:
+        Path(args.output).write_text(text)
+
+
+if __name__ == "__main__":
+    main()
diff --git a/tools/tracelens/gpu_arch_mi325.json b/tools/tracelens/gpu_arch_mi325.json
new file mode 100644
index 00000000000..d7f02f45679
--- /dev/null
+++ b/tools/tracelens/gpu_arch_mi325.json
@@ -0,0 +1,17 @@
+{
+  "name": "MI325X_OAM",
+  "mem_bw_gbps": 6000,
+  "max_achievable_tflops": {
+    "matrix_fp16": 1307,
+    "matrix_bf16": 1307,
+    "matrix_fp32": 163,
+    "matrix_fp64": 82,
+    "matrix_fp8": 2615,
+    "matrix_int8": 2615,
+    "vector_fp16": 163,
+    "vector_bf16": 163,
+    "vector_fp32": 163,
+    "vector_fp64": 82
+  },
+  "_reference": "Peak theoretical per-accelerator figures from AMD Instinct MI325X product brief / datasheet (FP16/BF16 matrix ~1307 TFLOPS, FP8/INT8 ~2615 TFLOPS, FP32 ~163 TFLOPS, FP64 ~82 TFLOPS, HBM3E peak memory bandwidth 6 TB/s). Sources: https://www.amd.com/content/dam/amd/en/documents/instinct-tech-docs/product-briefs/instinct-mi325x-datasheet.pdf , https://www.amd.com/en/products/accelerators/instinct/mi300/mi325x/platform.html . TraceLens expects MAF-style ceilings for roofline; replace these with measured MAF from https://rocm.blogs.amd.com/software-tools-optimization/measuring-max-achievable-flops-part2/README.html if you need tighter roofline bounds."
+}