Core_r0.17.0 IFU

.github/CODEOWNERS

@@ -9,6 +9,8 @@ megatron/core/ssm/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/hybrid-mamba
 
 megatron/core/datasets/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/datasets
 
+megatron/core/tokenizers/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/tokenizers
+
 megatron/core/distributed/fsdp/ @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/megatron-fsdp
 
 megatron/core/transformer/fsdp_dtensor_checkpoint.py @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/megatron-fsdp
@@ -37,6 +39,9 @@ megatron/post_training/ @NVIDIA/post-training
 
 megatron/core/transformer/cuda_graphs.py @NVIDIA/core-adlr @NVIDIA/core-nemo @NVIDIA/cuda-graphs
 
+megatron/training/ @NVIDIA/training-adlr @NVIDIA/training-nemo
+megatron/training/arguments.py
+
 .gitlab/ @NVIDIA/ci
 .github/ @NVIDIA/ci
 .gitlab-ci.yml @NVIDIA/ci

.github/actions/action.yml

@@ -45,51 +45,31 @@ inputs:
   PAT:
     description: "GitHub Personal Access Token"
     required: true
-  is_ci_workload:
-    description: "Is CI workload"
-    required: true
-
+  scope:
+    description: "Test scope (e.g. mr-github, mr-github-slim)"
+    required: false
+    default: "mr-github-slim"
+  n_repeat:
+    description: "Number of test repetitions"
+    required: false
+    default: "5"
+  lightweight:
+    description: "Enable lightweight mode"
+    required: false
+    default: "false"
+  platform:
+    description: "Platform to run tests on (e.g. dgx_h100, dgx_gb200)"
+    required: false
+    default: "dgx_h100"
 runs:
   using: "composite"
   steps:
     - name: Print node name
       shell: bash -x -e -u -o pipefail {0}
       run: echo "node_name=$NODE_NAME" | tee -a "$GITHUB_OUTPUT"
 
-    - name: GPU Sanity Check
-      shell: bash -x -e -u -o pipefail {0}
-      run: |
-        echo "Starting GPU Sanity Check..."
-
-        # 1. Check for active Compute Processes
-        # query-compute-apps returns a list of PIDs using the GPU. If empty, we are good.
-        OPEN_PROCESSES=$(docker run --rm --gpus all ubuntu nvidia-smi --query-compute-apps=pid,process_name --format=csv,noheader)
-
-        if [ -n "$OPEN_PROCESSES" ]; then
-          echo "::error::❌ GPU is not clean! Found active processes:"
-          echo "$OPEN_PROCESSES"
-        else
-          echo "✅ No active compute processes found."
-        fi
-
-        # 2. Check VRAM Usage (Optional but recommended)
-        # We allow a small buffer (e.g., < 300MiB) for driver overhead/Xorg, 
-        # though on headless K8s nodes this should be very close to 0.
-
-        MEMORY_USAGES=$(docker run --rm --gpus all ubuntu nvidia-smi --query-gpu=memory.used --format=csv,noheader,nounits)
-
-        # Check each GPU visible to the container
-        for MEMORY in $MEMORY_USAGES; do
-          if [ "$MEMORY" -gt 300 ]; then
-            echo "::error::❌ GPU VRAM usage is suspiciously high: ${MEMORY} MiB"
-          fi
-        done
-
-        echo "✅ GPU Memory is clean (all < 300 MiB)."
-        echo "Ready to start workflow."
-
     - name: Checkout repository
-      uses: actions/checkout@v2
+      uses: actions/checkout@v6
 
     - name: Change ownership of /home/runner/
       shell: bash
@@ -117,55 +97,25 @@ runs:
         export PYTHONPATH=$(pwd)
         export NEMORUN_HOME=$(pwd)
         export NCCL_DEBUG=INFO
-        pip install --no-cache-dir uv
-        uv sync --only-group test
+        pip install --no-cache-dir "uv<0.9.29"
+        uv venv .venv
+        uv cache clean
+        uv sync --no-cache --only-group test
         uv run python tests/test_utils/python_scripts/launch_nemo_run_workload.py \
           --scope unit-tests \
           --model unit-tests \
           --test-case "${{ inputs.test_case }}" \
           --environment dev \
-          --platform dgx_h100 \
+          --platform ${{ inputs.platform }} \
           --tag ${{ inputs.tag }} \
-          --container-image ${{ inputs.container-image }}
+          --container-image ${{ inputs.container-image }} \
+          --hf-home /mnt/datadrive/TestData/nemo-fw/TestData/HF_HOME
 
         RUN_TEST_EOF
         )
         echo "$cmd" | tee "job.sh"
         echo "::endgroup::"
 
-    - name: Get PR info
-      id: get-pr-info
-      if: startsWith(github.ref, 'refs/heads/pull-request/')
-      uses: nv-gha-runners/get-pr-info@main
-
-    - name: Install GH CLI
-      shell: bash -x -e -u -o pipefail {0}
-      run: |
-        apt-get update
-        apt-get install -y gh
-
-    - name: Has Run tests label
-      shell: bash -x -e -u -o pipefail {0}
-      id: has-run-tests-label
-      env:
-        GH_TOKEN: ${{ github.token }}
-      run: |
-        PR_NUMBER=${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number }}
-        HAS_RUN_TESTS_LABEL=$(gh pr view $PR_NUMBER --json labels | jq '[.labels[].name] | any(. == "Run tests")') || echo "false"
-        echo "main=$HAS_RUN_TESTS_LABEL" | tee -a $GITHUB_OUTPUT
-
-    - name: Has Run functional tests label
-      shell: bash -x -e -u -o pipefail {0}
-      id: has-run-functional-tests-label
-      env:
-        GH_TOKEN: ${{ github.token }}
-        IS_CI_WORKLOAD: ${{ inputs.is_ci_workload }}
-      run: |
-        PR_NUMBER=${{ fromJSON(steps.get-pr-info.outputs.pr-info || '{}').number }}
-        HAS_RUN_FUNCTIONAL_TESTS_LABEL=$(gh pr view $PR_NUMBER --json labels | jq '[.labels[].name] | any(. == "Run functional tests")') || echo "$IS_CI_WORKLOAD"
-        HAS_RUN_FUNCTIONAL_TESTS_LABEL=${HAS_RUN_FUNCTIONAL_TESTS_LABEL:-$IS_CI_WORKLOAD}
-        echo "main=$HAS_RUN_FUNCTIONAL_TESTS_LABEL" | tee -a $GITHUB_OUTPUT
-
     - name: Create run-script (e2e test)
       shell: bash -x -e -u -o pipefail {0}
       if: inputs.is_unit_test == 'false'
@@ -177,36 +127,29 @@ runs:
         #!/bin/bash
         set -euxo pipefail
 
-        if [ "${{ steps.has-run-tests-label.outputs.main }}" == "true" ]; then
-          ARGS=(
-            --scope mr-github
-            --enable-lightweight-mode
-            --n-repeat 1
-          )
-        elif [ "${{ steps.has-run-functional-tests-label.outputs.main }}" == "true" ]; then
-          ARGS=(
-            --scope mr-github
-            --n-repeat 5
-          )
-        else
-          ARGS=(
-            --scope mr-github-slim
-            --n-repeat 5
-          )
+        ARGS=(
+          --scope ${{ inputs.scope }}
+          --n-repeat ${{ inputs.n_repeat }}
+        )
+        if [ "${{ inputs.lightweight }}" == "true" ]; then
+          ARGS+=(--enable-lightweight-mode)
         fi
 
         export PYTHONPATH=$(pwd)
         export NEMORUN_HOME=$(pwd)
-        pip install --no-cache-dir uv
-        uv sync --only-group test
+        pip install --no-cache-dir "uv<0.9.29"
+        uv venv .venv
+        uv cache clean
+        uv sync --no-cache --only-group test
         uv run python tests/test_utils/python_scripts/launch_nemo_run_workload.py \
           ${ARGS[@]} \
           --model ${{ inputs.model }} \
           --test-case ${{ inputs.test_case }} \
           --environment dev \
-          --platform dgx_h100 \
+          --platform ${{ inputs.platform }} \
           --container-image ${{ inputs.container-image }} \
           --data-dir /mnt/datadrive/TestData/megatron-lm/artifacts \
+          --hf-home /mnt/datadrive/TestData/nemo-fw/TestData/HF_HOME
 
         RUN_TEST_EOF
         )

.github/copy-pr-bot.yaml

@@ -1,4 +1,4 @@
 enabled: true
 auto_sync_draft: false
 auto_sync_ready: true
-trustees_override: ["AAnoosheh", "ArEsKay3", "Autumn1998", "BestJuly", "BoxiangW", "ChenhanYu", "FDecaYed", "HaochenYuan", "ISEEKYAN", "JRD971000", "Phlip79", "QiZhangNV", "ShriyaRishab", "Victarry", "Wohox", "ZhiyuLi-Nvidia", "ahmadki", "aklife97", "ananthsub", "asolergi-nv", "buptzyb", "chtruong814", "cspades", "cuichenx", "deepakn94", "dimapihtar", "dingqingy-nv", "duncanriach", "erhoo82", "ericharper", "fanshiqing", "frsun-nvda", "gautham-kollu", "gdengk", "guyueh1", "hxbai", "jalbericiola", "janEbert", "jaredcasper", "jenchen13", "jiemingz", "jingqiny-99", "jkamalu", "jon-barker", "jstjohn", "kanz-nv", "kevalmorabia97", "ko3n1g", "kunlunl", "kvareddy", "kwyss-nvidia", "layalir", "lhb8125", "lmcafee-nvidia", "maanug-nv", "mathemakitten", "matthieule", "mehraakash", "mkhona-nvidia", "parthmannan", "prajwal1210", "pthombre", "rogerwaleffe", "sanandaraj5597", "sancha", "santhnm2", "sbak5", "shanmugamr1992", "shengf-nv", "shifangx", "shjwudp", "sidsingh-nvidia", "skyw", "sudhakarsingh27", "tdene", "theothermike", "thomasdhc", "trintamaki", "tylerpoon", "wdykas", "xiaoyao0115", "xuwchen", "yanring", "yaox12", "yaoyu-33", "yashaswikarnati", "yeyu-nvidia", "yobibyte", "youngeunkwon0405", "yuzhongw-nvidia", "zhongbozhu"]
+trustees_override: ["AAnoosheh", "ArEsKay3", "Autumn1998", "BestJuly", "BoxiangW", "CarlosGomes98", "ChenhanYu", "FDecaYed", "HaochenYuan", "ISEEKYAN", "JRD971000", "Phlip79", "QiZhangNV", "RPrenger", "ShriyaRishab", "Victarry", "WanZzzzzz", "Wohox", "ZhiyuLi-Nvidia", "ahmadki", "aklife97", "ananthsub", "asolergi-nv", "buptzyb", "chtruong814", "cjld", "cspades", "cuichenx", "deepakn94", "dimapihtar", "dingqingy-nv", "duncanriach", "erhoo82", "ericharper", "fanshiqing", "faradawn", "frsun-nvda", "gautham-kollu", "gdengk", "guyueh1", "hexinw-nvidia", "huvunvidia", "hxbai", "ilml", "jalbericiola", "janEbert", "jaredcasper", "jenchen13", "jiemingz", "jingqiny-99", "jkamalu", "jon-barker", "jstjohn", "kajalj22", "kanz-nv", "kevalmorabia97", "ko3n1g", "ksivaman", "kunlunl", "kvareddy", "kwyss-nvidia", "layalir", "lhb8125", "lmcafee-nvidia", "maanug-nv", "mathemakitten", "matthieule", "mchrzanowski", "mehraakash", "mkhona-nvidia", "nanz-nv", "parthmannan", "prajwal1210", "pthombre", "rhewett-nv", "rogerwaleffe", "sajadn", "sanandaraj5597", "sancha", "santhnm2", "sbak5", "shanmugamr1992", "sharathts", "shengf-nv", "shifangx", "shjwudp", "sidsingh-nvidia", "skyw", "sudhakarsingh27", "tdene", "theothermike", "thomasdhc", "tomlifu", "trintamaki", "tylerpoon", "wdykas", "wplf", "xiaoyao0115", "xuwchen", "yanring", "yaox12", "yaoyu-33", "yashaswikarnati", "yeyu-nvidia", "yobibyte", "youngeunkwon0405", "yueshen2016", "yuzhongw-nvidia", "zhongbozhu"]

.github/oncall_schedule.json

@@ -1,50 +1,50 @@
 [
     {
-        "user": "dimapihtar",
-        "date": "2026-01-28"
+        "user": "janEbert",
+        "date": "2026-03-25"
     },
     {
         "user": "gautham-kollu",
-        "date": "2026-02-04"
+        "date": "2026-04-01"
     },
     {
-        "user": "janEbert",
-        "date": "2026-02-11"
+        "user": "ilml",
+        "date": "2026-04-08"
     },
     {
         "user": "Phlip79",
-        "date": "2026-02-18"
+        "date": "2026-04-15"
     },
     {
         "user": "asolergi-nv",
-        "date": "2026-02-25"
+        "date": "2026-04-22"
     },
     {
         "user": "BoxiangW",
-        "date": "2026-03-04"
+        "date": "2026-04-29"
     },
     {
         "user": "maanug-nv",
-        "date": "2026-03-11"
+        "date": "2026-05-06"
     },
     {
         "user": "dimapihtar",
-        "date": "2026-03-18"
+        "date": "2026-05-13"
     },
     {
         "user": "gautham-kollu",
-        "date": "2026-03-25"
+        "date": "2026-05-20"
     },
     {
-        "user": "janEbert",
-        "date": "2026-04-01"
+        "user": "ilml",
+        "date": "2026-05-27"
     },
     {
-        "user": "maanug-nv",
-        "date": "2026-04-08"
+        "user": "janEbert",
+        "date": "2026-06-03"
     },
     {
-        "user": "BoxiangW",
-        "date": "2026-04-15"
+        "user": "maanug-nv",
+        "date": "2026-06-10"
     }
 ]

.github/pull_request_template.md

@@ -5,19 +5,8 @@
 
 ## Contribution process
 
-```mermaid
-flowchart LR
-    A[Pre-checks] --> B[PR Tests]
-    subgraph Code Review/Approval
-        C1[Expert Review] --> C2[Final Review]
-    end
-    B --> C1
-    C2 --> D[Merge]
-```
-
 ### Pre-checks
 
-- [ ] I want this PR in a versioned release and have added the appropriate Milestone (e.g., `Core 0.8`)
 - [ ] I have added relevant unit tests
 - [ ] I have added relevant functional tests
 - [ ] I have added proper typing to my code [Typing guidelines](https://docs.python.org/3/library/typing.html)
@@ -26,41 +15,36 @@ flowchart LR
 
 ### Code review
 
-The following process is enforced via the CODEOWNERS file for changes into `megatron/core`. For changes outside of `megatron/core`, it is up to the PR author whether or not to tag the Final Reviewer team.
+Feel free to message or comment the [@mcore-oncall](https://github.com/orgs/NVIDIA/teams/mcore-oncall) to help accelerate your merge into main. The less complex your PR is, the faster it will be approved and merged!
 
-<details>
-<summary>For MRs into `main` branch</summary>
+All PRs start as **draft**. If you open a non-draft PR, it will be automatically converted to draft.
 
-Feel free to message or comment the @mcore-oncall to help accelerate your merge into main. The less complex your PR is, the faster it will be approved and merged!
+#### Step 1: Mark PR as "Ready for Review"
 
-#### (Step 1): Add PR label `Expert Review`
+1. When your PR is ready, click **Ready for Review**.
+2. An oncall reviewer is auto-assigned and expert reviewers are notified based on your changes.
+   - Some PRs may jump straight to step 2. This is determined by `.github/CODEOWNERS`.
 
-#### (Step 2): Collect the expert reviewers reviews
+:warning: Only mark as ready once merge-conflicts are resolved and the CI is passing.
+Final Review might get declined if these requirements are not fulfilled.
 
-1. Attach the `Expert Review` label when your PR is ready for review.
-2. GitHub auto-assigns expert reviewers based on your changes. They will get notified and pick up your PR soon.
+#### Step 2: Final Review
 
-:warning: Only proceed to the next step once all reviewers have approved, merge-conflict are resolved and the CI is passing.  
-Final Review might get declined if these requirements are not fulfilled.
+For PRs that change `megatron/core`, once all expert reviewers have approved, the `Final Review` label is applied **automatically** and final reviewers are assigned.
 
-#### (Step 3): Final Review
+For PRs outside `megatron/core`, this step is skipped.
 
-1. Add `Final Review` label
-2. GitHub auto-assigns final reviewers based on your changes. They will get notified and pick up your PR soon.
+#### Step 3: Approved
 
-#### (Optional Step 4): Cherry-pick into release branch
+Once all required reviewers have approved, the `Approved` label is applied **automatically**.
 
-If this PR also needs to be merged into `core_r*` release branches, after this PR has been merged, select `Cherry-pick` to open a new PR into the release branch.
+### Merge
 
-</details>
+Any member of [mcore-engineers](https://github.com/orgs/NVIDIA/teams/mcore-engineers) will be able to merge your PR.
 
 <details>
 <summary>For MRs into `dev` branch</summary>
 The proposed review process for `dev` branch is under active discussion.
 
 MRs are mergable after one approval by either `eharper@nvidia.com` or `zijiey@nvidia.com`.
 </details>
-
-### Merging your PR
-
-Any member of [core-adlr](https://github.com/orgs/teams/NVIDIA/core-adlr) and [`core-nemo`](https://github.com/orgs/teams/NVIDIA/core-nemo) will be able to merge your PR.