ci: bump base image/runner to v26.4 and drop v26.3-only workarounds

.github/workflows/ci.yaml

@@ -25,10 +25,10 @@ permissions:
 env:
   PRIMUS_TURBO_COMMIT: edc8d2ccb0be4888e80ee7c6e765fd3956026a32 # feat: add quantized tensor support for blockwise grouped gemm (#422)
   PRIMUS_TURBO_AITER_COMMIT: 0f3c58e6edb6754940bcf9fd5f09ccb6f389f52e # AITER v0.1.14.post1 (tag commit) — required by Primus-Turbo main aiter_utils.py
-  ROCSHMEM_COMMIT: 17ff985c026f9f97f85068647e863ab541dd5645 # Update version to 3.2.0 for 7.2.0 rocm release (#351) (#355)
+  #ROCSHMEM_COMMIT: 17ff985c026f9f97f85068647e863ab541dd5645 # Update version to 3.2.0 for 7.2.0 rocm release (#351) (#355)
   UCCL_COMMIT: 5afb4117893c58cc0c8557d9286336141a301053 # [EP]: fix fp8 error of internode_ll on amd gfx950 arch. (#710)
-  TRITON_COMMIT: 88b227e23f0445f3f695bad05bbf1a363b4f50e0
-  BASE_IMAGE: docker.io/rocm/primus:v26.3
+  TRITON_COMMIT: 09500db9f0fe66fd176d1f080e2017b37e7e995d # pinned triton-lang/triton commit built from source in Dockerfile
+  BASE_IMAGE: docker.io/rocm/primus:v26.4
   MAXTEXT_BASE_IMAGE: docker.io/rocm/jax-training:maxtext-v26.2
 
 jobs:
@@ -123,16 +123,20 @@ jobs:
           echo "> Build Docker Image with tag: ${{ env.IMAGE_TAG }}"
           echo "> Build dependencies"
           start_time=$(date +%s)
+          # Optional build-args, off by default (kept as an array so a commented-out
+          # element here can't break the docker build \-continuation below).
+          EXTRA_BUILD_ARGS=()
+          # EXTRA_BUILD_ARGS+=(--build-arg ROCSHMEM_COMMIT=${ROCSHMEM_COMMIT})
           docker build -f $CWS/.github/workflows/docker/Dockerfile \
                --network=host \
                -t tasimage/primus:${{env.IMAGE_TAG}} \
                --build-arg BASE_IMAGE=${BASE_IMAGE} \
                --build-arg PRIMUS_TURBO_COMMIT=${PRIMUS_TURBO_COMMIT} \
                --build-arg PRIMUS_TURBO_AITER_COMMIT=${PRIMUS_TURBO_AITER_COMMIT} \
-               --build-arg ROCSHMEM_COMMIT=${ROCSHMEM_COMMIT} \
                --build-arg PRIMUS_TURBO_FRAMEWORK=PYTORCH \
                --build-arg UCCL_COMMIT=${UCCL_COMMIT} \
                --build-arg TRITON_COMMIT=${TRITON_COMMIT} \
+               "${EXTRA_BUILD_ARGS[@]}" \
                $CWS/.github/workflows/docker
           end_time=$(date +%s)
           elapsed=$((end_time - start_time))
@@ -143,7 +147,7 @@ jobs:
           docker push docker.io/tasimage/primus:${{env.IMAGE_TAG}}
           # docker login -u rocmshared -p ${{ secrets.ROCM_DOCKER_HUB_TOKEN }}
 
-          # Primus v26.3 already includes AINIC under /workspace. Re-enable this
+          # Primus v26.4 already includes AINIC under /workspace. Re-enable this
           # Dockerfile.ainic build only when we need to refresh the tasimage -ainic image.
           echo "> Build Docker Image with tag: ${{ env.IMAGE_TAG }}-ainic"
           start_time=$(date +%s)
@@ -304,16 +308,18 @@ jobs:
           : > "$RUNNER_TEMP/runtime.tsv"  # reset the CI runtime log for this job
           IMG="docker.io/tasimage/primus:${{ env.IMAGE_TAG }}"
           echo "Target image: ${IMG}"
-          # Reuse the container only if it is already running on the same image;
-          # otherwise recreate it from the freshly pushed image.
-          cur_img="$(docker inspect -f '{{.Config.Image}}' "${UT_CONTAINER}" 2>/dev/null || true)"
+          # Pull first, then compare image id (not tag name): IMAGE_TAG is
+          # constant ("latest") on main, so a tag-only check can't detect a
+          # same-tag repush and would keep reusing a stale container.
+          docker pull "${IMG}"
+          target_id="$(docker image inspect -f '{{.Id}}' "${IMG}" 2>/dev/null || true)"
+          cur_id="$(docker inspect -f '{{.Image}}' "${UT_CONTAINER}" 2>/dev/null || true)"
           running="$(docker inspect -f '{{.State.Running}}' "${UT_CONTAINER}" 2>/dev/null || true)"
-          if [ "${running}" = "true" ] && [ "${cur_img}" = "${IMG}" ]; then
-            echo "Reusing container ${UT_CONTAINER} (already on ${IMG})."
+          if [ "${running}" = "true" ] && [ -n "${target_id}" ] && [ "${cur_id}" = "${target_id}" ]; then
+            echo "Reusing container ${UT_CONTAINER} (already on ${IMG}, id=${target_id})."
           else
-            echo "Recreating container ${UT_CONTAINER} (running=${running}, image=${cur_img})."
+            echo "Recreating container ${UT_CONTAINER} (running=${running}, cur_id=${cur_id}, target_id=${target_id})."
             docker rm -f "${UT_CONTAINER}" >/dev/null 2>&1 || true
-            docker pull "${IMG}"
             docker run -d --name "${UT_CONTAINER}" \
               --network host --ipc host --privileged \
               --group-add video \
@@ -333,36 +339,6 @@ jobs:
           set -e
           pip install -r requirements.txt
           IN
-      # v26.3-only: drop this whole step on v26.4. Unit tests import torchtitan
-      # from the submodule source via conftest sys.path injection (v0.2.2 has
-      # PEP-420 namespace packages that need the source root on sys.path). This
-      # just removes any stale torchtitan in the image that would shadow it.
-      - name: Purge stale TorchTitan install (v26.3-only, drop on v26.4)
-        run: |
-          docker exec -i -e GITHUB_WORKSPACE="${CWS}" -w "${CWS}" "${UT_CONTAINER}" bash -s <<'IN'
-          set -e
-          pip uninstall -y torchtitan || true
-          python - <<'PY'
-          import glob, os, shutil, site
-          dirs = set(site.getsitepackages() + [site.getusersitepackages()])
-          patterns = (
-              "__editable__*torchtitan*",
-              "__editable__.torchtitan*.pth",
-              "torchtitan*.pth",
-              "torchtitan*.egg-link",
-              "torchtitan*.dist-info",
-              "torchtitan*.egg-info",
-          )
-          for d in dirs:
-              for pat in patterns:
-                  for f in glob.glob(os.path.join(d, pat)):
-                      try:
-                          shutil.rmtree(f) if os.path.isdir(f) else os.remove(f)
-                          print("removed", f)
-                      except OSError as e:
-                          print("skip", f, e)
-          PY
-          IN
       - name: Set UT_LOG_PATH
         run: |
           ts="$(date +%Y%m%d-%H%M%S)"

.github/workflows/docker/Dockerfile

@@ -1,10 +1,10 @@
-ARG BASE_IMAGE=docker.io/rocm/primus:v26.3
+ARG BASE_IMAGE=docker.io/rocm/primus:v26.4
 FROM ${BASE_IMAGE}
 
 ARG PRIMUS_TURBO_COMMIT
 ARG PRIMUS_TURBO_AITER_COMMIT
 ARG PRIMUS_TURBO_FRAMEWORK
-ARG ROCSHMEM_COMMIT
+# ARG ROCSHMEM_COMMIT # only needed if the rocSHMEM rebuild below is restored
 ARG UCCL_COMMIT
 ARG TRITON_COMMIT
 # Non-interactive APT
@@ -21,34 +21,25 @@ RUN rm -rf /var/lib/apt/lists/*
 # ---------------------------------------------------------------------------
 # Enviroment variables
 # ---------------------------------------------------------------------------
-ENV ROCSHMEM_HOME=/opt/rocshmem
-ENV UCX_HOME=/opt/ucx
-# ENV MPI_HOME=/opt/ompi
-# Use the system OpenMPI prefix from the v26.3 base image.
-ENV MPI_HOME=/usr/lib/x86_64-linux-gnu/openmpi
-ENV ROCM_HOME=/opt/rocm
 ENV PRIMUS_TURBO_FRAMEWORK=${PRIMUS_TURBO_FRAMEWORK}
 
-# ENV PATH="/opt/ompi/bin:/opt/ompi/sbin:${PATH}"
-# ENV LD_LIBRARY_PATH="/opt/ompi/lib:${LD_LIBRARY_PATH}"
 ENV GPU_ARCHS="gfx942;gfx950"
 ENV PYTORCH_ROCM_ARCH="gfx942;gfx950"
 ENV HCC_AMDGPU_TARGET="gfx942,gfx950"
-# ---------------------------------------------------------------------------
-# Install rocSHMEM
-# ---------------------------------------------------------------------------
-RUN mkdir -p /opt && cd /opt && \
-    git clone https://github.com/ROCm/rocSHMEM.git && \
-    cd rocSHMEM && \
-    git checkout ${ROCSHMEM_COMMIT} && \
-    mkdir build && \
-    cd build && \
-    MPI_ROOT=${MPI_HOME} UCX_ROOT=${UCX_HOME} INSTALL_PREFIX=${ROCSHMEM_HOME} ../scripts/build_configs/gda \
-                                                                                            -DGDA_IONIC=ON \
-                                                                                            -DGDA_MLX5=ON  \
-                                                                                            -DGDA_BNXT=ON  \
-                                                                                            -DUSE_IPC=ON
-RUN rm -rf /opt/rocSHMEM
+
+# rocSHMEM: used by Primus ODC via Primus-Turbo's odc_rocshmem_* extensions.
+# This base image already ships a prebuilt install (commit 17ff985c, version
+# 3.2.0) at $ROCSHMEM_HOME=/opt/rocshmem, so building it again is redundant.
+#
+# If a future base image drops it, restore by uncommenting below (and the
+# `ARG ROCSHMEM_COMMIT` above). Note: cmake reads the ROCm version from
+# -DROCM_PATH=, not $ROCM_PATH; omitting it falls back to the nonexistent
+# /opt/rocm and fails.
+# RUN mkdir -p /opt && cd /opt && git clone https://github.com/ROCm/rocSHMEM.git && \
+#     cd rocSHMEM && git checkout ${ROCSHMEM_COMMIT} && mkdir build && cd build && \
+#     MPI_ROOT=${MPI_HOME} UCX_ROOT=${UCX_HOME} INSTALL_PREFIX=${ROCSHMEM_HOME} \
+#     ../scripts/build_configs/gda -DROCM_PATH=${ROCM_PATH} \
+#       -DGDA_IONIC=ON -DGDA_MLX5=ON -DGDA_BNXT=ON -DUSE_IPC=ON
 
 # ---------------------------------------------------------------------------
 # Install Primus-Turbo
@@ -73,18 +64,41 @@ RUN cd /opt && \
 
 RUN rm -rf /opt/Primus-Turbo
 
+# Primus-Turbo links librocshmem.a with -l: (not --whole-archive) under
+# -fgpu-rdc/--hip-link, so rocSHMEM's team.cpp.o (which defines the plain,
+# non-namespaced rocshmem::ROCSHMEM_TEAM_WORLD symbol) never gets pulled into
+# libprimus_turbo_kernels.so; since it's a shared object the missing symbol
+# only surfaces as an ImportError at dlopen/import time, not at build time.
+# Work around it by re-exporting that symbol from a tiny shim .so, placed on
+# libprimus_turbo_kernels.so's existing rpath and wired in via patchelf.
+RUN PT_LIB=$(python3 -c "import primus_turbo, os; print(os.path.join(os.path.dirname(primus_turbo.__file__), 'lib', 'libprimus_turbo_kernels.so'))") && \
+    mkdir -p /tmp/rocshmem_team_shim && cd /tmp/rocshmem_team_shim && \
+    ar x ${ROCSHMEM_HOME}/lib/librocshmem.a team.cpp.o && \
+    g++ -shared -fPIC -o ${ROCSHMEM_HOME}/lib/librocshmem_team_shim.so team.cpp.o \
+      -L${ROCSHMEM_HOME}/lib -l:librocshmem.a -L${MPI_HOME}/lib -l:libmpi.so \
+      -Wl,-rpath,${ROCSHMEM_HOME}/lib -Wl,-rpath,${MPI_HOME}/lib && \
+    patchelf --add-needed librocshmem_team_shim.so "${PT_LIB}" && \
+    cd / && rm -rf /tmp/rocshmem_team_shim
+
 # ---------------------------------------------------------------------------
 # Install Triton
 # ---------------------------------------------------------------------------
 RUN cd /opt && \
     git clone https://github.com/triton-lang/triton.git && \
     cd triton && \
-    git checkout 09500db9 && \
+    git checkout ${TRITON_COMMIT} && \
     pip3 install -r python/requirements.txt && \
     MAX_JOBS=96 pip3 install --no-build-isolation --force-reinstall --no-deps .
 
 RUN rm -rf /opt/triton
 
+# torch's dist-info still pins the base image's stock triton build; without this,
+# any later "pip install" (e.g. CI unit tests) sees torch's triton requirement as
+# unsatisfiable and silently upgrades torch from PyPI, dragging in a CUDA stack.
+RUN TORCH_META=$(python3 -c "import importlib.metadata as m; print(m.distribution('torch')._path / 'METADATA')") && \
+    TRITON_VER=$(python3 -c "import importlib.metadata as m; print(m.version('triton'))") && \
+    sed -i -E "s/^Requires-Dist: triton==.*/Requires-Dist: triton==${TRITON_VER}/" "${TORCH_META}"
+
 # ---------------------------------------------------------------------------
 # Install UCCL-EP (skip for JAX framework)
 # ---------------------------------------------------------------------------
@@ -100,12 +114,14 @@ RUN if [ "$PRIMUS_TURBO_FRAMEWORK" != "JAX" ]; then \
     rm -rf /opt/uccl; \
     fi
 
-# ---------------------------------------------------------------------------
-# Install fixed origami (rocm-libraries@223648a) over the base image's bundled
-# 0.1.0. The bundled origami's rank_configs() raises
-# `ValueError: vector::reserve` during MoE grouped-gemm kernel selection and
-# crashes training (turbo's _safe_rank_configs only catches RuntimeError).
-# Skipped for JAX. TODO: drop once a base image ships origami with the fix.
+# Install origami (rocm-libraries@223648a). The v26.4 base image ships no
+# origami at all. primus_turbo's MoE grouped-gemm kernel selection tolerates
+# its absence (falls back to a heuristic, no measurable perf loss on gfx942),
+# but Primus's performance-projection tool (primus projection performance)
+# hard-depends on origami as its only supported GEMM simulation backend and
+# has no fallback (see primus/core/projection/simulation_backends/factory.py)
+# — dropping this install broke TestProjectionSimulate::* outright. Skipped
+# for JAX, where the projection tool isn't used.
 RUN if [ "$PRIMUS_TURBO_FRAMEWORK" != "JAX" ]; then \
     rm -rf /tmp/rocm-libraries && \
     git clone --filter=blob:none --no-checkout https://github.com/ROCm/rocm-libraries.git /tmp/rocm-libraries && \

.github/workflows/docker/Dockerfile.ainic

@@ -7,21 +7,35 @@ ARG AINIC_BUNDLE_PATH
 # Non-interactive APT
 ENV DEBIAN_FRONTEND=noninteractive
 
+# `cmd | tee log` below reports only tee's exit code by default, silently
+# swallowing real failures; pipefail makes the RUN fail with `cmd` instead.
+SHELL ["/bin/bash", "-o", "pipefail", "-c"]
 
 # ---------------------------------------------------------------------------
 # Install build dependencies
 # ---------------------------------------------------------------------------
+# initramfs-tools: required by the AINIC install.sh below.
 RUN apt-get update && \
-    apt-get install jq dpkg-dev kmod xz-utils \
+    apt-get install jq dpkg-dev kmod xz-utils initramfs-tools \
     libfmt-dev libboost-all-dev \
     libibverbs-dev ibverbs-utils infiniband-diags -y
 
 # ---------------------------------------------------------------------------
 # Enviroment variables
 # ---------------------------------------------------------------------------
 ENV WORKDIR=/workspace
-ENV ROCM_PATH=/opt/rocm
-ENV MPI_PATH=/usr/lib/x86_64-linux-gnu/openmpi
+# ROCM_PATH/MPI_HOME already come from the base image; mirror MPI_HOME here
+# instead of hardcoding the old v26.3 OpenMPI path.
+ENV MPI_PATH=${MPI_HOME}
+
+# v26.4 moved ROCm to a pip-installed rocm-sdk-devel, so /opt/rocm no longer
+# exists, and rocm-sdk-devel/bin/amdclang++ can't find its sibling clang++
+# (now under llvm/bin/). rccl/amd-anp still hardcode both, so symlink them in
+# rather than patching those repos.
+RUN ln -sfn "${ROCM_PATH}" /opt/rocm && \
+    for f in "${ROCM_PATH}"/llvm/bin/clang*; do \
+        ln -sf "$f" "${ROCM_PATH}/bin/$(basename "$f")"; \
+    done
 
 # =============================== Build AINIC Driver ===============================
 # WARNING: Please ensure the following environment variables are correctly set:

README.md

@@ -91,7 +91,7 @@ primus-cli deps sync --dir ~/.cache/Primus/third_party
 
     ```bash
     # For Megatron-LM and TorchTitan backends
-    docker pull rocm/primus:v26.3
+    docker pull rocm/primus:v26.4
     # For MaxText backend
     docker pull rocm/jax-training:maxtext-v26.4-jax0.9.1-te2.12.0
     ```
@@ -102,7 +102,7 @@ primus-cli deps sync --dir ~/.cache/Primus/third_party
     git clone --recurse-submodules https://github.com/AMD-AGI/Primus.git
     cd Primus
     # checkout the branch for the specific release
-    git checkout release/v26.3
+    git checkout release/v26.4
     git submodule update --init --recursive
     ```
 
@@ -113,7 +113,7 @@ primus-cli deps sync --dir ~/.cache/Primus/third_party
     # NOTE: If your config downloads weights/tokenizer from Hugging Face Hub,
     #       you typically need to pass HF_TOKEN into the container.
     # Run in the Primus repository root directory
-    ./primus-cli container --image rocm/primus:v26.3 \
+    ./primus-cli container --image rocm/primus:v26.4 \
       --env HF_TOKEN="hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" \
       -- train pretrain --config examples/megatron/configs/MI300X/llama2_7B-BF16-pretrain.yaml
     ```
@@ -131,7 +131,7 @@ For more detailed usage instructions, see the [CLI User Guide](./docs/02-user-gu
     python -m venv primus-env
     source primus-env/bin/activate
     # Install Primus
-    pip install "primus==26.3.1" --no-deps --extra-index-url https://amd-agi.github.io/Primus/simple/
+    pip install "primus==26.4.0" --no-deps --extra-index-url https://amd-agi.github.io/Primus/simple/
 
     ```
 
@@ -142,7 +142,7 @@ For more detailed usage instructions, see the [CLI User Guide](./docs/02-user-gu
 2. **Run training in container using pip-installed Primus**
 
     ```bash
-    primus-cli container --image rocm/primus:v26.3 \
+    primus-cli container --image rocm/primus:v26.4 \
     --env HF_TOKEN="hf_xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" \
     --volume /path/to/your/data:/data  -- --log_file /data/run.log \
     -- train pretrain --config /data/your/config.yaml

benchmark/kernel/rccl/run_slurm.sh

@@ -10,7 +10,7 @@
 # Usage:
 #   DOCKER_IMAGE=<image> sbatch run_slurm.sh
 #   DOCKER_IMAGE=<image> NNODES=2 PARTITION=my-gpu sbatch run_slurm.sh
-#   DOCKER_IMAGE=rocm/primus:v26.3 NNODES=2 sbatch -N2 -w smci355-ccs-aus-n04-[25,29] -p Compute-DCPT ./run_slurm.sh
+#   DOCKER_IMAGE=rocm/primus:v26.4 NNODES=2 sbatch -N2 -w smci355-ccs-aus-n04-[25,29] -p Compute-DCPT ./run_slurm.sh
 #
 # Environment variables (all optional except DOCKER_IMAGE):
 #   DOCKER_IMAGE        Docker image to use (required)

benchmark/kernel/rccl/submit_pairs.sh

@@ -1,7 +1,7 @@
 #!/usr/bin/env bash
 set -euo pipefail
 
-DOCKER_IMAGE="rocm/primus:v26.3"
+DOCKER_IMAGE="rocm/primus:v26.4"
 #DOCKER_IMAGE="docker.gpuperf:5000/aai_2026_training/rocm/primus_megatron:v25.11_gpt_oss_sink"
 #DOCKER_IMAGE="docker.gpuperf:5000/gpuperf/primus:v26.1_sinkfa"
 NNODES=2

docs/01-getting-started/quickstart.md

@@ -65,7 +65,7 @@ Install Primus in a virtual environment:
 ```bash
 python -m venv primus-env
 source primus-env/bin/activate
-pip install "primus==26.3.1" --no-deps --extra-index-url https://amd-agi.github.io/Primus/simple/
+pip install "primus==26.4.0" --no-deps --extra-index-url https://amd-agi.github.io/Primus/simple/
 ```
 
 > **Note:** This installs only the Primus CLI into your virtual environment (under `site-packages`), without other dependencies. Third-party submodules are downloaded on the first run of the container, and the complete training software stack is provided in the AMD-published Docker images. You can launch `primus-cli` from any directory.

examples/README.md

@@ -49,7 +49,7 @@ We recommend using the official [rocm/megatron-lm Docker image](https://hub.dock
 
 ```bash
 # Pull the latest Docker image
-docker pull docker.io/rocm/primus:v26.3
+docker pull docker.io/rocm/primus:v26.4
 
 ```
 
@@ -126,7 +126,7 @@ Multi-node training is launched via **SLURM**.
 Specify the number of nodes and the model config:
 
 ```bash
-export DOCKER_IMAGE="docker.io/rocm/primus:v26.3"
+export DOCKER_IMAGE="docker.io/rocm/primus:v26.4"
 export NNODES=8
 
 # Example for megatron llama3.1_8B
@@ -292,7 +292,7 @@ When using the `create` command to start a new training workload, the following
 | `--gpu`        | Number of GPUs                                       | 8                                        |
 | `--exp`        | Path to experiment (training config) file (required) | —                                        |
 | `--data_path`  | Path to training data                                | —                                        |
-| `--image`      | Docker image to use                                  | `docker.io/rocm/primus:v26.3` |
+| `--image`      | Docker image to use                                  | `docker.io/rocm/primus:v26.4` |
 | `--hf_token`   | HuggingFace token                                    | Read from env var `HF_TOKEN`             |
 | `--workspace`  | Workspace name                                       | `primus-safe-pretrain`                   |
 | `--nodelist`   | Comma-separated list of node hostnames to run on     | —                                        |