HRX CI: run on self-hosted ROCm GPU runners for gfx1151/gfx1201 #108
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: HRX CI | |
| on: | |
| push: | |
| branches: | |
| - hrx-integration | |
| pull_request: | |
| types: [opened, synchronize, reopened] | |
| branches: | |
| - hrx-integration | |
| permissions: | |
| contents: read | |
| actions: read | |
| concurrency: | |
| # EXPERIMENT (gfx1151-only sweep): unique group per run so spaced PR-synchronize | |
| # pushes get independent runs (a shared group auto-cancels the middle pending | |
| # runs even with cancel-in-progress: false). Revert to the head_ref/run_id | |
| # ternary + cancel-in-progress: true after the sweep. | |
| group: ${{ github.workflow }}-${{ github.run_id }} | |
| cancel-in-progress: false | |
| jobs: | |
| hrx-build: | |
| name: ${{ matrix.name }} (${{ matrix.gpu_target }}) | |
| strategy: | |
| # Per-arch failures are independent; keep both signals. | |
| fail-fast: false | |
| matrix: | |
| include: | |
| - name: gfx1151_strix-halo | |
| gpu_target: gfx1151 | |
| runs_on: linux-gfx1151-gpu-rocm | |
| extra_rocm_artifacts: rocwmma_dev_gfx1151 | |
| # - name: gfx1201_9070 | |
| # gpu_target: gfx1201 | |
| # runs_on: linux-gfx120X-gpu-rocm | |
| # extra_rocm_artifacts: rocwmma_dev_gfx120X-all | |
| runs-on: ${{ matrix.runs_on }} | |
| timeout-minutes: 15 | |
| defaults: | |
| run: | |
| # --noprofile skips loading ~/.bash_profile | |
| # --norc skips loading ~/.bashrc | |
| # -exo pipefail: | |
| # -e exit immediately if any command fails | |
| # -x print each command before executing | |
| # -o pipefail ensure pipeline fails if any command in it fails | |
| # {0} github actions placeholder for the command to run | |
| shell: bash --noprofile --norc -exo pipefail {0} | |
| container: | |
| image: ghcr.io/rocm/no_rocm_image_ubuntu24_04@sha256:fba5f55a122dbb15925e98c51fe65bffe88c36e11ebb25b73daf2bea04202dc3 | |
| # --user 0:0: actions running inside the container need to write to some | |
| # files set up outside the container by the runner agent. In | |
| # June 2026 some runner agents set those files up with | |
| # permissions for a "tester" user with UID/GID 1001, which | |
| # matches the user in no_rocm_image_ubuntu24_04, and some are | |
| # set up as root; accessing a file owned by root with user | |
| # "tester" gives an EACCES. Running as root is the common | |
| # denominator. | |
| # --device kfd/dri: GPU access. | |
| options: >- | |
| --user 0:0 | |
| --device /dev/kfd | |
| --device /dev/dri | |
| env: | |
| # Write out of tree build/install to a non-bind-mounted directory. In June | |
| # 2026 a debug test showed some runners had assets dated from 6 months ago | |
| # in the bind-mounted github workspace, it seems like nothing cleans it up | |
| # currently. Writing to an non-bind-mounted directory ensures that | |
| # rocm-artifacts/builds are cleaned up when docker container is removed. | |
| HRX_WORK_DIR: /work | |
| # Public location for bench tooling (rsuderman/llamacpp_ci and fork | |
| # AaronStGeorge/llamacpp_ci) | |
| #TODO: switch to ROCm/llamacpp-hrx-bench once it is open sourced. | |
| BENCH_REPOSITORY: AaronStGeorge/llamacpp_ci | |
| BENCH_REF: 'runner-health' | |
| # actions/checkout@v6 does auto-cleanup, an in-tree build would be | |
| # auto-cleaned as well. | |
| BENCH_DIR: ${{ github.workspace }}/bench | |
| LLAMA_SRC_DIR: ${{ github.workspace }}/llama-src | |
| # Build tree off the bind mount too (source stays under the workspace). | |
| LLAMA_BUILD_DIR: /work/llama-build | |
| HRX_EXTRA_ROCM_ARTIFACTS: ${{ matrix.extra_rocm_artifacts }} | |
| GGML_HRX_AMDGPU_TARGETS: ${{ matrix.gpu_target }} | |
| CCACHE_COMPILERCHECK: content | |
| steps: | |
| # Logged first (if: always(), before any checkout) so the specific | |
| # self-hosted runner behind a failure is greppable from the job log via | |
| # RUNNER_NAME, without querying the GitHub API. NOTE: a gfx1201 | |
| # "Initialize containers" failure happens before any step runs, so this | |
| # block cannot capture that mode -- only the API runner_name identifies it. | |
| - name: Debug (runner identity) | |
| if: always() | |
| run: | | |
| set +e | |
| echo "===== RUNNER IDENTITY =====" | |
| echo "RUNNER_NAME=${RUNNER_NAME}" | |
| echo "RUNNER_OS=${RUNNER_OS} RUNNER_ENVIRONMENT=${RUNNER_ENVIRONMENT}" | |
| echo "GITHUB_RUN_ID=${GITHUB_RUN_ID} attempt=${GITHUB_RUN_ATTEMPT} job=${GITHUB_JOB}" | |
| echo "matrix.name=${{ matrix.name }} gpu_target=${{ matrix.gpu_target }} runs_on=${{ matrix.runs_on }}" | |
| echo "hostname=$(hostname)" | |
| uname -a || true | |
| cat /etc/os-release || true | |
| id || true | |
| ls -l /dev/kfd || echo "MISSING /dev/kfd" | |
| ls -l /dev/dri || echo "MISSING /dev/dri" | |
| lspci | grep -iE 'vga|display|amd/ati' || echo "lspci unavailable / no GPU match" | |
| env | grep -E '^(RUNNER_|GITHUB_)' | sort || true | |
| echo "===== END RUNNER IDENTITY =====" | |
| true | |
| - name: Checkout llama.cpp (under test) | |
| uses: actions/checkout@v6 | |
| with: | |
| path: llama-src | |
| - name: Checkout bench tooling | |
| uses: actions/checkout@v6 | |
| with: | |
| repository: ${{ env.BENCH_REPOSITORY }} | |
| ref: ${{ env.BENCH_REF }} | |
| path: bench | |
| - name: Install ROCm build dependencies | |
| run: "${BENCH_DIR}/scripts/hrx/install-rocm-deps.sh" | |
| - name: ccache | |
| uses: ggml-org/ccache-action@v1.2.21 | |
| with: | |
| key: ${{ matrix.gpu_target }} | |
| evict-old-files: 1d | |
| save: ${{ github.event_name == 'pull_request' }} | |
| - name: Checkout HRX | |
| run: "${BENCH_DIR}/scripts/hrx/checkout-hrx.sh" | |
| - name: Fetch ROCm assets | |
| run: "${BENCH_DIR}/scripts/hrx/fetch-rocm-assets.sh" | |
| # Dump GPU/board identity (serial/VBIOS/fw via rocminfo/amd-smi/rocm-smi) | |
| # and fail fast on gross breakage (no /dev/kfd, dead rocminfo, expected | |
| # gfx arch not enumerated) before the costly HRX/llama build. Runs here | |
| # because the diag binaries only exist in the composed prefix after the | |
| # fetch step. Does NOT detect the gfx1151 coherence NaN flake. | |
| - name: GPU health check | |
| timeout-minutes: 3 | |
| run: "${BENCH_DIR}/scripts/hrx/runner-health.sh" | |
| - name: Build HRX | |
| run: "${BENCH_DIR}/scripts/hrx/build-hrx.sh" | |
| - name: Validate HRX | |
| # hrx-info has been observed hanging on some runners; fail fast. | |
| timeout-minutes: 5 | |
| run: "${BENCH_DIR}/scripts/hrx/validate-hrx.sh" | |
| - name: Build llama.cpp with HRX | |
| run: "${BENCH_DIR}/scripts/hrx/build-llama-hrx.sh" | |
| # Diagnostic: run the failing MUL_MAT config N times in THIS job to measure | |
| # the per-RUNNER failure rate -- does a bad machine fail every iteration or | |
| # only some? Advisory (continue-on-error, runs BEFORE the gating step so it | |
| # still executes on a failing machine). Correlate the printed count with | |
| # RUNNER_NAME (see the Debug step). Tune N via the MULMAT_LOOP_N var. | |
| - name: MUL_MAT failure-rate loop (diagnostic) | |
| continue-on-error: true | |
| timeout-minutes: 20 | |
| run: | | |
| . "${BENCH_DIR}/scripts/hrx/env.sh" | |
| . "${BENCH_DIR}/scripts/hrx/runtime-env.sh" | |
| set +e # count pass/fail from the output, don't let a NaN exit abort | |
| TB="${LLAMA_BUILD_DIR}/bin/test-backend-ops" | |
| TF="${BENCH_DIR}/benchmark-configs/test/mul_mat_f16.txt" | |
| N="${MULMAT_LOOP_N:-50}" | |
| pass=0; fail=0; other=0 | |
| for i in $(seq 1 "$N"); do | |
| out="$("$TB" test -o MUL_MAT -b HRX0 --test-file "$TF" 2>&1 | sed 's/\x1b\[[0-9;]*m//g')" | |
| if echo "$out" | grep -q 'NaN at index'; then | |
| fail=$((fail+1)) | |
| [ "$fail" -le 3 ] && echo "iter $i FAIL: $(echo "$out" | grep -m1 'NaN at index')" | |
| elif echo "$out" | grep -q '1/1 tests passed'; then | |
| pass=$((pass+1)) | |
| else | |
| other=$((other+1)) | |
| [ "$other" -le 2 ] && echo "iter $i OTHER: $(echo "$out" | tail -2 | tr '\n' ' ')" | |
| fi | |
| done | |
| echo "MULMAT_RATE runner=${RUNNER_NAME} fail=${fail} pass=${pass} other=${other} total=${N}" | |
| echo "::notice::MUL_MAT rate on ${RUNNER_NAME}: ${fail}/${N} FAIL (pass=${pass}, other=${other})" | |
| - name: Run sample MUL_MAT correctness config on HRX | |
| timeout-minutes: 15 | |
| run: | | |
| . "${BENCH_DIR}/scripts/hrx/env.sh" | |
| . "${BENCH_DIR}/scripts/hrx/runtime-env.sh" | |
| "${BENCH_DIR}/tools/run-op-test.py" \ | |
| --test-backend-ops "${LLAMA_BUILD_DIR}/bin/test-backend-ops" \ | |
| --test-file "${BENCH_DIR}/benchmark-configs/test/mul_mat_f16.txt" \ | |
| --op MUL_MAT \ | |
| --backend HRX0 \ | |
| --output benchmark-results/sample-mul-mat-f16-hrx-test.jsonl | |
| - name: Run sample MUL_MAT benchmark config on HRX | |
| timeout-minutes: 15 | |
| run: | | |
| . "${BENCH_DIR}/scripts/hrx/env.sh" | |
| . "${BENCH_DIR}/scripts/hrx/runtime-env.sh" | |
| "${BENCH_DIR}/tools/run-op-perf.py" \ | |
| --test-backend-ops "${LLAMA_BUILD_DIR}/bin/test-backend-ops" \ | |
| --test-file "${BENCH_DIR}/benchmark-configs/test/mul_mat_f16.txt" \ | |
| --op MUL_MAT \ | |
| --backend HRX0 \ | |
| --output benchmark-results/sample-mul-mat-f16-hrx-perf.jsonl | |
| - name: Upload benchmark results | |
| uses: actions/upload-artifact@v5 | |
| with: | |
| name: benchmark-results-${{ matrix.name }} | |
| path: benchmark-results/ |