From 35d436ef8d245237821c8915b9b074aeb8fefe45 Mon Sep 17 00:00:00 2001 From: yangzeyu <532183776@qq.com> Date: Mon, 20 Jul 2026 14:40:23 +0800 Subject: [PATCH 1/2] chore/add npu ci image build step Signed-off-by: yangzeyu <532183776@qq.com> --- .buildkite/npu_suites.py | 29 ++++-- .buildkite/pipeline-npu.yaml | 183 ++++++++++++++++++----------------- docker/Dockerfile.npu | 23 +++-- 3 files changed, 133 insertions(+), 102 deletions(-) diff --git a/.buildkite/npu_suites.py b/.buildkite/npu_suites.py index 986f4ffe..ab3f90c3 100644 --- a/.buildkite/npu_suites.py +++ b/.buildkite/npu_suites.py @@ -16,9 +16,10 @@ import json import os import subprocess +import sys NPU_QUEUE = "ascend-a3" -CI_IMAGE = os.environ.get("IMAGE_BUILD", "quay.io/ascend/vime:0.3.0-a3-vllm0.22.1rc1") +DEFAULT_CI_IMAGE = "quay.io/ascend/vime:0.3.0-a3-vllm0.22.1rc1" IMAGE_REGISTRY = "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry" IMAGE_NAME = "vime-ci-npu" VIME_IMAGE_TAG = os.environ.get("BUILDKITE_COMMIT", "latest") @@ -33,7 +34,7 @@ } -def selected_suites() -> list: +def _read_suite_values() -> list[str]: raw = os.environ.get("NPU_SUITES") if raw is None: try: @@ -45,8 +46,19 @@ def selected_suites() -> list: ).stdout except subprocess.CalledProcessError: raw = "" - values = [v.strip() for v in raw.replace(",", "\n").splitlines()] - unknown = [v for v in values if v and v not in SUITES] + return [v.strip() for v in raw.replace(",", "\n").splitlines()] + + +def _ci_image() -> str: + values = _read_suite_values() + if "image-build" in values: + return f"{IMAGE_REGISTRY}/{IMAGE_NAME}:{VIME_IMAGE_TAG}" + return DEFAULT_CI_IMAGE + + +def selected_suites() -> list: + values = _read_suite_values() + unknown = [v for v in values if v and v not in SUITES and v != "image-build"] if unknown: raise SystemExit(f"unknown suite(s) {unknown}; expected {sorted(SUITES)}") return [s for s in SUITES if s in values] @@ -87,7 +99,7 @@ def npu_step(suite: str, test_name: str, resource_class: str, extra_args: str, e "resource_class": resource_class, }, "timeout_in_minutes": 180, - "image": CI_IMAGE, + "image": _ci_image(), "plugins": [ { "kubernetes": { @@ -104,7 +116,12 @@ def npu_step(suite: str, test_name: str, resource_class: str, extra_args: str, e def main() -> None: steps = [npu_step(suite, *entry) for suite in selected_suites() for entry in SUITES[suite]] - print(json.dumps({"steps": steps}, indent=2)) + json_str = json.dumps({"steps": steps}, indent=2) + + print("--- Generated Pipeline JSON (debug):", file=sys.stderr) + print(json_str, file=sys.stderr) + + print(json_str) if __name__ == "__main__": diff --git a/.buildkite/pipeline-npu.yaml b/.buildkite/pipeline-npu.yaml index f2fd08a6..e101f994 100644 --- a/.buildkite/pipeline-npu.yaml +++ b/.buildkite/pipeline-npu.yaml @@ -45,97 +45,106 @@ steps: multiple: true required: true options: + - label: "run-npu-ci-image-build (test new image)" + value: image-build - label: "run-npu-ci-smk" value: smk - label: "run-npu-ci-nightly" value: nightly -# - label: ":buildkit: Build and Push NPU Test Image - A3" -# key: image-build-npu -# depends_on: pre-commit-npu -# if: __IMAGE_BUILD_IF__ == true -# timeout_in_minutes: 240 -# agents: -# queue: "ascend-a3" -# resource_class: "npu-2" -# plugins: -# - kubernetes: -# metadata: -# annotations: -# vault.hashicorp.com/agent-init-first: "true" -# vault.hashicorp.com/agent-inject: "true" -# vault.hashicorp.com/agent-inject-perms-ca.pem: "0400" -# vault.hashicorp.com/agent-inject-perms-cert.pem: "0400" -# vault.hashicorp.com/agent-inject-perms-config.json: "0400" -# vault.hashicorp.com/agent-inject-perms-key.pem: "0400" -# vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd -# vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd -# vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd -# vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd -# vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}" -# vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}" -# vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}" -# vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}" -# vault.hashicorp.com/agent-pre-populate-only: "true" -# vault.hashicorp.com/agent-run-as-group: "1000" -# vault.hashicorp.com/agent-run-as-user: "1000" -# vault.hashicorp.com/agent-service-account-token-volume-name: token-vol -# vault.hashicorp.com/role: ascend-gha-runners -# vault.hashicorp.com/secret-volume-path: /home/user/.docker/ -# vault.hashicorp.com/tls-skip-verify: "true" -# podSpecPatch: -# volumes: -# - name: token-vol -# projected: -# defaultMode: 420 -# sources: -# - serviceAccountToken: -# audience: api -# expirationSeconds: 600 -# path: token -# env: -# VIME_IMAGE_TAG: "${BUILDKITE_COMMIT}" -# IMAGE_NAME: "vime-ci-npu" -# IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry" -# BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234" -# command: | -# set -ex -# -# echo "--- Building and pushing NPU Test Image" -# echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VIME_IMAGE_TAG}" -# echo "buildkitd address: $${BUILDKITD_ADDR}" -# -# if ! command -v buildctl &> /dev/null; then -# echo "Installing buildctl..." -# mkdir -p /tmp/buildkit -# BUILDKIT_VERSION="v0.29.0" -# wget -q "https://gh-proxy.test.osinfra.cn/https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz -# tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit -# cp /tmp/buildkit/bin/buildctl /usr/local/bin/ -# fi -# -# sed -i '/^RUN git config --global http.sslVerify false/i RUN git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf "https://github.com/"' docker/Dockerfile.npu -# sed -i '/^# syntax=docker\/dockerfile:1\.7$$/d' docker/Dockerfile.npu -# -# export DOCKER_CONFIG=/home/user/.docker -# buildctl \ -# --addr="$${BUILDKITD_ADDR}" \ -# --tlscacert=/home/user/.docker/ca.pem \ -# --tlscert=/home/user/.docker/cert.pem \ -# --tlskey=/home/user/.docker/key.pem \ -# build \ -# --frontend dockerfile.v0 \ -# --local context=. \ -# --local dockerfile=./docker \ -# --opt filename=Dockerfile.npu \ -# --opt build-arg:APTMIRROR=http://cache-service.nginx-pypi-cache.svc.cluster.local:8081 \ -# --opt build-arg:PIP_INDEX_URL=http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \ -# --secret id=dockerconfig,src=/home/user/.docker/config.json \ -# --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VIME_IMAGE_TAG},push=true \ -# --progress=plain -# -# echo "--- Image pushed successfully" -# echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VIME_IMAGE_TAG}" + - label: ":buildkit: Build and Push NPU Test Image - A3" + key: image-build-npu + depends_on: + - pre-commit-npu + - npu-gate + timeout_in_minutes: 240 + agents: + queue: "ascend-a3" + resource_class: "npu-2" + plugins: + - kubernetes: + metadata: + annotations: + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-inject-perms-ca.pem: "0400" + vault.hashicorp.com/agent-inject-perms-cert.pem: "0400" + vault.hashicorp.com/agent-inject-perms-config.json: "0400" + vault.hashicorp.com/agent-inject-perms-key.pem: "0400" + vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}" + vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}" + vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}" + vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-run-as-group: "1000" + vault.hashicorp.com/agent-run-as-user: "1000" + vault.hashicorp.com/agent-service-account-token-volume-name: token-vol + vault.hashicorp.com/role: ascend-gha-runners + vault.hashicorp.com/secret-volume-path: /home/user/.docker/ + vault.hashicorp.com/tls-skip-verify: "true" + podSpecPatch: + volumes: + - name: token-vol + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: api + expirationSeconds: 600 + path: token + env: + VIME_IMAGE_TAG: "${BUILDKITE_COMMIT}" + IMAGE_NAME: "vime-ci-npu" + IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry" + BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234" + command: | + set -ex + NPU_SUITES=$$(buildkite-agent meta-data get "npu-suites" --default "") + if [[ "$$NPU_SUITES" != *"image-build"* ]]; then + echo "Skipping this step because image-build is not present in npu-suites." + exit 0 + fi + + echo "--- Building and pushing NPU Test Image" + echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VIME_IMAGE_TAG}" + echo "buildkitd address: $${BUILDKITD_ADDR}" + + if ! command -v buildctl &> /dev/null; then + echo "Installing buildctl..." + mkdir -p /tmp/buildkit + BUILDKIT_VERSION="v0.29.0" + wget -q "https://gh-proxy.test.osinfra.cn/https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz + tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit + cp /tmp/buildkit/bin/buildctl /usr/local/bin/ + fi + + sed -i '/^RUN git config --global http.sslVerify false/i RUN git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf "https://github.com/"' docker/Dockerfile.npu + sed -i '/^# syntax=docker\/dockerfile:1\.7$$/d' docker/Dockerfile.npu + + export DOCKER_CONFIG=/home/user/.docker + buildctl \ + --addr="$${BUILDKITD_ADDR}" \ + --tlscacert=/home/user/.docker/ca.pem \ + --tlscert=/home/user/.docker/cert.pem \ + --tlskey=/home/user/.docker/key.pem \ + build \ + --frontend dockerfile.v0 \ + --local context=. \ + --local dockerfile=./docker \ + --opt filename=Dockerfile.npu \ + --opt build-arg:BASE_IMAGE=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend/vllm-ascend \ + --opt build-arg:APTMIRROR=http://cache-service.nginx-pypi-cache.svc.cluster.local:8081 \ + --opt build-arg:PIP_INDEX_URL=http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \ + --secret id=dockerconfig,src=/home/user/.docker/config.json \ + --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VIME_IMAGE_TAG},push=true \ + --progress=plain + + echo "--- Image pushed successfully" + echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VIME_IMAGE_TAG}" - label: ":pipeline: upload NPU suites" key: upload-npu-suites @@ -147,7 +156,5 @@ steps: resource_class: "npu-2" timeout_in_minutes: 180 command: | - export IMAGE_BUILD="quay.io/ascend/vime:0.3.0-a3-vllm0.22.1rc1" - echo "IMAGE_BUILD: $${IMAGE_BUILD}" python .buildkite/npu_suites.py | buildkite-agent pipeline upload \ No newline at end of file diff --git a/docker/Dockerfile.npu b/docker/Dockerfile.npu index e672a419..95a758bc 100644 --- a/docker/Dockerfile.npu +++ b/docker/Dockerfile.npu @@ -1,7 +1,8 @@ # syntax=docker/dockerfile:1.7 -ARG BASE_IMAGE=quay.io/ascend/vllm-ascend:v0.22.1rc1-a3 -FROM ${BASE_IMAGE} +ARG BASE_IMAGE=quay.io/ascend/vllm-ascend +ARG BASE_IMAGE_TAG=v0.22.1rc1-a3 +FROM ${BASE_IMAGE}:${BASE_IMAGE_TAG} SHELL ["/bin/bash", "-o", "pipefail", "-c"] WORKDIR /root @@ -11,6 +12,8 @@ ARG MEGATRON_BRIDGE_COMMIT=3fd3768045422d0aa5c97e90a4e6c659aea9acb9 ARG MINDSPEED_COMMIT=fc63de5c48426dd019c3b3f39e65f5bdf56e4086 ARG MBRIDGE_COMMIT=89eb10887887bc74853f89a4de258c0702932a1c ARG SOC_VERSION="ascend910_9391" +ARG PIP_INDEX_URL="https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple" +ARG APTMIRROR="" ENV DEBIAN_FRONTEND=noninteractive \ PIP_NO_CACHE_DIR=1 \ @@ -33,21 +36,27 @@ COPY docker/npu_patch /tmp/npu_patch RUN git config --global http.sslVerify false # System and pip dependencies. -RUN sed -i '/ports\.ubuntu\.com/ {h;s|ports\.ubuntu\.com|mirrors.tuna.tsinghua.edu.cn|g;G}' /etc/apt/sources.list && \ +RUN if [ -n "$APTMIRROR" ];then sed -i "s@^\(deb.*\)https\?://[a-z0-9.-]*\.ubuntu\.com@\1$APTMIRROR@g" /etc/apt/sources.list; \ + else sed -i '/ports\.ubuntu\.com/ {h;s|ports\.ubuntu\.com|mirrors.tuna.tsinghua.edu.cn|g;G}' /etc/apt/sources.list; fi && \ apt-get update && \ apt-get install -y --no-install-recommends \ build-essential cmake curl git libnuma-dev ninja-build patch rsync wget && \ rm -rf /var/lib/apt/lists/* && \ - pip config set global.index-url \ - https://mirrors.tuna.tsinghua.edu.cn/pypi/web/simple && \ + pip config set global.index-url ${PIP_INDEX_URL} && \ + if [ "${PIP_INDEX_URL#http://}" != "${PIP_INDEX_URL}" ]; then pip config set global.trusted-host "$(echo ${PIP_INDEX_URL} | awk -F/ '{print $3}')"; fi && \ pip config set global.extra-index-url \ "https://download.pytorch.org/whl/cpu/ https://mirrors.huaweicloud.com/ascend/repos/pypi" && \ git config --global http.version HTTP/1.1 # vllm and vllm-ascend are installed editable (-e) in the base image; apply the # NPU colocate patches directly to their source trees. -RUN git -C /vllm-workspace/vllm apply --whitespace=nowarn \ +RUN git -C /vllm-workspace/vllm apply --check --whitespace=nowarn \ /tmp/npu_patch/vllm.patch && \ + git -C /vllm-workspace/vllm apply --whitespace=nowarn \ + /tmp/npu_patch/vllm.patch + +RUN git -C /vllm-workspace/vllm-ascend apply --check --whitespace=nowarn \ + /tmp/npu_patch/vllm-ascend.patch && \ git -C /vllm-workspace/vllm-ascend apply --whitespace=nowarn \ /tmp/npu_patch/vllm-ascend.patch @@ -100,8 +109,6 @@ RUN pip install --no-build-isolation "nvidia-modelopt[torch]>=0.37.0" && \ # ring_flash_attn is a CUDA extension and is not used on Ascend. RUN pip install \ - --index-url https://pypi.org/simple \ - --trusted-host pypi.org \ --constraint /tmp/vime-npu-constraints.txt \ --requirement /root/vime/requirements.txt && \ pip install --no-deps --no-build-isolation -e /root/vime From ab17afb785b9221f643802d1120a5f212fa2859e Mon Sep 17 00:00:00 2001 From: yangzeyu <532183776@qq.com> Date: Tue, 21 Jul 2026 19:40:54 +0800 Subject: [PATCH 2/2] add ci image build test Signed-off-by: yangzeyu <532183776@qq.com> --- .buildkite/README-NPU-CI.md | 37 +++++++--- .buildkite/npu_suites.py | 9 ++- .buildkite/pipeline-npu-image.yaml | 89 ++++++++++++++++++++++ .buildkite/pipeline-npu.yaml | 114 ++++------------------------- 4 files changed, 138 insertions(+), 111 deletions(-) create mode 100644 .buildkite/pipeline-npu-image.yaml diff --git a/.buildkite/README-NPU-CI.md b/.buildkite/README-NPU-CI.md index 55b7179c..62fe7294 100644 --- a/.buildkite/README-NPU-CI.md +++ b/.buildkite/README-NPU-CI.md @@ -12,9 +12,23 @@ Three steps run in order: **`pre-commit-npu`** runs the pre-commit gate on all files, always. -**`image-build-npu`** builds and pushes the NPU test image via buildctl/buildkit. -It only runs when `Dockerfile.npu` has changed (PR trigger) or on a scheduled -build — otherwise the step is skipped and the pre-built default image is used. +**`npu-gate`** is a block step that pauses the pipeline for manual interaction. +It appears only on PR triggers (`build.source != "schedule"`). You select which +NPU suites to run via a multi-select field: + +- **`image-build`** — triggers a fresh image build in the `image-build-npu` step; + when selected, `smk` is automatically included and all test steps use the newly + built image instead of the pre-built default. +- **`smk`** — runs the smoke test suite. +- **`nightly`** — runs the nightly test suite. + +**`image-build-npu`** the `image-build-npu` step will build and +push a fresh NPU test image tagged with the current commit. When `image-build` +is selected, all test steps generated by `upload-npu-suites` use the newly built +image instead of the pre-built default image. + +This allows testing code changes that require an updated NPU environment +(e.g., modifications to `docker/Dockerfile.npu`) before they are merged. **`upload-npu-suites`** reads the `NPU_SUITES` environment variable (or `buildkite-agent meta-data get npu-suites` for PR triggers) and generates @@ -22,21 +36,22 @@ individual test jobs via [`npu_suites.py`](./npu_suites.py). ## Triggers -The pipeline supports three trigger modes: +The pipeline supports two trigger modes: -**PR trigger.** On a PR trigger, the pre-built default image is used and -test suites are selected manually via the block step. +**PR trigger.** The `npu-gate` block step appears for manual suite selection. +Suites not selected in the block step are skipped. The `image-build-npu` step +builds a new image only when `image-build` is chosen in the block. -**Schedule trigger.** On a scheduled build, a new image is always built -regardless of file changes. The suites to run are determined by the +**Schedule trigger.** There is no block step — the `npu-gate` step is omitted +entirely. A new image is **always** built, and the suites are determined by the `NPU_SUITES` environment variable (set in the scheduled build's pipeline -configuration), which selects the corresponding entries from the `SUITES` dict. +configuration), which lists suite names from the `SUITES` dict. ## Adding a test Suites and test mappings are defined in [`npu_suites.py`](./npu_suites.py). Two -suites are predefined — `smk` (always runs) and `nightly` (runs on schedule or -with the `run-ci-npu-nightly` label). +suites are predefined — `smk` (runs with the `run-ci-npu-smk` label) and `nightly` +(runs on schedule or with the `run-ci-npu-nightly` label). Each entry is a 4-tuple: diff --git a/.buildkite/npu_suites.py b/.buildkite/npu_suites.py index ab3f90c3..3eac0ebf 100644 --- a/.buildkite/npu_suites.py +++ b/.buildkite/npu_suites.py @@ -23,6 +23,7 @@ IMAGE_REGISTRY = "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry" IMAGE_NAME = "vime-ci-npu" VIME_IMAGE_TAG = os.environ.get("BUILDKITE_COMMIT", "latest") +BUILDKITE_SOURCE = os.environ.get("BUILDKITE_SOURCE", "") # (test_name, resource_class, extra_args, env_overrides) SUITES = { @@ -51,7 +52,7 @@ def _read_suite_values() -> list[str]: def _ci_image() -> str: values = _read_suite_values() - if "image-build" in values: + if ("image-build" in values) or (BUILDKITE_SOURCE == "schedule"): return f"{IMAGE_REGISTRY}/{IMAGE_NAME}:{VIME_IMAGE_TAG}" return DEFAULT_CI_IMAGE @@ -61,6 +62,9 @@ def selected_suites() -> list: unknown = [v for v in values if v and v not in SUITES and v != "image-build"] if unknown: raise SystemExit(f"unknown suite(s) {unknown}; expected {sorted(SUITES)}") + if "image-build" in values: + # image-build auto-includes smk tests + values.append("smk") return [s for s in SUITES if s in values] @@ -93,6 +97,7 @@ def npu_step(suite: str, test_name: str, resource_class: str, extra_args: str, e label = f":fire: {suite}: {test_name}{' ' + extra_args if extra_args else ''}" step = { "label": label, + "depends_on": "image-build-npu", "command": command, "agents": { "queue": NPU_QUEUE, @@ -118,7 +123,7 @@ def main() -> None: steps = [npu_step(suite, *entry) for suite in selected_suites() for entry in SUITES[suite]] json_str = json.dumps({"steps": steps}, indent=2) - print("--- Generated Pipeline JSON (debug):", file=sys.stderr) + print("--- Generated Pipeline JSON:", file=sys.stderr) print(json_str, file=sys.stderr) print(json_str) diff --git a/.buildkite/pipeline-npu-image.yaml b/.buildkite/pipeline-npu-image.yaml new file mode 100644 index 00000000..8f259b59 --- /dev/null +++ b/.buildkite/pipeline-npu-image.yaml @@ -0,0 +1,89 @@ +steps: + - label: ":docker: Build and Push NPU Test Image - A3" + key: image-build-npu + depends_on: + - pre-commit-npu + - npu-gate + timeout_in_minutes: 240 + skip: __SKIP_IMAGE_BUILD__ + agents: + queue: "ascend-a3" + resource_class: "npu-2" + plugins: + - kubernetes: + metadata: + annotations: + vault.hashicorp.com/agent-init-first: "true" + vault.hashicorp.com/agent-inject: "true" + vault.hashicorp.com/agent-inject-perms-ca.pem: "0400" + vault.hashicorp.com/agent-inject-perms-cert.pem: "0400" + vault.hashicorp.com/agent-inject-perms-config.json: "0400" + vault.hashicorp.com/agent-inject-perms-key.pem: "0400" + vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd + vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}" + vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}" + vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}" + vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}" + vault.hashicorp.com/agent-pre-populate-only: "true" + vault.hashicorp.com/agent-run-as-group: "1000" + vault.hashicorp.com/agent-run-as-user: "1000" + vault.hashicorp.com/agent-service-account-token-volume-name: token-vol + vault.hashicorp.com/role: ascend-gha-runners + vault.hashicorp.com/secret-volume-path: /home/user/.docker/ + vault.hashicorp.com/tls-skip-verify: "true" + podSpecPatch: + volumes: + - name: token-vol + projected: + defaultMode: 420 + sources: + - serviceAccountToken: + audience: api + expirationSeconds: 600 + path: token + env: + VIME_IMAGE_TAG: "${BUILDKITE_COMMIT}" + IMAGE_NAME: "vime-ci-npu" + IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry" + BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234" + command: | + set -ex + echo "--- Building and pushing NPU Test Image" + echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VIME_IMAGE_TAG}" + echo "buildkitd address: $${BUILDKITD_ADDR}" + + if ! command -v buildctl &> /dev/null; then + echo "Installing buildctl..." + mkdir -p /tmp/buildkit + BUILDKIT_VERSION="v0.29.0" + wget -q "https://gh-proxy.test.osinfra.cn/https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz + tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit + cp /tmp/buildkit/bin/buildctl /usr/local/bin/ + fi + + sed -i '/^RUN git config --global http.sslVerify false/i RUN git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf "https://github.com/"' docker/Dockerfile.npu + sed -i '/^# syntax=docker\/dockerfile:1\.7$$/d' docker/Dockerfile.npu + + export DOCKER_CONFIG=/home/user/.docker + buildctl \ + --addr="$${BUILDKITD_ADDR}" \ + --tlscacert=/home/user/.docker/ca.pem \ + --tlscert=/home/user/.docker/cert.pem \ + --tlskey=/home/user/.docker/key.pem \ + build \ + --frontend dockerfile.v0 \ + --local context=. \ + --local dockerfile=./docker \ + --opt filename=Dockerfile.npu \ + --opt build-arg:BASE_IMAGE=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend/vllm-ascend \ + --opt build-arg:APTMIRROR=http://cache-service.nginx-pypi-cache.svc.cluster.local:8081 \ + --opt build-arg:PIP_INDEX_URL=http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \ + --secret id=dockerconfig,src=/home/user/.docker/config.json \ + --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VIME_IMAGE_TAG},push=true \ + --progress=plain + + echo "--- Image pushed successfully" + echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VIME_IMAGE_TAG}" \ No newline at end of file diff --git a/.buildkite/pipeline-npu.yaml b/.buildkite/pipeline-npu.yaml index e101f994..1414faba 100644 --- a/.buildkite/pipeline-npu.yaml +++ b/.buildkite/pipeline-npu.yaml @@ -45,116 +45,34 @@ steps: multiple: true required: true options: - - label: "run-npu-ci-image-build (test new image)" + - label: "run-npu-ci-image-build (test new image, auto include smk)" value: image-build - label: "run-npu-ci-smk" value: smk - label: "run-npu-ci-nightly" value: nightly - - label: ":buildkit: Build and Push NPU Test Image - A3" - key: image-build-npu - depends_on: - - pre-commit-npu - - npu-gate - timeout_in_minutes: 240 - agents: - queue: "ascend-a3" - resource_class: "npu-2" - plugins: - - kubernetes: - metadata: - annotations: - vault.hashicorp.com/agent-init-first: "true" - vault.hashicorp.com/agent-inject: "true" - vault.hashicorp.com/agent-inject-perms-ca.pem: "0400" - vault.hashicorp.com/agent-inject-perms-cert.pem: "0400" - vault.hashicorp.com/agent-inject-perms-config.json: "0400" - vault.hashicorp.com/agent-inject-perms-key.pem: "0400" - vault.hashicorp.com/agent-inject-secret-ca.pem: internal/data/ascend/buildkitd - vault.hashicorp.com/agent-inject-secret-cert.pem: internal/data/ascend/buildkitd - vault.hashicorp.com/agent-inject-secret-config.json: internal/data/ascend/buildkitd - vault.hashicorp.com/agent-inject-secret-key.pem: internal/data/ascend/buildkitd - vault.hashicorp.com/agent-inject-template-ca.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.RootCA }}\n{{- end }}" - vault.hashicorp.com/agent-inject-template-cert.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaCert }}\n{{- end }}" - vault.hashicorp.com/agent-inject-template-config.json: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.dockerConfig }}\n{{- end }}" - vault.hashicorp.com/agent-inject-template-key.pem: "{{- with secret \"internal/data/ascend/buildkitd\" -}}\n{{ .Data.data.ClientCaKey }}\n{{- end }}" - vault.hashicorp.com/agent-pre-populate-only: "true" - vault.hashicorp.com/agent-run-as-group: "1000" - vault.hashicorp.com/agent-run-as-user: "1000" - vault.hashicorp.com/agent-service-account-token-volume-name: token-vol - vault.hashicorp.com/role: ascend-gha-runners - vault.hashicorp.com/secret-volume-path: /home/user/.docker/ - vault.hashicorp.com/tls-skip-verify: "true" - podSpecPatch: - volumes: - - name: token-vol - projected: - defaultMode: 420 - sources: - - serviceAccountToken: - audience: api - expirationSeconds: 600 - path: token - env: - VIME_IMAGE_TAG: "${BUILDKITE_COMMIT}" - IMAGE_NAME: "vime-ci-npu" - IMAGE_REGISTRY: "swr.cn-southwest-2.myhuaweicloud.com/modelfoundry" - BUILDKITD_ADDR: "tcp://buildkitd-service.buildkitd:1234" - command: | - set -ex - NPU_SUITES=$$(buildkite-agent meta-data get "npu-suites" --default "") - if [[ "$$NPU_SUITES" != *"image-build"* ]]; then - echo "Skipping this step because image-build is not present in npu-suites." - exit 0 - fi - - echo "--- Building and pushing NPU Test Image" - echo "Image: $${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VIME_IMAGE_TAG}" - echo "buildkitd address: $${BUILDKITD_ADDR}" - - if ! command -v buildctl &> /dev/null; then - echo "Installing buildctl..." - mkdir -p /tmp/buildkit - BUILDKIT_VERSION="v0.29.0" - wget -q "https://gh-proxy.test.osinfra.cn/https://github.com/moby/buildkit/releases/download/$${BUILDKIT_VERSION}/buildkit-$${BUILDKIT_VERSION}.linux-arm64.tar.gz" -O /tmp/buildkit.tar.gz - tar -xzf /tmp/buildkit.tar.gz -C /tmp/buildkit - cp /tmp/buildkit/bin/buildctl /usr/local/bin/ - fi - - sed -i '/^RUN git config --global http.sslVerify false/i RUN git config --global url."https://gh-proxy.test.osinfra.cn/https://github.com/".insteadOf "https://github.com/"' docker/Dockerfile.npu - sed -i '/^# syntax=docker\/dockerfile:1\.7$$/d' docker/Dockerfile.npu - - export DOCKER_CONFIG=/home/user/.docker - buildctl \ - --addr="$${BUILDKITD_ADDR}" \ - --tlscacert=/home/user/.docker/ca.pem \ - --tlscert=/home/user/.docker/cert.pem \ - --tlskey=/home/user/.docker/key.pem \ - build \ - --frontend dockerfile.v0 \ - --local context=. \ - --local dockerfile=./docker \ - --opt filename=Dockerfile.npu \ - --opt build-arg:BASE_IMAGE=swr.cn-southwest-2.myhuaweicloud.com/base_image/ascend-ci/vllm-ascend/vllm-ascend \ - --opt build-arg:APTMIRROR=http://cache-service.nginx-pypi-cache.svc.cluster.local:8081 \ - --opt build-arg:PIP_INDEX_URL=http://cache-service.nginx-pypi-cache.svc.cluster.local/pypi/simple \ - --secret id=dockerconfig,src=/home/user/.docker/config.json \ - --output type=image,name=$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VIME_IMAGE_TAG},push=true \ - --progress=plain - - echo "--- Image pushed successfully" - echo "$${IMAGE_REGISTRY}/$${IMAGE_NAME}:$${VIME_IMAGE_TAG}" - - label: ":pipeline: upload NPU suites" key: upload-npu-suites depends_on: - npu-gate - - pre-commit-npu agents: queue: "ascend-a3" resource_class: "npu-2" timeout_in_minutes: 180 command: | - python .buildkite/npu_suites.py | buildkite-agent pipeline upload - \ No newline at end of file + NPU_SUITES=$$(buildkite-agent meta-data get "npu-suites" --default "") + if [[ "$$BUILDKITE_SOURCE" == "schedule" ]]; then + echo "Scheduled build — building the image." + echo "BUILDKITE_SOURCE: $$BUILDKITE_SOURCE" + SKIP_IMAGE_BUILD=false + elif [[ "$$NPU_SUITES" == *"image-build"* ]]; then + echo "image-build selected — building the image." + echo "NPU_SUITES: $$NPU_SUITES" + SKIP_IMAGE_BUILD=false + else + echo "Skipping image build because image-build is not present in the npu-suites." + SKIP_IMAGE_BUILD=true + fi + sed -e "s/__SKIP_IMAGE_BUILD__/$${SKIP_IMAGE_BUILD}/g" .buildkite/pipeline-npu-image.yaml | buildkite-agent pipeline upload + python .buildkite/npu_suites.py | buildkite-agent pipeline upload \ No newline at end of file