diff --git a/.github/scripts/dispatch.py b/.github/scripts/dispatch.py index d4a25558..c316c34c 100644 --- a/.github/scripts/dispatch.py +++ b/.github/scripts/dispatch.py @@ -166,6 +166,7 @@ def _build_inputs( head_sha: str, target_branch: str, upload: bool, + bot_comment_id: str, ) -> dict: inputs = { "kernel_name": kernel_name, @@ -184,6 +185,8 @@ def _build_inputs( inputs["target_branch"] = target_branch if not upload: inputs["upload"] = "false" + if upload and bot_comment_id: + inputs["bot_comment_id"] = bot_comment_id return inputs @@ -247,6 +250,7 @@ def _plan_build_actions( head_sha: str, target_branch: str, upload: bool, + bot_comment_id: str, requested_backends: list[str] | None = None, ) -> None: backends = read_backends(kernel_name) @@ -299,6 +303,7 @@ def _plan_build_actions( head_sha=head_sha, target_branch=target_branch, upload=upload, + bot_comment_id=bot_comment_id, ), }, description=f"for kernel `{kernel_name}` on ref `{ref}`", @@ -319,6 +324,7 @@ def plan_dispatch( head_sha: str = "", target_branch: str = "", upload: bool = True, + bot_comment_id: str = "", run_security: bool = False, security_only: bool = False, requested_backends: list[str] | None = None, @@ -339,6 +345,7 @@ def plan_dispatch( head_sha=head_sha, target_branch=target_branch, upload=upload, + bot_comment_id=bot_comment_id, requested_backends=requested_backends, ) @@ -487,6 +494,7 @@ def dispatch( head_sha: str = "", target_branch: str = "", upload: bool = True, + bot_comment_id: str = "", run_security: bool = False, security_only: bool = False, requested_backends: list[str] | None = None, @@ -509,6 +517,7 @@ def dispatch( head_sha=head_sha, target_branch=target_branch, upload=upload, + bot_comment_id=bot_comment_id, run_security=run_security, security_only=security_only, requested_backends=requested_backends, @@ -607,6 +616,11 @@ def main() -> int: default="", help="PR head SHA for commit status reporting", ) + parser.add_argument( + "--bot-comment-id", + default="", + help="Issue comment ID to update with Hub upload links", + ) parser.add_argument( "--target-branch", default="", @@ -675,6 +689,7 @@ def main() -> int: skip_build=args.skip_build, pr_number=args.pr_number, head_sha=args.head_sha, + bot_comment_id=args.bot_comment_id, target_branch=args.target_branch, upload=not args.no_upload, run_security=args.security, diff --git a/.github/scripts/format_hub_upload_comment.py b/.github/scripts/format_hub_upload_comment.py new file mode 100644 index 00000000..59cb6449 --- /dev/null +++ b/.github/scripts/format_hub_upload_comment.py @@ -0,0 +1,77 @@ +#!/usr/bin/env python3 +# Input: +# [--base ] ... +# +# Output: +# A Hub upload section, or with that section updated. +# +# Example: +# python3 .github/scripts/format_hub_upload_comment.py msa kernels-community "$RUNNER_TEMP"/upload*.json +# python3 .github/scripts/format_hub_upload_comment.py --base "$BODY" msa kernels-community "$RUNNER_TEMP"/upload*.json +# python3 .github/scripts/format_hub_upload_comment.py msa kernels-community '{"pull_requests":[{"url":"https://hf.co/kernels/MiniMaxAI/msa/discussions/1"}]}' +import argparse +import json +from pathlib import Path + +from hub_pr_upload_args import repo_id + +SECTION = "Hub uploads:" + + +def load_json(value): + if not value.lstrip().startswith(("{", "[")): + path = Path(value) + if path.is_file(): + with open(path) as f: + return json.load(f) + return json.loads(value) + + +def dedupe(lines): + return list(dict.fromkeys(lines)) + + +def upload_lines(kernel, repo_prefix, uploads): + urls = [ + pr["url"] + for upload in uploads + for pr in load_json(upload).get("pull_requests", []) + ] + lines = [ + f"- Hub repo: https://huggingface.co/kernels/{repo_id(kernel, repo_prefix)}" + ] + lines.extend(f"- Hub pull request: {url}" for url in urls) + return dedupe(lines) + + +def update_comment(base, lines): + body_lines = base.rstrip().splitlines() + if SECTION in body_lines: + section = body_lines.index(SECTION) + prefix = body_lines[:section] + existing = [line for line in body_lines[section + 1 :] if line.startswith("- ")] + else: + prefix = body_lines + existing = [] + body = "\n".join(prefix).rstrip() + section = "\n".join([SECTION, *dedupe(existing + lines)]) + return f"{body}\n\n{section}".strip() + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("--base") + parser.add_argument("kernel") + parser.add_argument("repo_prefix") + parser.add_argument("uploads", nargs="+") + args = parser.parse_args() + + lines = upload_lines(args.kernel, args.repo_prefix, args.uploads) + if args.base is not None: + print(update_comment(args.base, lines)) + else: + print("\n".join([SECTION, *lines])) + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/.github/scripts/hub_pr_upload_args.py b/.github/scripts/hub_pr_upload_args.py new file mode 100644 index 00000000..1f91b801 --- /dev/null +++ b/.github/scripts/hub_pr_upload_args.py @@ -0,0 +1,35 @@ +#!/usr/bin/env python3 +# Input: +# +# +# Output: +# The effective Hub repo-id, or upload args using --create-pr for external repos. +# +# Example: +# python3 .github/scripts/hub_pr_upload_args.py upload-args msa kernels-community +import sys +import tomllib +from pathlib import Path + +COMMUNITY = "kernels-community" +ROOT = Path(__file__).resolve().parents[2] + + +def external_repo_id(kernel): + with open(ROOT / kernel / "build.toml", "rb") as f: + repo_id = tomllib.load(f).get("general", {}).get("hub", {}).get("repo-id") + return repo_id if isinstance(repo_id, str) and repo_id and not repo_id.startswith(f"{COMMUNITY}/") else "" + + +def repo_id(kernel, repo_prefix): + return external_repo_id(kernel) or f"{repo_prefix}/{kernel}" + + +if __name__ == "__main__": + mode, kernel, repo_prefix = sys.argv[1:] + if mode == "repo-id": + print(repo_id(kernel, repo_prefix)) + elif mode == "upload-args": + print("--create-pr" if external_repo_id(kernel) else f"--repo-id {repo_prefix}/{kernel}") + else: + sys.exit(f"unknown mode: {mode}") diff --git a/.github/scripts/pr_comment_kernel_bot.py b/.github/scripts/pr_comment_kernel_bot.py index 12de8c9e..b76dbf9c 100644 --- a/.github/scripts/pr_comment_kernel_bot.py +++ b/.github/scripts/pr_comment_kernel_bot.py @@ -904,6 +904,7 @@ def main(*, dry_run: bool = False): head_sha=pr_head_sha or "", target_branch=target_branch, upload=dispatch_upload, + bot_comment_id=str(status_comment_id or "") if dispatch_upload else "", requested_backends=kernel_backends.get(kernel_name), # The audit is per-PR, so request it only once (on the first kernel). run_security=run_security and index == 0, diff --git a/.github/scripts/test_dispatch.py b/.github/scripts/test_dispatch.py index 906095ce..135e6473 100644 --- a/.github/scripts/test_dispatch.py +++ b/.github/scripts/test_dispatch.py @@ -184,6 +184,19 @@ def test_requested_backends_thread_through_dispatch_dry_run(): assert any("xpu" in c.split(",") for c in csvs) +def test_bot_comment_id_threads_to_build_inputs_only_when_set(): + without = _build_actions(_plan(backends=["cuda"])) + assert all("bot_comment_id" not in a.body["inputs"] for a in without) + + with_id = _build_actions(_plan(backends=["cuda"], bot_comment_id="12345")) + assert all(a.body["inputs"]["bot_comment_id"] == "12345" for a in with_id) + + no_upload = _build_actions( + _plan(backends=["cuda"], upload=False, bot_comment_id="12345") + ) + assert all("bot_comment_id" not in a.body["inputs"] for a in no_upload) + + # select_workflows: backend-union and Windows-gate cases (single-backend rows omitted). ROUTING_TRUTH_TABLE = [ (["cuda"], False, {"build.yaml"}), diff --git a/.github/workflows/build-mac.yaml b/.github/workflows/build-mac.yaml index 2957d1b4..e6bf5c68 100644 --- a/.github/workflows/build-mac.yaml +++ b/.github/workflows/build-mac.yaml @@ -52,6 +52,11 @@ on: required: false type: string default: "" + bot_comment_id: + description: "Issue comment ID to update with Hub upload links" + required: false + type: string + default: "" permissions: contents: read @@ -206,12 +211,15 @@ jobs: REPO_PREFIX: ${{ inputs.repo_prefix }} TARGET_BRANCH: ${{ inputs.target_branch }} run: | - cd "$KERNEL" BRANCH_FLAG="" if [ -n "$TARGET_BRANCH" ]; then BRANCH_FLAG="--branch $TARGET_BRANCH" fi - nix run -L github:huggingface/kernels#kernel-builder -- upload --repo-type kernel --repo-id "$REPO_PREFIX/$KERNEL" $BRANCH_FLAG + TARGET_FLAG=$(python3 .github/scripts/hub_pr_upload_args.py upload-args "$KERNEL" "$REPO_PREFIX") + cd "$KERNEL" + nix run -L github:huggingface/kernels#kernel-builder -- upload \ + --repo-type kernel $TARGET_FLAG $BRANCH_FLAG \ + --output-json "$RUNNER_TEMP/upload.json" # v1 kernels without an explicit branch override also get uploaded to main. - name: Upload v1 kernels to main @@ -221,21 +229,39 @@ jobs: KERNEL: ${{ steps.validate.outputs.kernel }} REPO_PREFIX: ${{ inputs.repo_prefix }} run: | + TARGET_FLAG=$(python3 .github/scripts/hub_pr_upload_args.py upload-args "$KERNEL" "$REPO_PREFIX") cd "$KERNEL" if [ -f "build.toml" ]; then VERSION=$(grep -E '^\s*version\s*=\s*1\s*$' build.toml || true) BRANCH=$(grep -E '^\s*branch\s*=' build.toml || true) if [ -n "$VERSION" ] && [ -z "$BRANCH" ]; then - nix run -L github:huggingface/kernels#kernel-builder -- upload --repo-type kernel --repo-id "$REPO_PREFIX/$KERNEL" --branch main + nix run -L github:huggingface/kernels#kernel-builder -- upload \ + --repo-type kernel $TARGET_FLAG --branch main \ + --output-json "$RUNNER_TEMP/upload-main.json" fi fi + - name: Upload Hub upload summary + if: steps.validate.outputs.skip == 'false' && inputs.skip_build != true && inputs.mode != 'pr' && inputs.upload != false + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: hub-upload-macos + path: ${{ runner.temp }}/upload*.json + retention-days: 1 + # Report the final build outcome as a commit status on the PR head SHA. report-status: if: always() && inputs.head_sha != '' needs: [build-kernel] runs-on: ubuntu-latest + permissions: + actions: read + contents: read + issues: write + statuses: write + env: + REPORT_HUB_UPLOAD: ${{ needs.build-kernel.result == 'success' && inputs.bot_comment_id != '' && inputs.mode != 'pr' && inputs.upload != false }} steps: - name: Set commit status env: @@ -263,3 +289,28 @@ jobs: -f target_url="$RUN_URL" \ -f description="$DESC" \ -f context="$STATUS_CONTEXT" + + - name: Checkout comment helpers + if: env.REPORT_HUB_UPLOAD == 'true' + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ inputs.pr_number != '' && format('refs/pull/{0}/head', inputs.pr_number) || github.ref }} + + - name: Download Hub upload summaries + if: env.REPORT_HUB_UPLOAD == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: hub-upload-* + path: hub-uploads + + - name: Update bot comment with Hub URLs + if: env.REPORT_HUB_UPLOAD == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BOT_COMMENT_ID: ${{ inputs.bot_comment_id }} + KERNEL: ${{ inputs.kernel_name }} + REPO_PREFIX: ${{ inputs.repo_prefix }} + run: | + BODY=$(gh api "repos/$GITHUB_REPOSITORY/issues/comments/$BOT_COMMENT_ID" --jq .body) + BODY=$(python3 .github/scripts/format_hub_upload_comment.py --base "$BODY" "$KERNEL" "$REPO_PREFIX" hub-uploads/**/*.json) + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$BOT_COMMENT_ID" -f body="$BODY" diff --git a/.github/workflows/build-windows.yaml b/.github/workflows/build-windows.yaml index fe75c55e..87234e9a 100644 --- a/.github/workflows/build-windows.yaml +++ b/.github/workflows/build-windows.yaml @@ -52,6 +52,11 @@ on: required: false type: string default: "" + bot_comment_id: + description: "Issue comment ID to update with Hub upload links" + required: false + type: string + default: "" permissions: contents: read diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index f0b80d4b..ae951168 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -52,6 +52,11 @@ on: required: false type: string default: "" + bot_comment_id: + description: "Issue comment ID to update with Hub upload links" + required: false + type: string + default: "" permissions: contents: read @@ -266,12 +271,15 @@ jobs: REPO_PREFIX: ${{ inputs.repo_prefix }} TARGET_BRANCH: ${{ inputs.target_branch }} run: | - cd "$KERNEL" BRANCH_FLAG="" if [ -n "$TARGET_BRANCH" ]; then BRANCH_FLAG="--branch $TARGET_BRANCH" fi - nix run -L github:huggingface/kernels#kernel-builder -- upload --repo-type kernel --repo-id "$REPO_PREFIX/$KERNEL" $BRANCH_FLAG + TARGET_FLAG=$(python3 .github/scripts/hub_pr_upload_args.py upload-args "$KERNEL" "$REPO_PREFIX") + cd "$KERNEL" + nix run -L github:huggingface/kernels#kernel-builder -- upload \ + --repo-type kernel $TARGET_FLAG $BRANCH_FLAG \ + --output-json "$RUNNER_TEMP/upload.json" # v1 kernels without an explicit branch override also get uploaded to main. - name: Upload v1 kernels to main @@ -281,6 +289,7 @@ jobs: KERNEL: ${{ needs.setup.outputs.kernel }} REPO_PREFIX: ${{ inputs.repo_prefix }} run: | + TARGET_FLAG=$(python3 .github/scripts/hub_pr_upload_args.py upload-args "$KERNEL" "$REPO_PREFIX") cd "$KERNEL" # Check if build.toml exists, has version = 1, and does not specify a branch. @@ -288,10 +297,20 @@ jobs: VERSION=$(grep -E '^\s*version\s*=\s*1\s*$' build.toml || true) BRANCH=$(grep -E '^\s*branch\s*=' build.toml || true) if [ -n "$VERSION" ] && [ -z "$BRANCH" ]; then - nix run -L github:huggingface/kernels#kernel-builder -- upload --repo-type kernel --repo-id "$REPO_PREFIX/$KERNEL" --branch main + nix run -L github:huggingface/kernels#kernel-builder -- upload \ + --repo-type kernel $TARGET_FLAG --branch main \ + --output-json "$RUNNER_TEMP/upload-main.json" fi fi + - name: Upload Hub upload summary + if: inputs.mode != 'pr' && inputs.upload != false + uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1 + with: + name: hub-upload-${{ matrix.backend }}-${{ matrix.arch }} + path: ${{ runner.temp }}/upload*.json + retention-days: 1 + # Build the ci-test derivation and export its Nix closure as an artifact # so the GPU test job can import it on a GPU-enabled runner. build-ci-test: @@ -406,6 +425,13 @@ jobs: if: always() && inputs.head_sha != '' needs: [build-kernel, test-kernel-gpu] runs-on: ubuntu-latest + permissions: + actions: read + contents: read + issues: write + statuses: write + env: + REPORT_HUB_UPLOAD: ${{ needs.build-kernel.result == 'success' && inputs.bot_comment_id != '' && inputs.mode != 'pr' && inputs.upload != false }} steps: - name: Set commit status env: @@ -434,3 +460,28 @@ jobs: -f target_url="$RUN_URL" \ -f description="$DESC" \ -f context="$STATUS_CONTEXT" + + - name: Checkout comment helpers + if: env.REPORT_HUB_UPLOAD == 'true' + uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 + with: + ref: ${{ inputs.pr_number != '' && format('refs/pull/{0}/head', inputs.pr_number) || github.ref }} + + - name: Download Hub upload summaries + if: env.REPORT_HUB_UPLOAD == 'true' + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + pattern: hub-upload-* + path: hub-uploads + + - name: Update bot comment with Hub URLs + if: env.REPORT_HUB_UPLOAD == 'true' + env: + GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} + BOT_COMMENT_ID: ${{ inputs.bot_comment_id }} + KERNEL: ${{ inputs.kernel_name }} + REPO_PREFIX: ${{ inputs.repo_prefix }} + run: | + BODY=$(gh api "repos/$GITHUB_REPOSITORY/issues/comments/$BOT_COMMENT_ID" --jq .body) + BODY=$(python3 .github/scripts/format_hub_upload_comment.py --base "$BODY" "$KERNEL" "$REPO_PREFIX" hub-uploads/**/*.json) + gh api -X PATCH "repos/$GITHUB_REPOSITORY/issues/comments/$BOT_COMMENT_ID" -f body="$BODY" diff --git a/.github/workflows/sign-old-builds.yaml b/.github/workflows/sign-old-builds.yaml index 6ecd8a47..d02e027c 100644 --- a/.github/workflows/sign-old-builds.yaml +++ b/.github/workflows/sign-old-builds.yaml @@ -74,8 +74,11 @@ jobs: VERSION=$(grep -E '^\s*version\s*=' "$kernel/build.toml" | grep -oE '[0-9]+') VERSION_BRANCH="v${VERSION}" + REPO_ID=$(python3 .github/scripts/hub_pr_upload_args.py repo-id "$kernel" "$REPO_PREFIX") + TARGET_FLAG=$(python3 .github/scripts/hub_pr_upload_args.py upload-args "$kernel" "$REPO_PREFIX") + # Download existing kernel artifacts from the version branch. - ( cd "$kernel" && uvx --with huggingface_hub python3 -c "from huggingface_hub import snapshot_download; snapshot_download('$REPO_PREFIX/$kernel', repo_type='kernel', local_dir='.', revision='$VERSION_BRANCH')" ) + ( cd "$kernel" && uvx --with huggingface_hub python3 -c "from huggingface_hub import snapshot_download; snapshot_download('$REPO_ID', repo_type='kernel', local_dir='.', revision='$VERSION_BRANCH')" ) # Remove build variants that are not suitable for hashing: # - no __init__.py (not a Python kernel build) @@ -108,13 +111,13 @@ jobs: # Upload signed artifacts back to the Hub (kernel repo type). ( cd "$kernel" && nix run -L github:huggingface/kernels#kernel-builder -- \ - upload --repo-type kernel --repo-id "$REPO_PREFIX/$kernel" --branch "$VERSION_BRANCH" ) + upload --repo-type kernel $TARGET_FLAG --branch "$VERSION_BRANCH" ) # v1 kernels without an explicit branch in build.toml also get uploaded to main. BRANCH=$(grep -E '^\s*branch\s*=' "$kernel/build.toml" || true) if [ "$VERSION" = "1" ] && [ -z "$BRANCH" ]; then ( cd "$kernel" && nix run -L github:huggingface/kernels#kernel-builder -- \ - upload --repo-type kernel --repo-id "$REPO_PREFIX/$kernel" --branch main ) + upload --repo-type kernel $TARGET_FLAG --branch main ) fi # Clean up to reclaim disk space before the next kernel. diff --git a/msa/build.toml b/msa/build.toml index 23438a74..8d74f25f 100644 --- a/msa/build.toml +++ b/msa/build.toml @@ -10,7 +10,7 @@ upstream = "https://github.com/MiniMax-AI/MSA" version = 0 [general.hub] -repo-id = "kernels-community/msa" +repo-id = "MiniMaxAI/msa" [general.cuda] minver = "12.8" diff --git a/sgl-flash-attn3/build.toml b/sgl-flash-attn3/build.toml index 353c699f..624adf64 100644 --- a/sgl-flash-attn3/build.toml +++ b/sgl-flash-attn3/build.toml @@ -8,7 +8,7 @@ backends = ["cuda"] minver = "12.8" [general.hub] -repo-id = "kernels-community/sgl-flash-attn3" +repo-id = "sgl-project/sgl-flash-attn3" [torch] src = [