From d0fb2716b61b54ac490ead7543f172cac9ed762b Mon Sep 17 00:00:00 2001 From: "liang.he" Date: Sun, 26 Mar 2023 12:19:45 +0800 Subject: [PATCH] Fix several issues in CI binary releasing (#2064) --- .github/workflows/build_iwasm_release.yml | 20 ++++++++ .github/workflows/build_wamr_vscode_ext.yml | 11 +++-- .github/workflows/build_wamrc.yml | 9 ++-- .github/workflows/release_process.yml | 47 ++++++++++++++----- product-mini/platforms/android/build_llvm.sh | 1 + product-mini/platforms/darwin/build_llvm.sh | 1 + product-mini/platforms/freebsd/build_llvm.sh | 1 + product-mini/platforms/linux/build_llvm.sh | 1 + .../wamr-ide/WASM-Toolchain/Docker/Dockerfile | 8 +++- wamr-compiler/build_llvm.sh | 1 + wamr-compiler/build_llvm_arc.sh | 1 + wamr-compiler/build_llvm_xtensa.sh | 1 + 12 files changed, 79 insertions(+), 23 deletions(-) diff --git a/.github/workflows/build_iwasm_release.yml b/.github/workflows/build_iwasm_release.yml index 557a81fc4b..ae1f6140b9 100644 --- a/.github/workflows/build_iwasm_release.yml +++ b/.github/workflows/build_iwasm_release.yml @@ -14,6 +14,10 @@ on: description: workfing directory type: string required: true + llvm_cache_key: + description: the cache key of llvm libraries + type: string + required: true runner: description: OS of compilation type: string @@ -33,6 +37,22 @@ jobs: steps: - uses: actions/checkout@v3 + - name: get cached LLVM libraries + id: retrieve_llvm_libs + uses: actions/cache@v3 + with: + path: | + ./core/deps/llvm/build/bin + ./core/deps/llvm/build/include + ./core/deps/llvm/build/lib + ./core/deps/llvm/build/libexec + ./core/deps/llvm/build/share + key: ${{ inputs.llvm_cache_key }} + + - name: Quit if cache miss + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' + run: echo "::error::can not get prebuilt llvm libraries" && exit 1 + - name: generate iwasm binary release run: | cmake -S . -B build \ diff --git a/.github/workflows/build_wamr_vscode_ext.yml b/.github/workflows/build_wamr_vscode_ext.yml index f62a4bc988..5c1853e24f 100644 --- a/.github/workflows/build_wamr_vscode_ext.yml +++ b/.github/workflows/build_wamr_vscode_ext.yml @@ -24,22 +24,25 @@ jobs: uses: actions/setup-node@v3 with: node-version: 14.x - - - name: set vscode extension to correct version + + - name: set vscode extension to correct version run: | npm install -g json json -I -f package.json -e "this.version=\"${{ inputs.ver_num }}\"" working-directory: test-tools/wamr-ide/VSCode-Extension + # [!workflow] + # bypass the step of publishing the extension to the Market. + # recover it after creating the secret in the Environment - name: generate wamr ide vscode extension - env: + env: credentials: ${{ secrets.TOKEN }} run: | npm install -g vsce rm -rf node_modules npm install vsce package - vsce publish -p ${{ secrets.TOKEN }} + # vsce publish -p ${{ secrets.TOKEN }} working-directory: test-tools/wamr-ide/VSCode-Extension - name: compress the vscode extension diff --git a/.github/workflows/build_wamrc.yml b/.github/workflows/build_wamrc.yml index 829b036ea7..a88a1f715f 100644 --- a/.github/workflows/build_wamrc.yml +++ b/.github/workflows/build_wamrc.yml @@ -38,7 +38,7 @@ jobs: - uses: actions/checkout@v3 - name: get cached LLVM libraries - id: cache_llvm + id: retrieve_llvm_libs uses: actions/cache@v3 with: path: | @@ -49,10 +49,9 @@ jobs: ./core/deps/llvm/build/share key: ${{ inputs.llvm_cache_key }} - - name: Build llvm and clang from source - if: steps.cache_llvm.outputs.cache-hit != 'true' - run: /usr/bin/env python3 ./build_llvm.py --arch X86 - working-directory: build-scripts + - name: Quit if cache miss + if: steps.retrieve_llvm_libs.outputs.cache-hit != 'true' + run: echo "::error::can not get prebuilt llvm libraries" && exit 1 - name: generate wamrc binary release run: | diff --git a/.github/workflows/release_process.yml b/.github/workflows/release_process.yml index 88f595dd11..4188b4d40a 100644 --- a/.github/workflows/release_process.yml +++ b/.github/workflows/release_process.yml @@ -49,36 +49,56 @@ jobs: draft: false body: ${{ env.RELEASE_NOTE }} + # + # LLVM_LIBRARIES + build_llvm_libraries_on_ubuntu_2004: + needs: [create_tag, create_release] + uses: ./.github/workflows/build_llvm_libraries.yml + with: + os: "ubuntu-20.04" + arch: "X86" + + build_llvm_libraries_on_ubuntu_2204: + needs: [create_tag, create_release] + uses: ./.github/workflows/build_llvm_libraries.yml + with: + os: "ubuntu-22.04" + arch: "X86" + + build_llvm_libraries_on_macos: + needs: [create_tag, create_release] + uses: ./.github/workflows/build_llvm_libraries.yml + with: + os: "macos-latest" + arch: "X86" + # # WAMRC release_wamrc_on_ubuntu_2004: - needs: [create_tag, create_release] + needs: [create_tag, create_release, build_llvm_libraries_on_ubuntu_2004] uses: ./.github/workflows/build_wamrc.yml with: - # can't take an env variable here - llvm_cache_key: ubuntu-20.04-build-llvm_libraries_ex + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }} release: true runner: ubuntu-20.04 upload_url: ${{ needs.create_release.outputs.upload_url }} ver_num: ${{ needs.create_tag.outputs.new_ver}} release_wamrc_on_ubuntu_2204: - needs: [create_tag, create_release] + needs: [create_tag, create_release, build_llvm_libraries_on_ubuntu_2204 ] uses: ./.github/workflows/build_wamrc.yml with: - # can't take an env variable here - llvm_cache_key: ubuntu-22.04-build-llvm_libraries_ex + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }} release: true runner: ubuntu-22.04 upload_url: ${{ needs.create_release.outputs.upload_url }} ver_num: ${{ needs.create_tag.outputs.new_ver }} release_wamrc_on_ubuntu_macos: - needs: [create_tag, create_release] + needs: [create_tag, create_release, build_llvm_libraries_on_macos] uses: ./.github/workflows/build_wamrc.yml with: - # can't take an env variable here - llvm_cache_key: macos-latest-build-llvm_libraries_ex + llvm_cache_key: ${{ needs.build_llvm_libraries_on_macos.outputs.cache_key }} release: true runner: macos-latest upload_url: ${{ needs.create_release.outputs.upload_url }} @@ -87,28 +107,31 @@ jobs: # # IWASM release_iwasm_on_ubuntu_2004: - needs: [create_tag, create_release] + needs: [create_tag, create_release, build_llvm_libraries_on_ubuntu_2004] uses: ./.github/workflows/build_iwasm_release.yml with: cwd: product-mini/platforms/linux + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2004.outputs.cache_key }} runner: ubuntu-20.04 upload_url: ${{ needs.create_release.outputs.upload_url }} ver_num: ${{ needs.create_tag.outputs.new_ver}} release_iwasm_on_ubuntu_2204: - needs: [create_tag, create_release] + needs: [create_tag, create_release, build_llvm_libraries_on_ubuntu_2204] uses: ./.github/workflows/build_iwasm_release.yml with: cwd: product-mini/platforms/linux + llvm_cache_key: ${{ needs.build_llvm_libraries_on_ubuntu_2204.outputs.cache_key }} runner: ubuntu-22.04 upload_url: ${{ needs.create_release.outputs.upload_url }} ver_num: ${{ needs.create_tag.outputs.new_ver}} release_iwasm_on_macos: - needs: [create_tag, create_release] + needs: [create_tag, create_release, build_llvm_libraries_on_macos] uses: ./.github/workflows/build_iwasm_release.yml with: cwd: product-mini/platforms/darwin + llvm_cache_key: ${{ needs.build_llvm_libraries_on_macos.outputs.cache_key }} runner: macos-latest upload_url: ${{ needs.create_release.outputs.upload_url }} ver_num: ${{ needs.create_tag.outputs.new_ver}} diff --git a/product-mini/platforms/android/build_llvm.sh b/product-mini/platforms/android/build_llvm.sh index 3ef343f982..145e2dbaa0 100755 --- a/product-mini/platforms/android/build_llvm.sh +++ b/product-mini/platforms/android/build_llvm.sh @@ -3,4 +3,5 @@ # Copyright (C) 2020 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +/usr/bin/env python3 -m pip install --user -r ../../../build-scripts/requirements.txt /usr/bin/env python3 ../../../build-scripts/build_llvm.py --platform android "$@" diff --git a/product-mini/platforms/darwin/build_llvm.sh b/product-mini/platforms/darwin/build_llvm.sh index 1cb00425a1..b8a9761f85 100755 --- a/product-mini/platforms/darwin/build_llvm.sh +++ b/product-mini/platforms/darwin/build_llvm.sh @@ -3,4 +3,5 @@ # Copyright (C) 2020 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +/usr/bin/env python3 -m pip install --user -r ../../../build-scripts/requirements.txt /usr/bin/env python3 ../../../build-scripts/build_llvm.py --platform darwin "$@" diff --git a/product-mini/platforms/freebsd/build_llvm.sh b/product-mini/platforms/freebsd/build_llvm.sh index 47387a3c32..c5666b7f5d 100755 --- a/product-mini/platforms/freebsd/build_llvm.sh +++ b/product-mini/platforms/freebsd/build_llvm.sh @@ -3,4 +3,5 @@ # Copyright (C) 2020 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +/usr/bin/env python3 -m pip install --user -r ../../../build-scripts/requirements.txt /usr/bin/env python3 ../../../build-scripts/build_llvm.py "$@" diff --git a/product-mini/platforms/linux/build_llvm.sh b/product-mini/platforms/linux/build_llvm.sh index 47387a3c32..c5666b7f5d 100755 --- a/product-mini/platforms/linux/build_llvm.sh +++ b/product-mini/platforms/linux/build_llvm.sh @@ -3,4 +3,5 @@ # Copyright (C) 2020 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +/usr/bin/env python3 -m pip install --user -r ../../../build-scripts/requirements.txt /usr/bin/env python3 ../../../build-scripts/build_llvm.py "$@" diff --git a/test-tools/wamr-ide/WASM-Toolchain/Docker/Dockerfile b/test-tools/wamr-ide/WASM-Toolchain/Docker/Dockerfile index 6ca2109471..b2e24e93d6 100644 --- a/test-tools/wamr-ide/WASM-Toolchain/Docker/Dockerfile +++ b/test-tools/wamr-ide/WASM-Toolchain/Docker/Dockerfile @@ -12,7 +12,7 @@ COPY resource /root/ ## - download cmake with wget and set up # hadolint ignore=DL3008 RUN wget --progress=dot:giga https://github.com/Kitware/CMake/releases/download/v3.21.1/cmake-3.21.1-linux-x86_64.tar.gz \ - && tar -zxvf cmake-3.21.1-linux-x86_64.tar.gz \ + && tar -zxf cmake-3.21.1-linux-x86_64.tar.gz \ && rm -f cmake-3.21.1-linux-x86_64.tar.gz \ && mv cmake-3.21.1-linux-x86_64 /opt/cmake \ && ln -s /opt/cmake/bin/cmake /bin/cmake \ @@ -26,13 +26,17 @@ RUN wget --progress=dot:giga https://github.com/Kitware/CMake/releases/download/ ## - download wasi-sdk with wget and set up to /opt/wasi-sdk RUN wget --progress=dot:giga https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-19/wasi-sdk-19.0-linux.tar.gz \ - && tar -zxvf wasi-sdk-*-linux.tar.gz \ + && tar -zxf wasi-sdk-*-linux.tar.gz \ && mv wasi-sdk-19.0 /opt/wasi-sdk/ \ && rm -f wasi-sdk-*-linux.tar.gz ## - clone wamr repo RUN git clone -b main --depth=1 https://github.com/bytecodealliance/wasm-micro-runtime.git +WORKDIR /root/wasm-micro-runtime/build-scripts +RUN apt-get update && apt-get install --no-install-recommends -y ccache ninja-build python3-pip +RUN pip3 install --user -r requirements.txt + WORKDIR /root/wasm-micro-runtime/wamr-compiler RUN ./build_llvm.sh \ && mkdir build diff --git a/wamr-compiler/build_llvm.sh b/wamr-compiler/build_llvm.sh index 35dc35edd9..c3ec54b61b 100755 --- a/wamr-compiler/build_llvm.sh +++ b/wamr-compiler/build_llvm.sh @@ -3,4 +3,5 @@ # Copyright (C) 2020 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +/usr/bin/env python3 -m pip install --user -r ../build-scripts/requirements.txt /usr/bin/env python3 ../build-scripts/build_llvm.py "$@" diff --git a/wamr-compiler/build_llvm_arc.sh b/wamr-compiler/build_llvm_arc.sh index f8f40edd9a..d148e11ecc 100755 --- a/wamr-compiler/build_llvm_arc.sh +++ b/wamr-compiler/build_llvm_arc.sh @@ -3,4 +3,5 @@ # Copyright (C) 2020 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +/usr/bin/env python3 -m pip install --user -r ../build-scripts/requirements.txt /usr/bin/env python3 ../build-scripts/build_llvm.py --platform arc "$@" diff --git a/wamr-compiler/build_llvm_xtensa.sh b/wamr-compiler/build_llvm_xtensa.sh index d75b62fe9f..183ea379fd 100755 --- a/wamr-compiler/build_llvm_xtensa.sh +++ b/wamr-compiler/build_llvm_xtensa.sh @@ -3,4 +3,5 @@ # Copyright (C) 2020 Intel Corporation. All rights reserved. # SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception +/usr/bin/env python3 -m pip install --user -r ../build-scripts/requirements.txt /usr/bin/env python3 ../build-scripts/build_llvm.py --platform xtensa "$@"