chore: modify rust tool chain version #507
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Coverage Report on Coveralls | |
| on: | |
| release: | |
| types: [created] | |
| pull_request: | |
| types: [opened, synchronize, reopened] | |
| paths: | |
| - "core/**" | |
| - "!core/src/ten_manager/designer_frontend/**" | |
| - "tests/**" | |
| - "build/**" | |
| - ".github/workflows/coverage.yml" | |
| push: | |
| branches: ["main"] | |
| paths: | |
| - "core/**" | |
| - "!core/src/ten_manager/designer_frontend/**" | |
| - "tests/**" | |
| - "build/**" | |
| - ".github/workflows/coverage.yml" | |
| permissions: | |
| contents: write | |
| concurrency: | |
| group: coverage-${{ github.head_ref }} | |
| cancel-in-progress: true | |
| jobs: | |
| call-check-pr-status: | |
| uses: ./.github/workflows/_check_pr_status.yml | |
| build: | |
| needs: call-check-pr-status | |
| if: ${{ needs.call-check-pr-status.outputs.should_continue == 'true' }} | |
| runs-on: ubuntu-22.04 | |
| # Remove container to get more disk space (runner has ~70GB free vs ~12GB with container) | |
| # container: | |
| # image: ghcr.io/ten-framework/ten_building_ubuntu2204 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Free up disk space on runner | |
| run: | | |
| echo "=== Initial disk space ===" | |
| df -h | |
| # Remove unnecessary pre-installed software on GitHub runner | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /opt/hostedtoolcache/CodeQL | |
| sudo docker image prune --all --force | |
| echo "=== Disk space after cleanup ===" | |
| df -h | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends \ | |
| build-essential \ | |
| wget \ | |
| curl \ | |
| git \ | |
| libssl-dev \ | |
| pkg-config \ | |
| cmake \ | |
| tree \ | |
| zip \ | |
| unzip \ | |
| jq \ | |
| python3 \ | |
| python3-dev \ | |
| python3-pip \ | |
| python3-venv \ | |
| ca-certificates \ | |
| software-properties-common \ | |
| gnupg \ | |
| lsb-release \ | |
| libunwind-dev \ | |
| libasound2 \ | |
| libavformat-dev \ | |
| libavfilter-dev \ | |
| libx264-dev \ | |
| nasm \ | |
| yasm | |
| # Clean up apt cache | |
| sudo apt-get clean | |
| sudo rm -rf /var/lib/apt/lists/* | |
| echo "=== Disk space after system dependencies ===" | |
| df -h | |
| - name: Debug context | |
| run: | | |
| echo "event_name=${{ github.event_name }}" | |
| echo "event_action=${{ github.event.action }}" | |
| echo "ref=${{ github.ref }}" | |
| echo "ref_name=${{ github.ref_name }}" | |
| echo "ref_type=${{ github.ref_type }}" | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: false | |
| - name: Trust working directory | |
| run: git config --global --add safe.directory "${GITHUB_WORKSPACE}" | |
| - name: Initialize and update submodules except portal/ | |
| run: | | |
| # Retrieve all submodule paths, excluding `portal/`. | |
| submodules=$(git config --file .gitmodules --get-regexp path | awk '$2 != "portal" { print $2 }') | |
| git submodule init | |
| for submodule in $submodules; do | |
| echo "Initializing submodule: $submodule" | |
| git submodule update --init --recursive --depth 1 "$submodule" | |
| done | |
| echo "=== Disk space after submodule initialization ===" | |
| df -h | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Install Python dependencies | |
| run: | | |
| pip3 install --upgrade setuptools pip | |
| pip3 install requests python-dotenv jinja2 | |
| pip3 install debugpy pytest pytest-cov pytest-mock cython pylint pylint-exit black | |
| echo "=== Disk space after Python dependencies ===" | |
| df -h | |
| - name: Update version | |
| run: | | |
| python3 tools/version/update_version_in_ten_framework.py | |
| python3 tools/version/check_version_in_ten_framework.py | |
| - name: Install LLVM 21 tools | |
| run: | | |
| # Add LLVM official repository for LLVM 21 | |
| wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc | |
| sudo add-apt-repository -y "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-21 main" | |
| sudo apt-get update | |
| # Purge preinstalled clang versions BEFORE installing clang-21 | |
| # | |
| # Root cause: | |
| # - GitHub Actions ubuntu-22.04 runner preinstalls Clang 13.0.1, 14.0.0, and 15.0.7 | |
| # - Ref: https://github.com/actions/runner-images/blob/main/images/ubuntu/Ubuntu2204-Readme.md | |
| # | |
| # - During the build stage, we primarily compile C code (.c files) from core/src/ten_runtime/*, | |
| # which does NOT trigger the `-stdlib=libc++` flag. Therefore, clang-21 only uses its own | |
| # headers without referencing system libc++ headers, avoiding version conflicts. | |
| # | |
| # - During the test stage, we compile C++ applications (.cc files) that include | |
| # C++ standard library headers (like <string>, <iostream>). When clang-21 uses | |
| # `-stdlib=libc++` (set in core/ten_gn/.gnfiles/build/platform/linux/BUILD.gn:68), it: | |
| # 1. Uses LLVM 21's C++ stdlib headers: /usr/lib/llvm-21/include/c++/v1/* | |
| # 2. BUT also picks up LLVM 13/14/15's compiler intrinsic headers from preinstalled versions | |
| # | |
| # This mixing causes compilation errors. Even with PATH prioritization and symlinks, | |
| # the compiler's default header search paths still include older clang directories. | |
| sudo apt-get purge -y clang-13 clang-14 clang-15 \ | |
| llvm-13-dev llvm-13-runtime llvm-13 \ | |
| llvm-14-dev llvm-14-runtime llvm-14 \ | |
| llvm-15-dev llvm-15-runtime llvm-15 || true | |
| sudo apt-get autopurge -y || true | |
| # Now install LLVM 21 with all required packages | |
| sudo apt-get install -y \ | |
| llvm-21 \ | |
| clang-21 \ | |
| libc++-21-dev \ | |
| libc++1-21 \ | |
| libc++abi-21-dev \ | |
| libc++abi1-21 | |
| # Clean up apt caches to save space | |
| sudo apt-get clean | |
| sudo rm -rf /var/lib/apt/lists/* | |
| # CRITICAL: GN build system hardcodes "clang"/"clang++" (without version suffix) | |
| # We must override these to point to clang-21 to ensure profraw format version 10 | |
| sudo ln -sf /usr/bin/clang-21 /usr/local/bin/clang | |
| sudo ln -sf /usr/bin/clang++-21 /usr/local/bin/clang++ | |
| sudo ln -sf /usr/bin/llvm-ar-21 /usr/local/bin/llvm-ar | |
| sudo ln -sf /usr/bin/llvm-nm-21 /usr/local/bin/llvm-nm | |
| sudo ln -sf /usr/bin/llvm-ranlib-21 /usr/local/bin/llvm-ranlib | |
| # Create symlinks for coverage tools | |
| sudo ln -sf /usr/bin/llvm-cov-21 /usr/local/bin/llvm-cov | |
| sudo ln -sf /usr/bin/llvm-profdata-21 /usr/local/bin/llvm-profdata | |
| sudo ln -sf /usr/bin/llvm-objdump-21 /usr/local/bin/llvm-objdump | |
| # Verify installation - should all show LLVM 21 | |
| echo "=== Verifying LLVM 21 tools are correctly linked ===" | |
| which clang | |
| clang --version | head -3 | |
| which clang++ | |
| clang++ --version | head -3 | |
| llvm-cov --version | head -3 | |
| llvm-profdata merge --version 2>&1 | head -3 || echo "llvm-profdata installed" | |
| echo "=== Disk space after LLVM installation ===" | |
| df -h | |
| - name: Build (clang + debug + x64) with coverage instrumentation enabled | |
| env: | |
| CARGO_INCREMENTAL: "0" | |
| CARGO_NET_GIT_FETCH_WITH_CLI: "true" | |
| CARGO_BUILD_JOBS: "2" | |
| MAKEFLAGS: "-j2" | |
| run: | | |
| # Ensure /usr/local/bin is at the front of PATH so our clang-21 symlinks are used | |
| export PATH=/usr/local/bin:$(pwd)/core/ten_gn:/usr/local/go/bin:$HOME/go/bin:$HOME/.cargo/bin:$PATH | |
| echo "PATH: $PATH" | |
| # Verify we're using the correct clang version | |
| echo "=== Compiler verification before build ===" | |
| which clang | |
| clang --version | head -3 | |
| which clang++ | |
| clang++ --version | head -3 | |
| # Install Go 1.24.3 | |
| go env -w GOFLAGS="-buildvcs=false" | |
| go install golang.org/dl/go1.24.3@latest | |
| export PATH=$PATH:$(go env GOPATH)/bin | |
| go1.24.3 download | |
| go1.24.3 version | |
| echo "=== Disk space after Go installation ===" | |
| df -h | |
| # Install Rust stable because we need to disable asan to avoid coverage instrumentation conflicts. | |
| # Therefore, rust nightly is not needed. | |
| rustup default stable | |
| # print rustc and llvm-cov's llvm version, make sure they are compatible | |
| rustc -vV | grep LLVM | |
| llvm-cov --version | head -3 | |
| echo "llvm-profdata path: $(which llvm-profdata)" | |
| llvm-profdata merge --version 2>&1 | head -3 || echo "(llvm-profdata --version not supported)" | |
| echo "=== Disk space before build ===" | |
| df -h | |
| # enable_coverage=true: generate instrumented test executables and dynamic libraries | |
| # which will be used to generate coverage report | |
| # enable_sanitizer=false: disable AddressSanitizer because it conflicts with coverage | |
| # instrumentation | |
| EXTRA_ARGS=" \ | |
| is_clang=true \ | |
| enable_sanitizer=false \ | |
| enable_coverage=true \ | |
| log_level=1 \ | |
| enable_serialized_actions=true \ | |
| ten_rust_enable_gen_cargo_config=false \ | |
| ten_enable_cargo_clean=true \ | |
| ten_enable_go_lint=false \ | |
| ten_enable_rust_incremental_build=false \ | |
| ten_manager_enable_frontend=false \ | |
| ten_enable_ffmpeg_extensions=true" | |
| echo $EXTRA_ARGS | |
| tgn gen linux x64 debug -- $EXTRA_ARGS | |
| echo "=== Disk space after tgn gen ===" | |
| df -h . | |
| # Monitor and clean during build | |
| # Start a background process to periodically clean cargo cache | |
| ( | |
| while true; do | |
| sleep 300 # Every 5 minutes | |
| # Clean cargo build cache for dependencies we've already compiled | |
| find $HOME/.cargo/registry -type f -name "*.crate" -mmin +10 -delete 2>/dev/null || true | |
| find core/src/ten_rust/target -type f -name "*.rlib" -mmin +10 2>/dev/null | head -100 | xargs -r rm -f 2>/dev/null || true | |
| find core/src/ten_manager/target -type f -name "*.rlib" -mmin +10 2>/dev/null | head -100 | xargs -r rm -f 2>/dev/null || true | |
| done | |
| ) & | |
| CLEANUP_PID=$! | |
| tgn build linux x64 debug | |
| # Stop the cleanup background process | |
| kill $CLEANUP_PID 2>/dev/null || true | |
| echo "=== Disk space after build ===" | |
| df -h | |
| # Clean up intermediate build files to save space | |
| echo "=== Cleaning up intermediate build files ===" | |
| rm -rf out/linux/x64/obj | |
| echo "=== Disk space after cleanup ===" | |
| df -h | |
| tree -I 'gen|obj' out | |
| # Package CORE_BINARIES_AND_LIBS for coverage analysis | |
| - name: Clean up unnecessary files before packaging | |
| run: | | |
| echo "=== Disk space before cleanup ===" | |
| df -h | |
| # Remove gen directory except libten_rust.a (which is needed for coverage) | |
| find out/linux/x64/gen -type f ! -name "libten_rust.a" -delete | |
| find out/linux/x64/gen -type d -empty -delete | |
| # Remove Rust build artifacts | |
| rm -rf core/src/ten_manager/target | |
| rm -rf core/src/ten_rust/target | |
| # Clean cargo cache | |
| rm -rf $HOME/.cargo/registry | |
| rm -rf $HOME/.cargo/git | |
| echo "=== Disk space after cleanup ===" | |
| df -h | |
| - name: Package binaries and libraries for coverage analysis | |
| run: | | |
| mkdir -p binaries_and_libs | |
| # Define files to copy: "source_path:destination_name" (destination defaults to basename if not specified) | |
| declare -A FILES_TO_COPY=( | |
| ["out/linux/x64/tests/standalone/ten_utils_unit_test"]="" | |
| ["out/linux/x64/tests/standalone/ten_rust/unit_test"]="ten_rust_unit_test" | |
| ["out/linux/x64/tests/standalone/ten_rust/integration_test"]="ten_rust_integration_test" | |
| ["out/linux/x64/tests/standalone/ten_manager/unit_test"]="ten_manager_unit_test" | |
| ["out/linux/x64/tests/standalone/ten_manager/integration_test"]="ten_manager_integration_test" | |
| ["out/linux/x64/ten_manager/bin/tman"]="" | |
| ["out/linux/x64/tests/standalone/ten_runtime_smoke_test"]="" | |
| ["out/linux/x64/tests/standalone/ten_runtime_unit_test"]="" | |
| ["out/linux/x64/libten_runtime.so"]="" | |
| ["out/linux/x64/libten_runtime_go.so"]="" | |
| ["out/linux/x64/libten_runtime_python.so"]="" | |
| ["out/linux/x64/libten_runtime_nodejs.node"]="" | |
| ["out/linux/x64/libten_utils.so"]="" | |
| ["out/linux/x64/gen/core/src/ten_rust/libten_rust.a"]="" | |
| ) | |
| # Copy files with optional renaming | |
| for src in "${!FILES_TO_COPY[@]}"; do | |
| dst="${FILES_TO_COPY[$src]}" | |
| [ -z "$dst" ] && dst="$(basename "$src")" | |
| if [ -e "$src" ]; then | |
| cp "$src" "binaries_and_libs/$dst" | |
| [ "$dst" != "$(basename "$src")" ] && echo "✓ $src -> $dst" || echo "✓ $dst" | |
| else | |
| echo "✗ Missing: $src" | |
| fi | |
| done | |
| echo -e "\nPackaged files:" | |
| ls -lh binaries_and_libs/ | |
| tar -czf coverage-binaries-and-libs.tar.gz binaries_and_libs | |
| - name: Upload binaries and libraries for coverage analysis | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-binaries-and-libs | |
| path: coverage-binaries-and-libs.tar.gz | |
| # Package the tests artifacts into a tar file while preserving file | |
| # permissions. | |
| - name: Package tests relevant artifacts preserving permissions | |
| run: | | |
| files="" | |
| for item in tests ten_manager tgn_args.txt; do | |
| if [ -e "out/linux/x64/$item" ]; then | |
| files="$files out/linux/x64/$item" | |
| fi | |
| done | |
| if [ -n "$files" ]; then | |
| tar -czvf tests-artifacts.tar.gz $files | |
| fi | |
| - name: Upload tests relevant artifacts | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: tests-artifacts-for-coverage | |
| path: tests-artifacts.tar.gz | |
| if-no-files-found: ignore | |
| test-standalone: | |
| needs: build | |
| runs-on: ubuntu-22.04 | |
| # Remove container to get more disk space | |
| # With debug mode enabled, we need more disk space for coverage instrumentation | |
| # container: | |
| # image: ghcr.io/ten-framework/ten_building_ubuntu2204 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Free up disk space on runner | |
| run: | | |
| echo "=== Initial disk space ===" | |
| df -h | |
| # Remove unnecessary pre-installed software on GitHub runner | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /opt/hostedtoolcache/CodeQL | |
| sudo docker image prune --all --force | |
| echo "=== Disk space after cleanup ===" | |
| df -h | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends \ | |
| build-essential \ | |
| wget \ | |
| curl \ | |
| git \ | |
| libssl-dev \ | |
| pkg-config \ | |
| cmake \ | |
| autoconf \ | |
| libtool \ | |
| tree \ | |
| zip \ | |
| unzip \ | |
| jq \ | |
| python3 \ | |
| python3-dev \ | |
| python3-pip \ | |
| python3-venv \ | |
| ca-certificates \ | |
| software-properties-common \ | |
| gnupg \ | |
| lsb-release \ | |
| libunwind-dev \ | |
| libasound2 \ | |
| libavformat-dev \ | |
| libavfilter-dev \ | |
| libx264-dev \ | |
| nasm \ | |
| yasm \ | |
| uuid-dev \ | |
| libexpat1-dev \ | |
| libcurl4-openssl-dev \ | |
| zlib1g-dev \ | |
| libncurses5-dev \ | |
| libffi-dev \ | |
| libreadline-dev \ | |
| libmsgpack-dev \ | |
| libcrypto++-dev | |
| # Clean up apt cache | |
| sudo apt-get clean | |
| sudo rm -rf /var/lib/apt/lists/* | |
| echo "=== Disk space after system dependencies ===" | |
| df -h | |
| - uses: actions/checkout@v4 | |
| - name: Install LLVM 21 tools | |
| run: | | |
| # Add LLVM official repository for LLVM 21 | |
| wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc | |
| sudo add-apt-repository -y "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-21 main" | |
| sudo apt-get update | |
| # Purge preinstalled clang-13/14/15 BEFORE installing clang-21 (see build job for details) | |
| sudo apt-get purge -y clang-13 clang-14 clang-15 \ | |
| llvm-13-dev llvm-13-runtime llvm-13 \ | |
| llvm-14-dev llvm-14-runtime llvm-14 \ | |
| llvm-15-dev llvm-15-runtime llvm-15 || true | |
| sudo apt-get autopurge -y || true | |
| # Now install LLVM 21 | |
| sudo apt-get install -y \ | |
| llvm-21 \ | |
| clang-21 \ | |
| libc++-21-dev \ | |
| libc++1-21 \ | |
| libc++abi-21-dev \ | |
| libc++abi1-21 | |
| # Clean up apt cache | |
| sudo apt-get clean | |
| sudo rm -rf /var/lib/apt/lists/* | |
| # Create symlinks to make clang-21 and coverage tools available without version suffix | |
| sudo ln -sf /usr/bin/clang-21 /usr/local/bin/clang | |
| sudo ln -sf /usr/bin/clang++-21 /usr/local/bin/clang++ | |
| sudo ln -sf /usr/bin/llvm-cov-21 /usr/local/bin/llvm-cov | |
| sudo ln -sf /usr/bin/llvm-profdata-21 /usr/local/bin/llvm-profdata | |
| sudo ln -sf /usr/bin/llvm-objdump-21 /usr/local/bin/llvm-objdump | |
| # Verify installation | |
| echo "=== Verifying LLVM 21 installation ===" | |
| which clang | |
| clang --version | head -1 | |
| llvm-cov --version | head -1 | |
| llvm-profdata merge --version 2>&1 | head -1 || echo "llvm-profdata installed" | |
| - name: Clean profraw files before tests | |
| run: | | |
| find . -name "*.profraw" -type f -delete | |
| echo "Cleaned all .profraw files" | |
| - name: Download build artifacts (tar archive) | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: tests-artifacts-for-coverage | |
| path: out/linux/x64 | |
| - name: Extract tests artifacts preserving permissions | |
| run: | | |
| tar -xzf out/linux/x64/tests-artifacts.tar.gz | |
| - name: View folder structure content | |
| run: | | |
| df -h . | |
| tree -I ".*|*.h|*.hpp|*.py" out/linux/x64 | |
| - name: Set ulimit and sysctl | |
| run: | | |
| # Increase max number of open file descriptors as much as allowed. | |
| TARGET=102400 | |
| HARD=$(ulimit -Hn) | |
| echo "Current hard limit for open files: $HARD" | |
| if [ "$HARD" != "unlimited" ] && [ "$HARD" -lt "$TARGET" ]; then | |
| echo "Target ($TARGET) is greater than hard limit ($HARD), using hard limit instead." | |
| TARGET="$HARD" | |
| fi | |
| # Try to set the soft limit; if it fails, just warn and continue. | |
| if ! ulimit -n "$TARGET"; then | |
| echo "WARNING: failed to increase ulimit -n to $TARGET, continuing with existing limit." | |
| fi | |
| # Adjust somaxconn; ignore failure if not permitted. | |
| if ! sudo sysctl -w net.core.somaxconn=8192; then | |
| echo "WARNING: failed to set net.core.somaxconn, continuing." | |
| fi | |
| shell: bash | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Update tman config.json | |
| run: | | |
| CONFIG_FILE="out/linux/x64/tests/local_registry/config.json" | |
| echo "Before update:" | |
| cat $CONFIG_FILE | |
| sed -i "s|\(file://\)[^\"]*\(out\/linux\/x64\/tests\/local_registry\)|\1${GITHUB_WORKSPACE}/\2|" $CONFIG_FILE | |
| echo "After update:" | |
| cat $CONFIG_FILE | |
| - name: Run Tests (ten_utils_unit_test) | |
| env: | |
| ASAN_OPTIONS: detect_leaks=1:detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1 | |
| MALLOC_CHECK_: 3 | |
| TEN_ENABLE_MEMORY_TRACKING: "true" | |
| TEN_ENABLE_BACKTRACE_DUMP: "true" | |
| # Set custom profraw filename pattern for better identification during debugging. | |
| # Coverage data will still be collected even if LLVM_PROFILE_FILE is not set, | |
| # but with default naming (default.profraw). Pattern: %p=PID, %m=unique ID. | |
| LLVM_PROFILE_FILE: "ten_utils_unit_test-%p-%m.profraw" | |
| run: | | |
| echo "=== Verifying clang version before test ===" | |
| which clang | |
| clang --version | head -1 | |
| chmod +x out/linux/x64/tests/standalone/ten_utils_unit_test | |
| out/linux/x64/tests/standalone/ten_utils_unit_test || { echo "test failed"; exit 1; } | |
| df -h . | |
| - name: Merge profraw for ten_utils_unit_test | |
| run: | | |
| mkdir -p out/linux/x64/coverage | |
| # Find profraw files for this test | |
| PROFRAW_FILES=$(find . -name "*.profraw" -type f) | |
| if [ -n "$PROFRAW_FILES" ]; then | |
| llvm-profdata merge -sparse $PROFRAW_FILES -o out/linux/x64/coverage/ten_utils_unit_test.profdata | |
| echo "Created profdata: ten_utils_unit_test.profdata" | |
| # Clean up profraw files for this test | |
| find . -name "*.profraw" -type f -delete | |
| fi | |
| - name: Clean up | |
| continue-on-error: true | |
| run: | | |
| rm -rf out/linux/x64/tests/standalone/ten_utils_unit_test | |
| df -h . | |
| - name: Run Tests (ten_runtime_unit_test) | |
| env: | |
| ASAN_OPTIONS: detect_leaks=1:detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1 | |
| MALLOC_CHECK_: 3 | |
| TEN_ENABLE_MEMORY_TRACKING: "true" | |
| TEN_ENABLE_BACKTRACE_DUMP: "true" | |
| # Set custom profraw filename pattern for better identification during debugging. | |
| # Coverage data will still be collected even if LLVM_PROFILE_FILE is not set, | |
| # but with default naming (default.profraw). Pattern: %p=PID, %m=unique ID. | |
| LLVM_PROFILE_FILE: "ten_runtime_unit_test-%p-%m.profraw" | |
| run: | | |
| echo "=== Verifying clang version before test ===" | |
| which clang | |
| clang --version | head -1 | |
| chmod +x out/linux/x64/tests/standalone/ten_runtime_unit_test | |
| out/linux/x64/tests/standalone/ten_runtime_unit_test || { echo "test failed"; exit 1; } | |
| df -h | |
| - name: Merge profraw for ten_runtime_unit_test | |
| run: | | |
| mkdir -p out/linux/x64/coverage | |
| # Find profraw files for this test | |
| PROFRAW_FILES=$(find . -name "*.profraw" -type f) | |
| if [ -n "$PROFRAW_FILES" ]; then | |
| llvm-profdata merge -sparse $PROFRAW_FILES -o out/linux/x64/coverage/ten_runtime_unit_test.profdata | |
| echo "Created profdata: ten_runtime_unit_test.profdata" | |
| # Clean up profraw files for this test | |
| find . -name "*.profraw" -type f -delete | |
| fi | |
| - name: Clean up | |
| continue-on-error: true | |
| run: | | |
| rm -rf out/linux/x64/tests/standalone/ten_runtime_unit_test | |
| df -h . | |
| - name: Run Tests (ten_rust standalone tests) | |
| env: | |
| ASAN_OPTIONS: detect_leaks=1:detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1 | |
| MALLOC_CHECK_: 3 | |
| TEN_ENABLE_MEMORY_TRACKING: "true" | |
| TEN_ENABLE_BACKTRACE_DUMP: "true" | |
| RUST_BACKTRACE: "full" | |
| # Set custom profraw filename pattern for better identification during debugging. | |
| # Coverage data will still be collected even if LLVM_PROFILE_FILE is not set, | |
| # but with default naming (default.profraw). Pattern: %p=PID, %m=unique ID. | |
| LLVM_PROFILE_FILE: "ten_rust-%p-%m.profraw" | |
| run: | | |
| echo "=== Verifying clang version before test ===" | |
| which clang | |
| clang --version | head -1 | |
| cd out/linux/x64/tests/standalone/ten_rust | |
| chmod +x unit_test | |
| chmod +x integration_test | |
| ./unit_test --nocapture --test-threads=1 || { echo "ten_rust unit test failed"; exit 1; } | |
| ./integration_test --nocapture --test-threads=1 || { echo "ten_rust integration test failed"; exit 1; } | |
| df -h . | |
| - name: Merge profraw for ten_rust tests | |
| run: | | |
| mkdir -p out/linux/x64/coverage | |
| # Find profraw files for this test | |
| PROFRAW_FILES=$(find . -name "*.profraw" -type f) | |
| if [ -n "$PROFRAW_FILES" ]; then | |
| llvm-profdata merge -sparse $PROFRAW_FILES -o out/linux/x64/coverage/ten_rust.profdata | |
| echo "Created profdata: ten_rust.profdata" | |
| # Clean up profraw files for this test | |
| find . -name "*.profraw" -type f -delete | |
| fi | |
| - name: Clean up | |
| continue-on-error: true | |
| run: | | |
| rm -rf out/linux/x64/tests/standalone/ten_rust | |
| df -h . | |
| - name: Run Tests (ten_manager standalone tests) | |
| env: | |
| ASAN_OPTIONS: detect_leaks=1:detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1 | |
| MALLOC_CHECK_: 3 | |
| TEN_ENABLE_MEMORY_TRACKING: "true" | |
| TEN_ENABLE_BACKTRACE_DUMP: "true" | |
| RUST_BACKTRACE: "full" | |
| # Set custom profraw filename pattern for better identification during debugging. | |
| # Coverage data will still be collected even if LLVM_PROFILE_FILE is not set, | |
| # but with default naming (default.profraw). Pattern: %p=PID, %m=unique ID. | |
| LLVM_PROFILE_FILE: "ten_manager-%p-%m.profraw" | |
| run: | | |
| echo "=== Verifying clang version before test ===" | |
| which clang | |
| clang --version | head -1 | |
| cd out/linux/x64/tests/standalone/ten_manager | |
| chmod +x unit_test | |
| chmod +x integration_test | |
| ./unit_test --nocapture --test-threads=1 || { echo "ten_manager unit test failed"; exit 1; } | |
| ./integration_test --nocapture --test-threads=1 || { echo "ten_manager integration test failed"; exit 1; } | |
| df -h . | |
| - name: Merge profraw for ten_manager standalone tests | |
| run: | | |
| mkdir -p out/linux/x64/coverage | |
| # Find profraw files for this test | |
| PROFRAW_FILES=$(find . -name "*.profraw" -type f) | |
| if [ -n "$PROFRAW_FILES" ]; then | |
| llvm-profdata merge -sparse $PROFRAW_FILES -o out/linux/x64/coverage/ten_manager.profdata | |
| echo "Created profdata: ten_manager.profdata" | |
| # Clean up profraw files for this test | |
| find . -name "*.profraw" -type f -delete | |
| fi | |
| - name: Clean up | |
| continue-on-error: true | |
| run: | | |
| rm -rf out/linux/x64/tests/standalone/ten_manager | |
| df -h . | |
| - name: Run Tests (ten_runtime_smoke_test) | |
| env: | |
| ASAN_OPTIONS: detect_leaks=1:detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1 | |
| MALLOC_CHECK_: 3 | |
| TEN_ENABLE_MEMORY_TRACKING: "true" | |
| TEN_ENABLE_BACKTRACE_DUMP: "true" | |
| # Set custom profraw filename pattern for better identification during debugging. | |
| # Coverage data will still be collected even if LLVM_PROFILE_FILE is not set, | |
| # but with default naming (default.profraw). Pattern: %p=PID, %m=unique ID. | |
| LLVM_PROFILE_FILE: "ten_runtime_smoke_test-%p-%m.profraw" | |
| run: | | |
| echo "=== Verifying clang version before test ===" | |
| which clang | |
| clang --version | head -1 | |
| chmod +x out/linux/x64/tests/standalone/ten_runtime_smoke_test | |
| out/linux/x64/tests/standalone/ten_runtime_smoke_test || { echo "test failed"; exit 1; } | |
| df -h . | |
| - name: Merge profraw for ten_runtime_smoke_test | |
| run: | | |
| mkdir -p out/linux/x64/coverage | |
| # Find profraw files for this test | |
| PROFRAW_FILES=$(find . -name "*.profraw" -type f) | |
| if [ -n "$PROFRAW_FILES" ]; then | |
| llvm-profdata merge -sparse $PROFRAW_FILES -o out/linux/x64/coverage/ten_runtime_smoke_test.profdata | |
| echo "Created profdata: ten_runtime_smoke_test.profdata" | |
| # Clean up profraw files for this test | |
| find . -name "*.profraw" -type f -delete | |
| fi | |
| - name: Upload test-standalone profdata | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-standalone-coverage | |
| path: | | |
| out/linux/x64/coverage/ten_utils_unit_test.profdata | |
| out/linux/x64/coverage/ten_runtime_unit_test.profdata | |
| out/linux/x64/coverage/ten_rust.profdata | |
| out/linux/x64/coverage/ten_runtime_smoke_test.profdata | |
| out/linux/x64/coverage/ten_manager.profdata | |
| - name: Clean up | |
| continue-on-error: true | |
| run: | | |
| # Clean up test directories | |
| rm -rf out/linux/x64/tests/standalone/ten_runtime_smoke_test | |
| # Clean up any remaining profraw files (should be none) | |
| find . -name "*.profraw" -type f -delete | |
| df -h . | |
| test-integration-ten-manager: | |
| needs: build | |
| runs-on: ubuntu-22.04 | |
| # Remove container to get more disk space | |
| # With debug mode enabled, we need more disk space for coverage instrumentation | |
| # container: | |
| # image: ghcr.io/ten-framework/ten_building_ubuntu2204 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Free up disk space on runner | |
| run: | | |
| echo "=== Initial disk space ===" | |
| df -h | |
| # Remove unnecessary pre-installed software on GitHub runner | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /opt/hostedtoolcache/CodeQL | |
| sudo docker image prune --all --force | |
| echo "=== Disk space after cleanup ===" | |
| df -h | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends \ | |
| build-essential \ | |
| wget \ | |
| curl \ | |
| git \ | |
| libssl-dev \ | |
| pkg-config \ | |
| cmake \ | |
| autoconf \ | |
| libtool \ | |
| tree \ | |
| zip \ | |
| unzip \ | |
| jq \ | |
| python3 \ | |
| python3-dev \ | |
| python3-pip \ | |
| python3-venv \ | |
| ca-certificates \ | |
| software-properties-common \ | |
| gnupg \ | |
| lsb-release \ | |
| libunwind-dev \ | |
| libasound2 \ | |
| libavformat-dev \ | |
| libavfilter-dev \ | |
| libx264-dev \ | |
| nasm \ | |
| yasm \ | |
| uuid-dev \ | |
| libexpat1-dev \ | |
| libcurl4-openssl-dev \ | |
| zlib1g-dev \ | |
| libncurses5-dev \ | |
| libffi-dev \ | |
| libreadline-dev \ | |
| libmsgpack-dev \ | |
| libcrypto++-dev | |
| # Clean up apt cache | |
| sudo apt-get clean | |
| sudo rm -rf /var/lib/apt/lists/* | |
| echo "=== Disk space after system dependencies ===" | |
| df -h | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: false | |
| - name: Trust working directory | |
| run: git config --global --add safe.directory "${GITHUB_WORKSPACE}" | |
| - name: Initialize and update submodules except portal/ | |
| run: | | |
| # Retrieve all submodule paths, excluding `portal/`. | |
| submodules=$(git config --file .gitmodules --get-regexp path | awk '$2 != "portal" { print $2 }') | |
| git submodule init | |
| for submodule in $submodules; do | |
| echo "Initializing submodule: $submodule" | |
| git submodule update --init --recursive --depth 1 "$submodule" | |
| done | |
| - name: Install LLVM 21 tools | |
| run: | | |
| # Add LLVM official repository for LLVM 21 | |
| wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc | |
| sudo add-apt-repository -y "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-21 main" | |
| sudo apt-get update | |
| # Purge preinstalled clang-13/14/15 BEFORE installing clang-21 (see build job for details) | |
| sudo apt-get purge -y clang-13 clang-14 clang-15 \ | |
| llvm-13-dev llvm-13-runtime llvm-13 \ | |
| llvm-14-dev llvm-14-runtime llvm-14 \ | |
| llvm-15-dev llvm-15-runtime llvm-15 || true | |
| sudo apt-get autopurge -y || true | |
| # Now install LLVM 21 | |
| sudo apt-get install -y \ | |
| llvm-21 \ | |
| clang-21 \ | |
| libc++-21-dev \ | |
| libc++1-21 \ | |
| libc++abi-21-dev \ | |
| libc++abi1-21 | |
| # Clean up apt cache | |
| sudo apt-get clean | |
| sudo rm -rf /var/lib/apt/lists/* | |
| # Create symlinks to make clang-21 and coverage tools available without version suffix | |
| sudo ln -sf /usr/bin/clang-21 /usr/local/bin/clang | |
| sudo ln -sf /usr/bin/clang++-21 /usr/local/bin/clang++ | |
| sudo ln -sf /usr/bin/llvm-cov-21 /usr/local/bin/llvm-cov | |
| sudo ln -sf /usr/bin/llvm-profdata-21 /usr/local/bin/llvm-profdata | |
| sudo ln -sf /usr/bin/llvm-objdump-21 /usr/local/bin/llvm-objdump | |
| # Verify installation | |
| echo "=== Verifying LLVM 21 installation ===" | |
| which clang | |
| clang --version | head -1 | |
| llvm-cov --version | head -1 | |
| llvm-profdata merge --version 2>&1 | head -1 || echo "llvm-profdata installed" | |
| - name: Clean profraw files before tests | |
| run: | | |
| find . -name "*.profraw" -type f -delete | |
| echo "Cleaned all .profraw files" | |
| - name: Download build artifacts (tar archive) | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: tests-artifacts-for-coverage | |
| path: out/linux/x64 | |
| - name: Extract tests artifacts preserving permissions | |
| run: | | |
| tar -xzf out/linux/x64/tests-artifacts.tar.gz | |
| - name: View folder structure content | |
| run: | | |
| df -h . | |
| tree -I ".*|*.h|*.hpp|*.py" out/linux/x64 | |
| - name: Set ulimit and sysctl | |
| run: | | |
| # Increase max number of open file descriptors as much as allowed. | |
| TARGET=102400 | |
| HARD=$(ulimit -Hn) | |
| echo "Current hard limit for open files: $HARD" | |
| if [ "$HARD" != "unlimited" ] && [ "$HARD" -lt "$TARGET" ]; then | |
| echo "Target ($TARGET) is greater than hard limit ($HARD), using hard limit instead." | |
| TARGET="$HARD" | |
| fi | |
| # Try to set the soft limit; if it fails, just warn and continue. | |
| if ! ulimit -n "$TARGET"; then | |
| echo "WARNING: failed to increase ulimit -n to $TARGET, continuing with existing limit." | |
| fi | |
| # Adjust somaxconn; ignore failure if not permitted. | |
| if ! sudo sysctl -w net.core.somaxconn=8192; then | |
| echo "WARNING: failed to set net.core.somaxconn, continuing." | |
| fi | |
| shell: bash | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Update tman config.json | |
| run: | | |
| CONFIG_FILE="out/linux/x64/tests/local_registry/config.json" | |
| echo "Before update:" | |
| cat $CONFIG_FILE | |
| sed -i "s|\(file://\)[^\"]*\(out\/linux\/x64\/tests\/local_registry\)|\1${GITHUB_WORKSPACE}/\2|" $CONFIG_FILE | |
| echo "After update:" | |
| cat $CONFIG_FILE | |
| - name: Install Python dependencies via script | |
| run: | | |
| python .github/tools/setup_pytest_dependencies.py | |
| df -h . | |
| - name: Run Tests (ten_manager pytest tests) | |
| env: | |
| ASAN_OPTIONS: detect_leaks=1:detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1 | |
| MALLOC_CHECK_: 3 | |
| TEN_ENABLE_MEMORY_TRACKING: "true" | |
| TEN_ENABLE_BACKTRACE_DUMP: "true" | |
| # Set custom profraw filename pattern for better identification during debugging. | |
| # Coverage data will still be collected even if LLVM_PROFILE_FILE is not set, | |
| # but with default naming (default.profraw). Pattern: %p=PID, %m=unique ID. | |
| LLVM_PROFILE_FILE: "test-integration-ten-manager-%p-%m.profraw" | |
| run: | | |
| echo "=== Verifying clang version before test ===" | |
| which clang | |
| clang --version | head -1 | |
| cd out/linux/x64/ | |
| pytest -s tests/ten_manager/ | |
| df -h . | |
| - name: Merge profraw files for test-integration-ten-manager | |
| run: | | |
| mkdir -p out/linux/x64/coverage | |
| # Find all profraw files recursively | |
| PROFRAW_FILES=$(find . -name "*.profraw" -type f) | |
| if [ -z "$PROFRAW_FILES" ]; then | |
| echo "No profraw files found" | |
| exit 1 | |
| fi | |
| echo "Found profraw files:" | |
| echo "$PROFRAW_FILES" | |
| # Merge profraw files into profdata | |
| llvm-profdata merge -sparse $PROFRAW_FILES -o out/linux/x64/coverage/test-integration-ten-manager.profdata | |
| echo "Merged profdata file created: test-integration-ten-manager.profdata" | |
| llvm-profdata show out/linux/x64/coverage/test-integration-ten-manager.profdata | head -20 | |
| - name: Upload test-integration-ten-manager profdata | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-integration-ten-manager-coverage | |
| path: out/linux/x64/coverage/test-integration-ten-manager.profdata | |
| - name: Clean up | |
| continue-on-error: true | |
| run: | | |
| # Clean up profraw files for this test | |
| find . -name "*.profraw" -type f -delete | |
| rm -rf out/linux/x64/tests/ten_manager | |
| df -h . | |
| test-integration-cpp: | |
| needs: build | |
| runs-on: ubuntu-22.04 | |
| # Remove container to get more disk space | |
| # With debug mode enabled, we need more disk space for coverage instrumentation | |
| # container: | |
| # image: ghcr.io/ten-framework/ten_building_ubuntu2204 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Free up disk space on runner | |
| run: | | |
| echo "=== Initial disk space ===" | |
| df -h | |
| # Remove unnecessary pre-installed software on GitHub runner | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /opt/hostedtoolcache/CodeQL | |
| sudo docker image prune --all --force | |
| echo "=== Disk space after cleanup ===" | |
| df -h | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends \ | |
| build-essential \ | |
| wget \ | |
| curl \ | |
| git \ | |
| libssl-dev \ | |
| pkg-config \ | |
| cmake \ | |
| autoconf \ | |
| libtool \ | |
| tree \ | |
| zip \ | |
| unzip \ | |
| jq \ | |
| python3 \ | |
| python3-dev \ | |
| python3-pip \ | |
| python3-venv \ | |
| ca-certificates \ | |
| software-properties-common \ | |
| gnupg \ | |
| lsb-release \ | |
| libunwind-dev \ | |
| libasound2 \ | |
| libavformat-dev \ | |
| libavfilter-dev \ | |
| libx264-dev \ | |
| nasm \ | |
| yasm \ | |
| uuid-dev \ | |
| libexpat1-dev \ | |
| libcurl4-openssl-dev \ | |
| zlib1g-dev \ | |
| libncurses5-dev \ | |
| libffi-dev \ | |
| libreadline-dev \ | |
| libmsgpack-dev \ | |
| libcrypto++-dev | |
| # Clean up apt cache | |
| sudo apt-get clean | |
| sudo rm -rf /var/lib/apt/lists/* | |
| echo "=== Disk space after system dependencies ===" | |
| df -h | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: false | |
| - name: Trust working directory | |
| run: git config --global --add safe.directory "${GITHUB_WORKSPACE}" | |
| - name: Initialize and update submodules except portal/ | |
| run: | | |
| # Retrieve all submodule paths, excluding `portal/`. | |
| submodules=$(git config --file .gitmodules --get-regexp path | awk '$2 != "portal" { print $2 }') | |
| git submodule init | |
| for submodule in $submodules; do | |
| echo "Initializing submodule: $submodule" | |
| git submodule update --init --recursive --depth 1 "$submodule" | |
| done | |
| - name: Install LLVM 21 tools | |
| run: | | |
| # Add LLVM official repository for LLVM 21 | |
| wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc | |
| sudo add-apt-repository -y "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-21 main" | |
| sudo apt-get update | |
| # Purge preinstalled clang-13/14/15 BEFORE installing clang-21 (see build job for details) | |
| sudo apt-get purge -y clang-13 clang-14 clang-15 \ | |
| llvm-13-dev llvm-13-runtime llvm-13 \ | |
| llvm-14-dev llvm-14-runtime llvm-14 \ | |
| llvm-15-dev llvm-15-runtime llvm-15 || true | |
| sudo apt-get autopurge -y || true | |
| # Now install LLVM 21 | |
| sudo apt-get install -y \ | |
| llvm-21 \ | |
| clang-21 \ | |
| libc++-21-dev \ | |
| libc++1-21 \ | |
| libc++abi-21-dev \ | |
| libc++abi1-21 | |
| # Clean up apt cache | |
| sudo apt-get clean | |
| sudo rm -rf /var/lib/apt/lists/* | |
| # Create symlinks to make clang-21 and coverage tools available without version suffix | |
| sudo ln -sf /usr/bin/clang-21 /usr/local/bin/clang | |
| sudo ln -sf /usr/bin/clang++-21 /usr/local/bin/clang++ | |
| sudo ln -sf /usr/bin/llvm-cov-21 /usr/local/bin/llvm-cov | |
| sudo ln -sf /usr/bin/llvm-profdata-21 /usr/local/bin/llvm-profdata | |
| sudo ln -sf /usr/bin/llvm-objdump-21 /usr/local/bin/llvm-objdump | |
| # Verify installation | |
| echo "=== Verifying LLVM 21 installation ===" | |
| which clang | |
| clang --version | head -1 | |
| llvm-cov --version | head -1 | |
| llvm-profdata merge --version 2>&1 | head -1 || echo "llvm-profdata installed" | |
| - name: Clean profraw files before tests | |
| run: | | |
| find . -name "*.profraw" -type f -delete | |
| echo "Cleaned all .profraw files" | |
| - name: Download build artifacts (tar archive) | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: tests-artifacts-for-coverage | |
| path: out/linux/x64 | |
| - name: Extract tests artifacts preserving permissions | |
| run: | | |
| tar -xzf out/linux/x64/tests-artifacts.tar.gz | |
| - name: View folder structure content | |
| run: | | |
| df -h . | |
| tree -I ".*|*.h|*.hpp|*.py" out/linux/x64 | |
| - name: Set ulimit and sysctl | |
| run: | | |
| # Increase max number of open file descriptors as much as allowed. | |
| TARGET=102400 | |
| HARD=$(ulimit -Hn) | |
| echo "Current hard limit for open files: $HARD" | |
| if [ "$HARD" != "unlimited" ] && [ "$HARD" -lt "$TARGET" ]; then | |
| echo "Target ($TARGET) is greater than hard limit ($HARD), using hard limit instead." | |
| TARGET="$HARD" | |
| fi | |
| # Try to set the soft limit; if it fails, just warn and continue. | |
| if ! ulimit -n "$TARGET"; then | |
| echo "WARNING: failed to increase ulimit -n to $TARGET, continuing with existing limit." | |
| fi | |
| # Adjust somaxconn; ignore failure if not permitted. | |
| if ! sudo sysctl -w net.core.somaxconn=8192; then | |
| echo "WARNING: failed to set net.core.somaxconn, continuing." | |
| fi | |
| shell: bash | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Update tman config.json | |
| run: | | |
| CONFIG_FILE="out/linux/x64/tests/local_registry/config.json" | |
| echo "Before update:" | |
| cat $CONFIG_FILE | |
| sed -i "s|\(file://\)[^\"]*\(out\/linux\/x64\/tests\/local_registry\)|\1${GITHUB_WORKSPACE}/\2|" $CONFIG_FILE | |
| echo "After update:" | |
| cat $CONFIG_FILE | |
| - name: Install Python dependencies via script | |
| run: | | |
| python .github/tools/setup_pytest_dependencies.py | |
| df -h . | |
| - name: Run tests (ten_runtime C++ integration tests) | |
| env: | |
| ASAN_OPTIONS: detect_leaks=1:detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1 | |
| MALLOC_CHECK_: 3 | |
| TEN_ENABLE_MEMORY_TRACKING: "true" | |
| TEN_ENABLE_BACKTRACE_DUMP: "true" | |
| GOTRACEBACK: crash | |
| # Set custom profraw filename pattern for better identification during debugging. | |
| # Coverage data will still be collected even if LLVM_PROFILE_FILE is not set, | |
| # but with default naming (default.profraw). Pattern: %p=PID, %m=unique ID. | |
| LLVM_PROFILE_FILE: "test-integration-cpp-%p-%m.profraw" | |
| run: | | |
| echo "=== Verifying clang version before test ===" | |
| which clang | |
| clang --version | head -1 | |
| df -h . | |
| export PATH=$(pwd)/core/ten_gn:$PATH | |
| cd out/linux/x64/ | |
| pytest -s tests/ten_runtime/integration/cpp/ | |
| df -h . | |
| - name: Merge profraw files for test-integration-cpp | |
| run: | | |
| mkdir -p out/linux/x64/coverage | |
| # Find all profraw files recursively | |
| PROFRAW_FILES=$(find . -name "*.profraw" -type f) | |
| if [ -z "$PROFRAW_FILES" ]; then | |
| echo "No profraw files found" | |
| exit 1 | |
| fi | |
| echo "Found profraw files:" | |
| echo "$PROFRAW_FILES" | |
| # Merge profraw files into profdata | |
| llvm-profdata merge -sparse $PROFRAW_FILES -o out/linux/x64/coverage/test-integration-cpp.profdata | |
| echo "Merged profdata file created: test-integration-cpp.profdata" | |
| llvm-profdata show out/linux/x64/coverage/test-integration-cpp.profdata | head -20 | |
| - name: Upload test-integration-cpp profdata | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-integration-cpp-coverage | |
| path: | | |
| out/linux/x64/coverage/test-integration-cpp.profdata | |
| - name: Clean up | |
| continue-on-error: true | |
| run: | | |
| find . -name "*.profraw" -type f -delete | |
| df -h . | |
| test-integration-go: | |
| needs: build | |
| runs-on: ubuntu-22.04 | |
| # Remove container to get more disk space | |
| # With debug mode enabled, we need more disk space for coverage instrumentation | |
| # container: | |
| # image: ghcr.io/ten-framework/ten_building_ubuntu2204 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Free up disk space on runner | |
| run: | | |
| echo "=== Initial disk space ===" | |
| df -h | |
| # Remove unnecessary pre-installed software on GitHub runner | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /opt/hostedtoolcache/CodeQL | |
| sudo docker image prune --all --force | |
| echo "=== Disk space after cleanup ===" | |
| df -h | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends \ | |
| build-essential \ | |
| wget \ | |
| curl \ | |
| git \ | |
| libssl-dev \ | |
| pkg-config \ | |
| cmake \ | |
| autoconf \ | |
| libtool \ | |
| tree \ | |
| zip \ | |
| unzip \ | |
| jq \ | |
| python3 \ | |
| python3-dev \ | |
| python3-pip \ | |
| python3-venv \ | |
| ca-certificates \ | |
| software-properties-common \ | |
| gnupg \ | |
| lsb-release \ | |
| libunwind-dev \ | |
| libasound2 \ | |
| libavformat-dev \ | |
| libavfilter-dev \ | |
| libx264-dev \ | |
| nasm \ | |
| yasm \ | |
| uuid-dev \ | |
| libexpat1-dev \ | |
| libcurl4-openssl-dev \ | |
| zlib1g-dev \ | |
| libncurses5-dev \ | |
| libffi-dev \ | |
| libreadline-dev \ | |
| libmsgpack-dev \ | |
| libcrypto++-dev | |
| # Clean up apt cache | |
| sudo apt-get clean | |
| sudo rm -rf /var/lib/apt/lists/* | |
| echo "=== Disk space after system dependencies ===" | |
| df -h | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: false | |
| - name: Trust working directory | |
| run: git config --global --add safe.directory "${GITHUB_WORKSPACE}" | |
| - name: Initialize and update submodules except portal/ | |
| run: | | |
| # Retrieve all submodule paths, excluding `portal/`. | |
| submodules=$(git config --file .gitmodules --get-regexp path | awk '$2 != "portal" { print $2 }') | |
| git submodule init | |
| for submodule in $submodules; do | |
| echo "Initializing submodule: $submodule" | |
| git submodule update --init --recursive --depth 1 "$submodule" | |
| done | |
| - name: Install LLVM 21 tools | |
| run: | | |
| # Add LLVM official repository for LLVM 21 | |
| wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc | |
| sudo add-apt-repository -y "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-21 main" | |
| sudo apt-get update | |
| # Purge preinstalled clang-13/14/15 BEFORE installing clang-21 (see build job for details) | |
| sudo apt-get purge -y clang-13 clang-14 clang-15 \ | |
| llvm-13-dev llvm-13-runtime llvm-13 \ | |
| llvm-14-dev llvm-14-runtime llvm-14 \ | |
| llvm-15-dev llvm-15-runtime llvm-15 || true | |
| sudo apt-get autopurge -y || true | |
| # Now install LLVM 21 | |
| sudo apt-get install -y \ | |
| llvm-21 \ | |
| clang-21 \ | |
| libc++-21-dev \ | |
| libc++1-21 \ | |
| libc++abi-21-dev \ | |
| libc++abi1-21 | |
| # Clean up apt cache | |
| sudo apt-get clean | |
| sudo rm -rf /var/lib/apt/lists/* | |
| # Create symlinks to make clang-21 and coverage tools available without version suffix | |
| sudo ln -sf /usr/bin/clang-21 /usr/local/bin/clang | |
| sudo ln -sf /usr/bin/clang++-21 /usr/local/bin/clang++ | |
| sudo ln -sf /usr/bin/llvm-cov-21 /usr/local/bin/llvm-cov | |
| sudo ln -sf /usr/bin/llvm-profdata-21 /usr/local/bin/llvm-profdata | |
| sudo ln -sf /usr/bin/llvm-objdump-21 /usr/local/bin/llvm-objdump | |
| # Verify installation | |
| echo "=== Verifying LLVM 21 installation ===" | |
| which clang | |
| clang --version | head -1 | |
| llvm-cov --version | head -1 | |
| llvm-profdata merge --version 2>&1 | head -1 || echo "llvm-profdata installed" | |
| - name: Clean profraw files before tests | |
| run: | | |
| find . -name "*.profraw" -type f -delete | |
| echo "Cleaned all .profraw files" | |
| - name: Download build artifacts (tar archive) | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: tests-artifacts-for-coverage | |
| path: out/linux/x64 | |
| - name: Extract tests artifacts preserving permissions | |
| run: | | |
| tar -xzf out/linux/x64/tests-artifacts.tar.gz | |
| - name: View folder structure content | |
| run: | | |
| df -h . | |
| tree -I ".*|*.h|*.hpp|*.py" out/linux/x64 | |
| - name: Set ulimit and sysctl | |
| run: | | |
| # Increase max number of open file descriptors as much as allowed. | |
| TARGET=102400 | |
| HARD=$(ulimit -Hn) | |
| echo "Current hard limit for open files: $HARD" | |
| if [ "$HARD" != "unlimited" ] && [ "$HARD" -lt "$TARGET" ]; then | |
| echo "Target ($TARGET) is greater than hard limit ($HARD), using hard limit instead." | |
| TARGET="$HARD" | |
| fi | |
| # Try to set the soft limit; if it fails, just warn and continue. | |
| if ! ulimit -n "$TARGET"; then | |
| echo "WARNING: failed to increase ulimit -n to $TARGET, continuing with existing limit." | |
| fi | |
| # Adjust somaxconn; ignore failure if not permitted. | |
| if ! sudo sysctl -w net.core.somaxconn=8192; then | |
| echo "WARNING: failed to set net.core.somaxconn, continuing." | |
| fi | |
| shell: bash | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Update tman config.json | |
| run: | | |
| CONFIG_FILE="out/linux/x64/tests/local_registry/config.json" | |
| echo "Before update:" | |
| cat $CONFIG_FILE | |
| sed -i "s|\(file://\)[^\"]*\(out\/linux\/x64\/tests\/local_registry\)|\1${GITHUB_WORKSPACE}/\2|" $CONFIG_FILE | |
| echo "After update:" | |
| cat $CONFIG_FILE | |
| - name: Install Python dependencies via script | |
| run: | | |
| python .github/tools/setup_pytest_dependencies.py | |
| df -h . | |
| - name: Run tests (ten_runtime Go integration tests) | |
| env: | |
| ASAN_OPTIONS: detect_leaks=1:detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1 | |
| MALLOC_CHECK_: 3 | |
| TEN_ENABLE_MEMORY_TRACKING: "true" | |
| TEN_ENABLE_BACKTRACE_DUMP: "true" | |
| GOTRACEBACK: crash | |
| # Set custom profraw filename pattern for better identification during debugging. | |
| # Coverage data will still be collected even if LLVM_PROFILE_FILE is not set, | |
| # but with default naming (default.profraw). Pattern: %p=PID, %m=unique ID. | |
| LLVM_PROFILE_FILE: "test-integration-go-%p-%m.profraw" | |
| run: | | |
| echo "=== Verifying clang version before test ===" | |
| which clang | |
| clang --version | head -1 | |
| df -h . | |
| export PATH=$(pwd)/core/ten_gn:$PATH | |
| cd out/linux/x64/ | |
| pytest -s tests/ten_runtime/integration/go/ | |
| df -h . | |
| - name: Merge profraw files for test-integration-go | |
| run: | | |
| mkdir -p out/linux/x64/coverage | |
| # Find all profraw files recursively | |
| PROFRAW_FILES=$(find . -name "*.profraw" -type f) | |
| if [ -z "$PROFRAW_FILES" ]; then | |
| echo "No profraw files found" | |
| exit 1 | |
| fi | |
| echo "Found profraw files:" | |
| echo "$PROFRAW_FILES" | |
| # Merge profraw files into profdata | |
| llvm-profdata merge -sparse $PROFRAW_FILES -o out/linux/x64/coverage/test-integration-go.profdata | |
| echo "Merged profdata file created: test-integration-go.profdata" | |
| llvm-profdata show out/linux/x64/coverage/test-integration-go.profdata | head -20 | |
| - name: Upload test-integration-go profdata | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-integration-go-coverage | |
| path: | | |
| out/linux/x64/coverage/test-integration-go.profdata | |
| - name: Clean up | |
| continue-on-error: true | |
| run: | | |
| find . -name "*.profraw" -type f -delete | |
| df -h . | |
| test-integration-python: | |
| needs: build | |
| runs-on: ubuntu-22.04 | |
| # Remove container to get more disk space | |
| # With debug mode enabled, we need more disk space for coverage instrumentation | |
| # container: | |
| # image: ghcr.io/ten-framework/ten_building_ubuntu2204 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Free up disk space on runner | |
| run: | | |
| echo "=== Initial disk space ===" | |
| df -h | |
| # Remove unnecessary pre-installed software on GitHub runner | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /opt/hostedtoolcache/CodeQL | |
| sudo docker image prune --all --force | |
| echo "=== Disk space after cleanup ===" | |
| df -h | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends \ | |
| build-essential \ | |
| wget \ | |
| curl \ | |
| git \ | |
| libssl-dev \ | |
| pkg-config \ | |
| cmake \ | |
| autoconf \ | |
| libtool \ | |
| tree \ | |
| zip \ | |
| unzip \ | |
| jq \ | |
| python3 \ | |
| python3-dev \ | |
| python3-pip \ | |
| python3-venv \ | |
| ca-certificates \ | |
| software-properties-common \ | |
| gnupg \ | |
| lsb-release \ | |
| libunwind-dev \ | |
| libasound2 \ | |
| libavformat-dev \ | |
| libavfilter-dev \ | |
| libx264-dev \ | |
| nasm \ | |
| yasm \ | |
| uuid-dev \ | |
| libexpat1-dev \ | |
| libcurl4-openssl-dev \ | |
| zlib1g-dev \ | |
| libncurses5-dev \ | |
| libffi-dev \ | |
| libreadline-dev \ | |
| libmsgpack-dev \ | |
| libcrypto++-dev | |
| # Clean up apt cache | |
| sudo apt-get clean | |
| sudo rm -rf /var/lib/apt/lists/* | |
| echo "=== Disk space after system dependencies ===" | |
| df -h | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: false | |
| - name: Trust working directory | |
| run: git config --global --add safe.directory "${GITHUB_WORKSPACE}" | |
| - name: Initialize and update submodules except portal/ | |
| run: | | |
| # Retrieve all submodule paths, excluding `portal/`. | |
| submodules=$(git config --file .gitmodules --get-regexp path | awk '$2 != "portal" { print $2 }') | |
| git submodule init | |
| for submodule in $submodules; do | |
| echo "Initializing submodule: $submodule" | |
| git submodule update --init --recursive --depth 1 "$submodule" | |
| done | |
| - name: Install LLVM 21 tools | |
| run: | | |
| # Add LLVM official repository for LLVM 21 | |
| wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc | |
| sudo add-apt-repository -y "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-21 main" | |
| sudo apt-get update | |
| # Purge preinstalled clang-13/14/15 BEFORE installing clang-21 (see build job for details) | |
| sudo apt-get purge -y clang-13 clang-14 clang-15 \ | |
| llvm-13-dev llvm-13-runtime llvm-13 \ | |
| llvm-14-dev llvm-14-runtime llvm-14 \ | |
| llvm-15-dev llvm-15-runtime llvm-15 || true | |
| sudo apt-get autopurge -y || true | |
| # Now install LLVM 21 | |
| sudo apt-get install -y \ | |
| llvm-21 \ | |
| clang-21 \ | |
| libc++-21-dev \ | |
| libc++1-21 \ | |
| libc++abi-21-dev \ | |
| libc++abi1-21 | |
| # Clean up apt cache | |
| sudo apt-get clean | |
| sudo rm -rf /var/lib/apt/lists/* | |
| # Create symlinks to make clang-21 and coverage tools available without version suffix | |
| sudo ln -sf /usr/bin/clang-21 /usr/local/bin/clang | |
| sudo ln -sf /usr/bin/clang++-21 /usr/local/bin/clang++ | |
| sudo ln -sf /usr/bin/llvm-cov-21 /usr/local/bin/llvm-cov | |
| sudo ln -sf /usr/bin/llvm-profdata-21 /usr/local/bin/llvm-profdata | |
| sudo ln -sf /usr/bin/llvm-objdump-21 /usr/local/bin/llvm-objdump | |
| # Verify installation | |
| echo "=== Verifying LLVM 21 installation ===" | |
| which clang | |
| clang --version | head -1 | |
| llvm-cov --version | head -1 | |
| llvm-profdata merge --version 2>&1 | head -1 || echo "llvm-profdata installed" | |
| - name: Clean profraw files before tests | |
| run: | | |
| find . -name "*.profraw" -type f -delete | |
| echo "Cleaned all .profraw files" | |
| - name: Download build artifacts (tar archive) | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: tests-artifacts-for-coverage | |
| path: out/linux/x64 | |
| - name: Extract tests artifacts preserving permissions | |
| run: | | |
| tar -xzf out/linux/x64/tests-artifacts.tar.gz | |
| - name: View folder structure content | |
| run: | | |
| df -h . | |
| tree -I ".*|*.h|*.hpp|*.py" out/linux/x64 | |
| - name: Set ulimit and sysctl | |
| run: | | |
| # Increase max number of open file descriptors as much as allowed. | |
| TARGET=102400 | |
| HARD=$(ulimit -Hn) | |
| echo "Current hard limit for open files: $HARD" | |
| if [ "$HARD" != "unlimited" ] && [ "$HARD" -lt "$TARGET" ]; then | |
| echo "Target ($TARGET) is greater than hard limit ($HARD), using hard limit instead." | |
| TARGET="$HARD" | |
| fi | |
| # Try to set the soft limit; if it fails, just warn and continue. | |
| if ! ulimit -n "$TARGET"; then | |
| echo "WARNING: failed to increase ulimit -n to $TARGET, continuing with existing limit." | |
| fi | |
| # Adjust somaxconn; ignore failure if not permitted. | |
| if ! sudo sysctl -w net.core.somaxconn=8192; then | |
| echo "WARNING: failed to set net.core.somaxconn, continuing." | |
| fi | |
| shell: bash | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Update tman config.json | |
| run: | | |
| CONFIG_FILE="out/linux/x64/tests/local_registry/config.json" | |
| echo "Before update:" | |
| cat $CONFIG_FILE | |
| sed -i "s|\(file://\)[^\"]*\(out\/linux\/x64\/tests\/local_registry\)|\1${GITHUB_WORKSPACE}/\2|" $CONFIG_FILE | |
| echo "After update:" | |
| cat $CONFIG_FILE | |
| - name: Install Python dependencies via script | |
| run: | | |
| python .github/tools/setup_pytest_dependencies.py | |
| df -h . | |
| - name: Run tests (ten_runtime Python integration tests) | |
| env: | |
| ASAN_OPTIONS: detect_leaks=1:detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1 | |
| MALLOC_CHECK_: 3 | |
| TEN_ENABLE_MEMORY_TRACKING: "true" | |
| TEN_ENABLE_BACKTRACE_DUMP: "true" | |
| GOTRACEBACK: crash | |
| # Set custom profraw filename pattern for better identification during debugging. | |
| # Coverage data will still be collected even if LLVM_PROFILE_FILE is not set, | |
| # but with default naming (default.profraw). Pattern: %p=PID, %m=unique ID. | |
| LLVM_PROFILE_FILE: "test-integration-python-%p-%m.profraw" | |
| run: | | |
| echo "=== Verifying clang version before test ===" | |
| which clang | |
| clang --version | head -1 | |
| curl -fsSL https://ollama.com/install.sh | sh | |
| # Start Ollama service. | |
| ollama serve & | |
| # Wait for Ollama to be fully operational. | |
| for i in {1..30}; do | |
| if curl -s http://localhost:11434 >/dev/null; then | |
| echo "✓ Ollama is running" | |
| break | |
| fi | |
| echo "Waiting for Ollama to start..." | |
| sleep 2 | |
| done | |
| ollama pull smollm:135m | |
| # Verify model is pulled correctly. | |
| ollama list | |
| df -h . | |
| export PATH=$(pwd)/core/ten_gn:$PATH | |
| cd out/linux/x64/ | |
| pytest -s tests/ten_runtime/integration/python/ | |
| df -h . | |
| - name: Merge profraw files for test-integration-python | |
| run: | | |
| mkdir -p out/linux/x64/coverage | |
| # Find all profraw files recursively | |
| PROFRAW_FILES=$(find . -name "*.profraw" -type f) | |
| if [ -z "$PROFRAW_FILES" ]; then | |
| echo "No profraw files found" | |
| exit 1 | |
| fi | |
| echo "Found profraw files:" | |
| echo "$PROFRAW_FILES" | |
| # Merge profraw files into profdata | |
| llvm-profdata merge -sparse $PROFRAW_FILES -o out/linux/x64/coverage/test-integration-python.profdata | |
| echo "Merged profdata file created: test-integration-python.profdata" | |
| llvm-profdata show out/linux/x64/coverage/test-integration-python.profdata | head -20 | |
| - name: Upload test-integration-python profdata | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-integration-python-coverage | |
| path: | | |
| out/linux/x64/coverage/test-integration-python.profdata | |
| - name: Clean up | |
| continue-on-error: true | |
| run: | | |
| find . -name "*.profraw" -type f -delete | |
| df -h . | |
| test-integration-nodejs: | |
| needs: build | |
| runs-on: ubuntu-22.04 | |
| # Remove container to get more disk space | |
| # With debug mode enabled, we need more disk space for coverage instrumentation | |
| # container: | |
| # image: ghcr.io/ten-framework/ten_building_ubuntu2204 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - name: Free up disk space on runner | |
| run: | | |
| echo "=== Initial disk space ===" | |
| df -h | |
| # Remove unnecessary pre-installed software on GitHub runner | |
| sudo rm -rf /usr/share/dotnet | |
| sudo rm -rf /usr/local/lib/android | |
| sudo rm -rf /opt/ghc | |
| sudo rm -rf /opt/hostedtoolcache/CodeQL | |
| sudo docker image prune --all --force | |
| echo "=== Disk space after cleanup ===" | |
| df -h | |
| - name: Install system dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends \ | |
| build-essential \ | |
| wget \ | |
| curl \ | |
| git \ | |
| libssl-dev \ | |
| pkg-config \ | |
| cmake \ | |
| autoconf \ | |
| libtool \ | |
| tree \ | |
| zip \ | |
| unzip \ | |
| jq \ | |
| python3 \ | |
| python3-dev \ | |
| python3-pip \ | |
| python3-venv \ | |
| ca-certificates \ | |
| software-properties-common \ | |
| gnupg \ | |
| lsb-release \ | |
| libunwind-dev \ | |
| libasound2 \ | |
| libavformat-dev \ | |
| libavfilter-dev \ | |
| libx264-dev \ | |
| nasm \ | |
| yasm \ | |
| uuid-dev \ | |
| libexpat1-dev \ | |
| libcurl4-openssl-dev \ | |
| zlib1g-dev \ | |
| libncurses5-dev \ | |
| libffi-dev \ | |
| libreadline-dev \ | |
| libmsgpack-dev \ | |
| libcrypto++-dev | |
| # Clean up apt cache | |
| sudo apt-get clean | |
| sudo rm -rf /var/lib/apt/lists/* | |
| echo "=== Disk space after system dependencies ===" | |
| df -h | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: false | |
| - name: Trust working directory | |
| run: git config --global --add safe.directory "${GITHUB_WORKSPACE}" | |
| - name: Initialize and update submodules except portal/ | |
| run: | | |
| # Retrieve all submodule paths, excluding `portal/`. | |
| submodules=$(git config --file .gitmodules --get-regexp path | awk '$2 != "portal" { print $2 }') | |
| git submodule init | |
| for submodule in $submodules; do | |
| echo "Initializing submodule: $submodule" | |
| git submodule update --init --recursive --depth 1 "$submodule" | |
| done | |
| - name: Install LLVM 21 tools | |
| run: | | |
| # Add LLVM official repository for LLVM 21 | |
| wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc | |
| sudo add-apt-repository -y "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-21 main" | |
| sudo apt-get update | |
| # Purge preinstalled clang-13/14/15 BEFORE installing clang-21 (see build job for details) | |
| sudo apt-get purge -y clang-13 clang-14 clang-15 \ | |
| llvm-13-dev llvm-13-runtime llvm-13 \ | |
| llvm-14-dev llvm-14-runtime llvm-14 \ | |
| llvm-15-dev llvm-15-runtime llvm-15 || true | |
| sudo apt-get autopurge -y || true | |
| # Now install LLVM 21 | |
| sudo apt-get install -y \ | |
| llvm-21 \ | |
| clang-21 \ | |
| libc++-21-dev \ | |
| libc++1-21 \ | |
| libc++abi-21-dev \ | |
| libc++abi1-21 | |
| # Clean up apt cache | |
| sudo apt-get clean | |
| sudo rm -rf /var/lib/apt/lists/* | |
| # Create symlinks to make clang-21 and coverage tools available without version suffix | |
| sudo ln -sf /usr/bin/clang-21 /usr/local/bin/clang | |
| sudo ln -sf /usr/bin/clang++-21 /usr/local/bin/clang++ | |
| sudo ln -sf /usr/bin/llvm-cov-21 /usr/local/bin/llvm-cov | |
| sudo ln -sf /usr/bin/llvm-profdata-21 /usr/local/bin/llvm-profdata | |
| sudo ln -sf /usr/bin/llvm-objdump-21 /usr/local/bin/llvm-objdump | |
| # Verify installation | |
| echo "=== Verifying LLVM 21 installation ===" | |
| which clang | |
| clang --version | head -1 | |
| llvm-cov --version | head -1 | |
| llvm-profdata merge --version 2>&1 | head -1 || echo "llvm-profdata installed" | |
| - name: Install Go for building Go apps | |
| run: | | |
| # Install Go 1.24.3 for building Go applications with nodejs extensions | |
| export PATH=$PATH:/usr/local/go/bin:$HOME/go/bin | |
| go env -w GOFLAGS="-buildvcs=false" | |
| go install golang.org/dl/go1.24.3@latest | |
| export PATH=$PATH:$(go env GOPATH)/bin | |
| go1.24.3 download | |
| go1.24.3 version | |
| # Create symlink so 'go' points to go1.24.3 | |
| # This is needed because build scripts use 'go' command | |
| GO_BIN_DIR=$(go env GOPATH)/bin | |
| sudo ln -sf $GO_BIN_DIR/go1.24.3 /usr/local/bin/go | |
| # Verify | |
| which go | |
| go version | |
| - name: Clean profraw files before tests | |
| run: | | |
| find . -name "*.profraw" -type f -delete | |
| echo "Cleaned all .profraw files." | |
| - name: Download build artifacts (tar archive) | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: tests-artifacts-for-coverage | |
| path: out/linux/x64 | |
| - name: Extract tests artifacts preserving permissions | |
| run: | | |
| tar -xzf out/linux/x64/tests-artifacts.tar.gz | |
| - name: View folder structure content | |
| run: | | |
| df -h . | |
| tree -I ".*|*.h|*.hpp|*.py" out/linux/x64 | |
| - name: Set ulimit and sysctl | |
| run: | | |
| # Increase max number of open file descriptors as much as allowed. | |
| TARGET=102400 | |
| HARD=$(ulimit -Hn) | |
| echo "Current hard limit for open files: $HARD" | |
| if [ "$HARD" != "unlimited" ] && [ "$HARD" -lt "$TARGET" ]; then | |
| echo "Target ($TARGET) is greater than hard limit ($HARD), using hard limit instead." | |
| TARGET="$HARD" | |
| fi | |
| # Try to set the soft limit; if it fails, just warn and continue. | |
| if ! ulimit -n "$TARGET"; then | |
| echo "WARNING: failed to increase ulimit -n to $TARGET, continuing with existing limit." | |
| fi | |
| # Adjust somaxconn; ignore failure if not permitted. | |
| if ! sudo sysctl -w net.core.somaxconn=8192; then | |
| echo "WARNING: failed to set net.core.somaxconn, continuing." | |
| fi | |
| shell: bash | |
| - uses: actions/setup-node@v4 | |
| with: | |
| node-version: 20 | |
| - name: Update tman config.json | |
| run: | | |
| CONFIG_FILE="out/linux/x64/tests/local_registry/config.json" | |
| echo "Before update:" | |
| cat $CONFIG_FILE | |
| sed -i "s|\(file://\)[^\"]*\(out\/linux\/x64\/tests\/local_registry\)|\1${GITHUB_WORKSPACE}/\2|" $CONFIG_FILE | |
| echo "After update:" | |
| cat $CONFIG_FILE | |
| - name: Install Python dependencies via script | |
| run: | | |
| python .github/tools/setup_pytest_dependencies.py | |
| df -h . | |
| - name: Run tests (ten_runtime Nodejs integration tests) | |
| env: | |
| ASAN_OPTIONS: detect_leaks=1:detect_stack_use_after_return=1:color=always:unmap_shadow_on_exit=1:abort_on_error=1 | |
| MALLOC_CHECK_: 3 | |
| TEN_ENABLE_MEMORY_TRACKING: "true" | |
| TEN_ENABLE_BACKTRACE_DUMP: "true" | |
| GOTRACEBACK: crash | |
| # Set custom profraw filename pattern for better identification during debugging. | |
| # Coverage data will still be collected even if LLVM_PROFILE_FILE is not set, | |
| # but with default naming (default.profraw). Pattern: %p=PID, %m=unique ID. | |
| LLVM_PROFILE_FILE: "test-integration-nodejs-%p-%m.profraw" | |
| run: | | |
| df -h . | |
| # Add tgn and Go to PATH for building C++/Go apps with nodejs extensions | |
| export PATH=$(pwd)/core/ten_gn:/usr/local/go/bin:$HOME/go/bin:$PATH | |
| # Verify required tools are available | |
| echo "=== Verifying build tools ===" | |
| which tgn | |
| which go | |
| go version | |
| which clang | |
| clang --version | head -1 | |
| cd out/linux/x64/ | |
| pytest -s tests/ten_runtime/integration/nodejs/ | |
| df -h . | |
| - name: Merge profraw files for test-integration-nodejs | |
| run: | | |
| mkdir -p out/linux/x64/coverage | |
| # Find all profraw files recursively | |
| PROFRAW_FILES=$(find . -name "*.profraw" -type f) | |
| if [ -z "$PROFRAW_FILES" ]; then | |
| echo "No profraw files found" | |
| exit 1 | |
| fi | |
| echo "Found profraw files:" | |
| echo "$PROFRAW_FILES" | |
| # Merge profraw files into profdata | |
| llvm-profdata merge -sparse $PROFRAW_FILES -o out/linux/x64/coverage/test-integration-nodejs.profdata | |
| echo "Merged profdata file created: test-integration-nodejs.profdata" | |
| llvm-profdata show out/linux/x64/coverage/test-integration-nodejs.profdata | head -20 | |
| - name: Upload test-integration-nodejs profdata | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: test-integration-nodejs-coverage | |
| path: | | |
| out/linux/x64/coverage/test-integration-nodejs.profdata | |
| - name: Clean up | |
| continue-on-error: true | |
| run: | | |
| find . -name "*.profraw" -type f -delete | |
| df -h . | |
| gather-coverage-information: | |
| needs: | |
| [ | |
| test-standalone, | |
| test-integration-ten-manager, | |
| test-integration-cpp, | |
| test-integration-go, | |
| test-integration-python, | |
| test-integration-nodejs, | |
| ] | |
| runs-on: ubuntu-22.04 | |
| # Remove container to match build job's environment. | |
| # | |
| # Path consistency chain: | |
| # Build (no container, to save disk space) → binaries embed /home/runner/work/... paths | |
| # → llvm-cov → all paths in .lcov file are /home/runner/work/... | |
| # → Coveralls looks for files at these paths! | |
| # | |
| # Using container here would mount workspace at /__w/.../ | |
| # → Coveralls error: "Nothing to report" ❌ because coveralls app can't find any | |
| # code files under /home/runner/work/... as it reads lcov file. | |
| # | |
| # container: | |
| # image: ghcr.io/ten-framework/ten_building_ubuntu2204 | |
| defaults: | |
| run: | |
| shell: bash | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| submodules: false | |
| - name: Install basic dependencies | |
| run: | | |
| sudo apt-get update | |
| sudo apt-get install -y --no-install-recommends \ | |
| build-essential \ | |
| wget \ | |
| curl \ | |
| software-properties-common \ | |
| gnupg \ | |
| ca-certificates | |
| # Clean up apt cache | |
| sudo apt-get clean | |
| sudo rm -rf /var/lib/apt/lists/* | |
| - name: Install LLVM 21 tools | |
| run: | | |
| # Add LLVM official repository for LLVM 21 | |
| wget -qO- https://apt.llvm.org/llvm-snapshot.gpg.key | sudo tee /etc/apt/trusted.gpg.d/apt.llvm.org.asc | |
| sudo add-apt-repository -y "deb http://apt.llvm.org/jammy/ llvm-toolchain-jammy-21 main" | |
| sudo apt-get update | |
| sudo apt-get install -y llvm-21 clang-21 lcov | |
| # Clean up apt cache | |
| sudo apt-get clean | |
| sudo rm -rf /var/lib/apt/lists/* | |
| # Purge preinstalled clang-13/14/15 to prevent header conflicts. | |
| sudo apt-get purge -y clang-13 clang-14 clang-15 \ | |
| llvm-13-dev llvm-13-runtime llvm-13 \ | |
| llvm-14-dev llvm-14-runtime llvm-14 \ | |
| llvm-15-dev llvm-15-runtime llvm-15 || true | |
| sudo apt-get autopurge -y || true | |
| # Create symlinks to make clang-21 and coverage tools available without version suffix | |
| sudo ln -sf /usr/bin/clang-21 /usr/local/bin/clang | |
| sudo ln -sf /usr/bin/clang++-21 /usr/local/bin/clang++ | |
| sudo ln -sf /usr/bin/llvm-cov-21 /usr/local/bin/llvm-cov | |
| sudo ln -sf /usr/bin/llvm-profdata-21 /usr/local/bin/llvm-profdata | |
| sudo ln -sf /usr/bin/llvm-objdump-21 /usr/local/bin/llvm-objdump | |
| # Verify installation | |
| echo "=== Verifying LLVM 21 installation ===" | |
| which clang | |
| clang --version | head -1 | |
| llvm-cov --version | head -1 | |
| llvm-profdata merge --version 2>&1 | head -1 || echo "llvm-profdata installed" | |
| lcov --version | |
| - name: Download binaries and libraries for coverage analysis | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: coverage-binaries-and-libs | |
| path: . | |
| - name: Extract binaries and libraries | |
| run: | | |
| tar -xzf coverage-binaries-and-libs.tar.gz | |
| # Move to target location | |
| mkdir -p out/linux/x64/coverage | |
| mv binaries_and_libs out/linux/x64/coverage/binaries_and_libs | |
| echo "Extracted binaries and libraries:" | |
| find out/linux/x64/coverage/binaries_and_libs -type f | |
| - name: Download all profdata files | |
| uses: actions/download-artifact@v4 | |
| with: | |
| pattern: "*-coverage" | |
| path: profdata_artifacts | |
| merge-multiple: false | |
| - name: List downloaded profdata files | |
| run: | | |
| echo "Downloaded profdata artifacts:" | |
| find profdata_artifacts -name "*.profdata" -type f | |
| # Create coverage directory | |
| mkdir -p out/linux/x64/coverage | |
| - name: Gather coverage information | |
| run: | | |
| CORE_BINARIES_AND_LIBS=( | |
| "out/linux/x64/coverage/binaries_and_libs/ten_utils_unit_test" | |
| "out/linux/x64/coverage/binaries_and_libs/ten_rust_unit_test" | |
| "out/linux/x64/coverage/binaries_and_libs/ten_rust_integration_test" | |
| "out/linux/x64/coverage/binaries_and_libs/ten_manager_unit_test" | |
| "out/linux/x64/coverage/binaries_and_libs/ten_manager_integration_test" | |
| "out/linux/x64/coverage/binaries_and_libs/tman" | |
| "out/linux/x64/coverage/binaries_and_libs/ten_runtime_smoke_test" | |
| "out/linux/x64/coverage/binaries_and_libs/ten_runtime_unit_test" | |
| "out/linux/x64/coverage/binaries_and_libs/libten_runtime.so" | |
| "out/linux/x64/coverage/binaries_and_libs/libten_runtime_go.so" | |
| "out/linux/x64/coverage/binaries_and_libs/libten_runtime_python.so" | |
| "out/linux/x64/coverage/binaries_and_libs/libten_runtime_nodejs.node" | |
| "out/linux/x64/coverage/binaries_and_libs/libten_utils.so" | |
| "out/linux/x64/coverage/binaries_and_libs/libten_rust.a" | |
| ) | |
| # Find and merge all profdata files | |
| PROFDATA_FILES=$(find profdata_artifacts -name "*.profdata" -type f) | |
| echo "Found profdata files:" | |
| echo "$PROFDATA_FILES" | |
| llvm-profdata merge -sparse $PROFDATA_FILES -o out/linux/x64/coverage/merged.profdata | |
| echo "Intermediate .profdata file:" | |
| llvm-profdata show out/linux/x64/coverage/merged.profdata | |
| # Use --object flag to ensure all binaries and libraries are processed correctly | |
| # This prevents llvm-cov from ignoring shared libraries when executables are present | |
| OBJECT_ARGS=() | |
| for bin in "${CORE_BINARIES_AND_LIBS[@]}"; do | |
| if [ -f "$bin" ]; then | |
| OBJECT_ARGS+=(--object "$bin") | |
| fi | |
| done | |
| llvm-cov export \ | |
| --ignore-filename-regex='(^|/)\.cargo/|(^|/).rustup/|(^|/)out/|(^|/)tests/|(^|/)third_party/|(^|/)python3.10/|(^|/)packages/' \ | |
| --format=lcov \ | |
| --instr-profile="out/linux/x64/coverage/merged.profdata" \ | |
| "${OBJECT_ARGS[@]}" \ | |
| > "out/linux/x64/coverage/coverage.lcov" | |
| - name: Filter coverage.lcov for core directory | |
| run: | | |
| if [ -f "out/linux/x64/coverage/coverage.lcov" ]; then | |
| echo "Filtering coverage.lcov to include only core directory..." | |
| # Use lcov to extract only core directory coverage | |
| lcov --extract out/linux/x64/coverage/coverage.lcov \ | |
| "*/core/*" \ | |
| --output-file out/linux/x64/coverage/coverage_filtered.lcov \ | |
| --ignore-errors source | |
| echo "Filtered coverage file created" | |
| ls -lh out/linux/x64/coverage/coverage_filtered.lcov | |
| else | |
| echo "Warning: coverage.lcov file not found, cannot prepare filtered coverage" | |
| ls -la out/linux/x64/coverage/ || true | |
| exit 1 | |
| fi | |
| - name: Upload coverage report to Coveralls | |
| uses: coverallsapp/github-action@v2 | |
| env: | |
| COVERALLS_REPO_TOKEN: ${{ secrets.TEN_COVERALLS_REPO_TOKEN }} | |
| with: | |
| github-token: ${{ secrets.GITHUB_TOKEN }} | |
| file: out/linux/x64/coverage/coverage_filtered.lcov | |
| format: lcov | |
| debug: false #enable debug logging. | |
| measure: false #enable time measurement logging. | |
| - name: Upload filtered coverage report to artifacts | |
| if: success() || failure() | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: coverage-report-filtered | |
| path: out/linux/x64/coverage/coverage_filtered.lcov | |
| if-no-files-found: warn | |
| - name: Add coverage link to job summary | |
| if: always() | |
| run: | | |
| echo "## 📊 Coverage Report" >> "$GITHUB_STEP_SUMMARY" | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| echo "**Repository**: ${{ github.repository }}" >> "$GITHUB_STEP_SUMMARY" | |
| echo "**Branch/Tag**: ${{ github.ref_name }}" >> "$GITHUB_STEP_SUMMARY" | |
| echo "" >> "$GITHUB_STEP_SUMMARY" | |
| # Coverage report URL (branch overview) | |
| COVERALLS_URL="https://coveralls.io/github/${{ github.repository }}?branch=${{ github.ref_name }}" | |
| echo "**Coverage Report**: [${COVERALLS_URL}](${COVERALLS_URL})" >> "$GITHUB_STEP_SUMMARY" | |
| echo "**Please refresh the website after several minutes to see the correct coverage data, especially for the tree structure view**" >> "$GITHUB_STEP_SUMMARY" | |
| echo "**Sometimes when the tree structure view is displayed, but actually the coverage data is not fully loaded, please refresh again for correct coverage data.**" >> "$GITHUB_STEP_SUMMARY" |