From 8cc1316360c35042ac2f4397179cca83af892faf Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 15 Oct 2024 10:09:52 -0500 Subject: [PATCH 01/10] build wheels without build isolation --- .github/workflows/pr.yaml | 123 +------------------------------------- ci/build_wheel.sh | 36 ++++++++++- 2 files changed, 34 insertions(+), 125 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index fe8e730921..6e904fcede 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,144 +12,23 @@ concurrency: jobs: pr-builder: needs: - - changed-files - - checks - - conda-cpp-build - - conda-cpp-tests - - conda-cpp-checks - - conda-python-build - - conda-python-tests - - docs-build - wheel-build-pylibraft - - wheel-tests-pylibraft - wheel-build-raft-dask - - wheel-tests-raft-dask - - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12 if: always() with: needs: ${{ toJSON(needs) }} - changed-files: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-24.12 - with: - files_yaml: | - test_cpp: - - '**' - - '!.devcontainer/**' - - '!.pre-commit-config.yaml' - - '!CONTRIBUTING.md' - - '!README.md' - - '!docs/**' - - '!img/**' - - '!notebooks/**' - - '!python/**' - - '!thirdparty/LICENSES/**' - test_notebooks: - - '**' - - '!.devcontainer/**' - - '!.pre-commit-config.yaml' - - '!CONTRIBUTING.md' - - '!README.md' - - '!thirdparty/LICENSES/**' - test_python: - - '**' - - '!.devcontainer/**' - - '!.pre-commit-config.yaml' - - '!CONTRIBUTING.md' - - '!README.md' - - '!docs/**' - - '!img/**' - - '!notebooks/**' - - '!thirdparty/LICENSES/**' - checks: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.12 - with: - enable_check_generated_files: false - conda-cpp-build: - needs: checks - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 - with: - build_type: pull-request - node_type: cpu16 - conda-cpp-tests: - needs: [conda-cpp-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp - with: - build_type: pull-request - conda-cpp-checks: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 - with: - build_type: pull-request - enable_check_symbols: true - symbol_exclusions: raft_cutlass - conda-python-build: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 - with: - build_type: pull-request - conda-python-tests: - needs: [conda-python-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/build_docs.sh" wheel-build-pylibraft: - needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_pylibraft.sh - wheel-tests-pylibraft: - needs: [wheel-build-pylibraft, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_pylibraft.sh wheel-build-raft-dask: - needs: wheel-tests-pylibraft + needs: wheel-build-pylibraft secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: "ci/build_wheel_raft_dask.sh" - wheel-tests-raft-dask: - needs: [wheel-build-raft-dask, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_raft_dask.sh - devcontainer: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.12 - with: - arch: '["amd64"]' - cuda: '["12.5"]' - build_command: | - sccache -z; - build-all -DBUILD_PRIMS_BENCH=ON -DBUILD_ANN_BENCH=ON --verbose; - sccache -s; diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index e7ae52f33a..9d02127716 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -16,7 +16,28 @@ source rapids-date-string RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -rapids-generate-version > VERSION +rapids-logger "Generating build requirements" +matrix_selectors="cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" + +rapids-dependency-file-generator \ + --output requirements \ + --file-key "py_build_${underscore_package_name}" \ + --matrix "${matrix_selectors}" \ +| tee /tmp/requirements-build.txt + +rapids-dependency-file-generator \ + --output requirements \ + --file-key "py_rapids_build_${underscore_package_name}" \ + --matrix "${matrix_selectors}" \ +| tee -a /tmp/requirements-build.txt + +rapids-logger "Installing build requirements" +python -m pip install \ + -v \ + --prefer-binary \ + -r /tmp/requirements-build.txt + +rapids-generate-version > ./VERSION cd "${package_dir}" @@ -39,8 +60,17 @@ case "${RAPIDS_CUDA_VERSION}" in ;; esac -# Hardcode the output dir -python -m pip wheel . -w dist -vvv --no-deps --disable-pip-version-check +rapids-logger "Building '${package_name}' wheel" +python -m pip wheel \ + -w dist \ + -v \ + --no-build-isolation \ + --no-deps \ + --disable-pip-version-check \ + --extra-index-url https://pypi.nvidia.com \ + . + +sccache --show-adv-stats mkdir -p final_dist python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* From 4d356e3f326dd3ac75ec26ab4718c7df86e7dddd Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 15 Oct 2024 12:42:24 -0500 Subject: [PATCH 02/10] revert unrelated index change --- ci/build_wheel.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index 9d02127716..c187506877 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -67,7 +67,6 @@ python -m pip wheel \ --no-build-isolation \ --no-deps \ --disable-pip-version-check \ - --extra-index-url https://pypi.nvidia.com \ . sccache --show-adv-stats From 51cdd7d0e08ee7facb10432b7a0899312d678faf Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 15 Oct 2024 12:42:46 -0500 Subject: [PATCH 03/10] enable all CI --- .github/workflows/pr.yaml | 123 +++++++++++++++++++++++++++++++++++++- 1 file changed, 122 insertions(+), 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 6e904fcede..fe8e730921 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,23 +12,144 @@ concurrency: jobs: pr-builder: needs: + - changed-files + - checks + - conda-cpp-build + - conda-cpp-tests + - conda-cpp-checks + - conda-python-build + - conda-python-tests + - docs-build - wheel-build-pylibraft + - wheel-tests-pylibraft - wheel-build-raft-dask + - wheel-tests-raft-dask + - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12 if: always() with: needs: ${{ toJSON(needs) }} + changed-files: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-24.12 + with: + files_yaml: | + test_cpp: + - '**' + - '!.devcontainer/**' + - '!.pre-commit-config.yaml' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!notebooks/**' + - '!python/**' + - '!thirdparty/LICENSES/**' + test_notebooks: + - '**' + - '!.devcontainer/**' + - '!.pre-commit-config.yaml' + - '!CONTRIBUTING.md' + - '!README.md' + - '!thirdparty/LICENSES/**' + test_python: + - '**' + - '!.devcontainer/**' + - '!.pre-commit-config.yaml' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!notebooks/**' + - '!thirdparty/LICENSES/**' + checks: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.12 + with: + enable_check_generated_files: false + conda-cpp-build: + needs: checks + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 + with: + build_type: pull-request + node_type: cpu16 + conda-cpp-tests: + needs: [conda-cpp-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp + with: + build_type: pull-request + conda-cpp-checks: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 + with: + build_type: pull-request + enable_check_symbols: true + symbol_exclusions: raft_cutlass + conda-python-build: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 + with: + build_type: pull-request + conda-python-tests: + needs: [conda-python-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + docs-build: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/build_docs.sh" wheel-build-pylibraft: + needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_pylibraft.sh + wheel-tests-pylibraft: + needs: [wheel-build-pylibraft, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_pylibraft.sh wheel-build-raft-dask: - needs: wheel-build-pylibraft + needs: wheel-tests-pylibraft secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: "ci/build_wheel_raft_dask.sh" + wheel-tests-raft-dask: + needs: [wheel-build-raft-dask, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_raft_dask.sh + devcontainer: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.12 + with: + arch: '["amd64"]' + cuda: '["12.5"]' + build_command: | + sccache -z; + build-all -DBUILD_PRIMS_BENCH=ON -DBUILD_ANN_BENCH=ON --verbose; + sccache -s; From b667737bf8a4359abf618ca4a785f407ec1fa9b6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 23 Oct 2024 15:38:47 -0500 Subject: [PATCH 04/10] update to new conventions --- .pre-commit-config.yaml | 2 +- ci/build_cpp.sh | 4 ++++ ci/build_python.sh | 10 ++++++++++ ci/build_wheel.sh | 25 +++---------------------- ci/build_wheel_pylibraft.sh | 23 ++++++++++++++++++++++- 5 files changed, 40 insertions(+), 24 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 458d8b1b51..5a5342a74e 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -117,7 +117,7 @@ repos: cpp/cmake/modules/FindAVX[.]cmake - id: verify-alpha-spec - repo: https://github.com/rapidsai/dependency-file-generator - rev: v1.13.11 + rev: v1.16.0 hooks: - id: rapids-dependency-file-generator args: ["--clean"] diff --git a/ci/build_cpp.sh b/ci/build_cpp.sh index c456bcae80..92586c7c0a 100755 --- a/ci/build_cpp.sh +++ b/ci/build_cpp.sh @@ -15,6 +15,10 @@ rapids-print-env rapids-logger "Begin cpp build" +sccache --zero-stats + RAPIDS_PACKAGE_VERSION=$(rapids-generate-version) rapids-conda-retry mambabuild conda/recipes/libraft +sccache --show-adv-stats + rapids-upload-conda-to-s3 cpp diff --git a/ci/build_python.sh b/ci/build_python.sh index 80d37b5ae3..93dbd473c7 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -22,6 +22,8 @@ git_commit=$(git rev-parse HEAD) export RAPIDS_PACKAGE_VERSION=${version} echo "${version}" > VERSION +sccache --zero-stats + # TODO: Remove `--no-test` flags once importing on a CPU # node works correctly rapids-conda-retry mambabuild \ @@ -29,6 +31,9 @@ rapids-conda-retry mambabuild \ --channel "${CPP_CHANNEL}" \ conda/recipes/pylibraft +sccache --show-adv-stats +sccache --zero-stats + rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \ @@ -42,6 +47,9 @@ rapids-conda-retry mambabuild \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/raft-ann-bench +sccache --show-adv-stats +sccache --zero-stats + # Build ann-bench-cpu only in CUDA 11 jobs since it only depends on python # version RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" @@ -51,6 +59,8 @@ if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then --channel "${CPP_CHANNEL}" \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/raft-ann-bench-cpu + + sccache --show-adv-stats fi rapids-upload-conda-to-s3 python diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index c187506877..b82dd222f2 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -16,27 +16,6 @@ source rapids-date-string RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" -rapids-logger "Generating build requirements" -matrix_selectors="cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" - -rapids-dependency-file-generator \ - --output requirements \ - --file-key "py_build_${underscore_package_name}" \ - --matrix "${matrix_selectors}" \ -| tee /tmp/requirements-build.txt - -rapids-dependency-file-generator \ - --output requirements \ - --file-key "py_rapids_build_${underscore_package_name}" \ - --matrix "${matrix_selectors}" \ -| tee -a /tmp/requirements-build.txt - -rapids-logger "Installing build requirements" -python -m pip install \ - -v \ - --prefer-binary \ - -r /tmp/requirements-build.txt - rapids-generate-version > ./VERSION cd "${package_dir}" @@ -60,11 +39,13 @@ case "${RAPIDS_CUDA_VERSION}" in ;; esac +sccache --zero-stats + rapids-logger "Building '${package_name}' wheel" + python -m pip wheel \ -w dist \ -v \ - --no-build-isolation \ --no-deps \ --disable-pip-version-check \ . diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh index ce9f0ed172..26c36e8df9 100755 --- a/ci/build_wheel_pylibraft.sh +++ b/ci/build_wheel_pylibraft.sh @@ -3,6 +3,27 @@ set -euo pipefail +package_name="cuvs" + +rapids-logger "Generating build requirements" + +rapids-dependency-file-generator \ + --output requirements \ + --file-key "py_build_${package_name}" \ + --file-key "py_rapids_build_${package_name}" \ + --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ +| tee /tmp/requirements-build.txt + +rapids-logger "Installing build requirements" +python -m pip install \ + -v \ + --prefer-binary \ + -r /tmp/requirements-build.txt + +# build with '--no-build-isolation', for better sccache hit rate +# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) +export PIP_NO_BUILD_ISOLATION=0 + case "${RAPIDS_CUDA_VERSION}" in 12.*) EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" @@ -15,4 +36,4 @@ esac # Set up skbuild options. Enable sccache in skbuild config options export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF${EXTRA_CMAKE_ARGS}" -ci/build_wheel.sh pylibraft python/pylibraft +ci/build_wheel.sh "${package_name}" python/pylibraft From 059bfcea3c7831caff20e8217a2d7247c92a4f0d Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 23 Oct 2024 15:41:19 -0500 Subject: [PATCH 05/10] print sccache stats for raft-dask too --- .github/workflows/pr.yaml | 107 -------------------------------------- ci/build_python.sh | 3 ++ 2 files changed, 3 insertions(+), 107 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index fe8e730921..be98edb494 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,107 +12,27 @@ concurrency: jobs: pr-builder: needs: - - changed-files - - checks - conda-cpp-build - - conda-cpp-tests - - conda-cpp-checks - conda-python-build - - conda-python-tests - - docs-build - wheel-build-pylibraft - - wheel-tests-pylibraft - wheel-build-raft-dask - - wheel-tests-raft-dask - - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12 if: always() with: needs: ${{ toJSON(needs) }} - changed-files: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-24.12 - with: - files_yaml: | - test_cpp: - - '**' - - '!.devcontainer/**' - - '!.pre-commit-config.yaml' - - '!CONTRIBUTING.md' - - '!README.md' - - '!docs/**' - - '!img/**' - - '!notebooks/**' - - '!python/**' - - '!thirdparty/LICENSES/**' - test_notebooks: - - '**' - - '!.devcontainer/**' - - '!.pre-commit-config.yaml' - - '!CONTRIBUTING.md' - - '!README.md' - - '!thirdparty/LICENSES/**' - test_python: - - '**' - - '!.devcontainer/**' - - '!.pre-commit-config.yaml' - - '!CONTRIBUTING.md' - - '!README.md' - - '!docs/**' - - '!img/**' - - '!notebooks/**' - - '!thirdparty/LICENSES/**' - checks: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.12 - with: - enable_check_generated_files: false conda-cpp-build: - needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 with: build_type: pull-request node_type: cpu16 - conda-cpp-tests: - needs: [conda-cpp-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp - with: - build_type: pull-request - conda-cpp-checks: - needs: conda-cpp-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 - with: - build_type: pull-request - enable_check_symbols: true - symbol_exclusions: raft_cutlass conda-python-build: needs: conda-cpp-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 with: build_type: pull-request - conda-python-tests: - needs: [conda-python-build, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - docs-build: - needs: conda-python-build - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 - with: - build_type: pull-request - node_type: "gpu-v100-latest-1" - arch: "amd64" - container_image: "rapidsai/ci-conda:latest" - run_script: "ci/build_docs.sh" wheel-build-pylibraft: needs: checks secrets: inherit @@ -120,36 +40,9 @@ jobs: with: build_type: pull-request script: ci/build_wheel_pylibraft.sh - wheel-tests-pylibraft: - needs: [wheel-build-pylibraft, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_pylibraft.sh wheel-build-raft-dask: - needs: wheel-tests-pylibraft secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: "ci/build_wheel_raft_dask.sh" - wheel-tests-raft-dask: - needs: [wheel-build-raft-dask, changed-files] - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 - if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python - with: - build_type: pull-request - script: ci/test_wheel_raft_dask.sh - devcontainer: - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.12 - with: - arch: '["amd64"]' - cuda: '["12.5"]' - build_command: | - sccache -z; - build-all -DBUILD_PRIMS_BENCH=ON -DBUILD_ANN_BENCH=ON --verbose; - sccache -s; diff --git a/ci/build_python.sh b/ci/build_python.sh index 93dbd473c7..47c95ee969 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -40,6 +40,9 @@ rapids-conda-retry mambabuild \ --channel "${RAPIDS_CONDA_BLD_OUTPUT_DIR}" \ conda/recipes/raft-dask +sccache --show-adv-stats +sccache --zero-stats + # Build ann-bench for each cuda and python version rapids-conda-retry mambabuild \ --no-test \ From 8310b104d0e0e00b79b92f727b07761b7d305928 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 23 Oct 2024 15:42:30 -0500 Subject: [PATCH 06/10] rapids-upload-wheels-to-s3 --- ci/build_wheel.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel.sh b/ci/build_wheel.sh index b82dd222f2..326ee9a4c7 100755 --- a/ci/build_wheel.sh +++ b/ci/build_wheel.sh @@ -55,4 +55,4 @@ sccache --show-adv-stats mkdir -p final_dist python -m auditwheel repair -w final_dist "${EXCLUDE_ARGS[@]}" dist/* -RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 final_dist +RAPIDS_PY_WHEEL_NAME="${underscore_package_name}_${RAPIDS_PY_CUDA_SUFFIX}" rapids-upload-wheels-to-s3 python final_dist From 11696ce2b8069c7cba1a1438edc33dd6ed9367a6 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 23 Oct 2024 16:44:42 -0500 Subject: [PATCH 07/10] github actions --- .github/workflows/pr.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index be98edb494..d8f6e6c08d 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -34,7 +34,6 @@ jobs: with: build_type: pull-request wheel-build-pylibraft: - needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: From 5e468268b31d28025d4e3811237bd4cd13df87f1 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Wed, 23 Oct 2024 17:04:22 -0500 Subject: [PATCH 08/10] copy-paste error --- ci/build_wheel_pylibraft.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh index 26c36e8df9..3e4be0f97f 100755 --- a/ci/build_wheel_pylibraft.sh +++ b/ci/build_wheel_pylibraft.sh @@ -3,7 +3,7 @@ set -euo pipefail -package_name="cuvs" +package_name="pylibraft" rapids-logger "Generating build requirements" From 474d33d9eb48906264873843e821d1c2fd9ce530 Mon Sep 17 00:00:00 2001 From: James Lamb Date: Tue, 29 Oct 2024 09:32:25 -0500 Subject: [PATCH 09/10] revert no-build-isolation stuff --- .github/workflows/pr.yaml | 108 ++++++++++++++++++++++++++++++++++++ ci/build_wheel_pylibraft.sh | 23 +------- 2 files changed, 109 insertions(+), 22 deletions(-) diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index d8f6e6c08d..fe8e730921 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -12,36 +12,144 @@ concurrency: jobs: pr-builder: needs: + - changed-files + - checks - conda-cpp-build + - conda-cpp-tests + - conda-cpp-checks - conda-python-build + - conda-python-tests + - docs-build - wheel-build-pylibraft + - wheel-tests-pylibraft - wheel-build-raft-dask + - wheel-tests-raft-dask + - devcontainer secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/pr-builder.yaml@branch-24.12 if: always() with: needs: ${{ toJSON(needs) }} + changed-files: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/changed-files.yaml@branch-24.12 + with: + files_yaml: | + test_cpp: + - '**' + - '!.devcontainer/**' + - '!.pre-commit-config.yaml' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!notebooks/**' + - '!python/**' + - '!thirdparty/LICENSES/**' + test_notebooks: + - '**' + - '!.devcontainer/**' + - '!.pre-commit-config.yaml' + - '!CONTRIBUTING.md' + - '!README.md' + - '!thirdparty/LICENSES/**' + test_python: + - '**' + - '!.devcontainer/**' + - '!.pre-commit-config.yaml' + - '!CONTRIBUTING.md' + - '!README.md' + - '!docs/**' + - '!img/**' + - '!notebooks/**' + - '!thirdparty/LICENSES/**' + checks: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/checks.yaml@branch-24.12 + with: + enable_check_generated_files: false conda-cpp-build: + needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-build.yaml@branch-24.12 with: build_type: pull-request node_type: cpu16 + conda-cpp-tests: + needs: [conda-cpp-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-tests.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_cpp + with: + build_type: pull-request + conda-cpp-checks: + needs: conda-cpp-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-cpp-post-build-checks.yaml@branch-24.12 + with: + build_type: pull-request + enable_check_symbols: true + symbol_exclusions: raft_cutlass conda-python-build: needs: conda-cpp-build secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/conda-python-build.yaml@branch-24.12 with: build_type: pull-request + conda-python-tests: + needs: [conda-python-build, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/conda-python-tests.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + docs-build: + needs: conda-python-build + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/custom-job.yaml@branch-24.12 + with: + build_type: pull-request + node_type: "gpu-v100-latest-1" + arch: "amd64" + container_image: "rapidsai/ci-conda:latest" + run_script: "ci/build_docs.sh" wheel-build-pylibraft: + needs: checks secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: ci/build_wheel_pylibraft.sh + wheel-tests-pylibraft: + needs: [wheel-build-pylibraft, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_pylibraft.sh wheel-build-raft-dask: + needs: wheel-tests-pylibraft secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-build.yaml@branch-24.12 with: build_type: pull-request script: "ci/build_wheel_raft_dask.sh" + wheel-tests-raft-dask: + needs: [wheel-build-raft-dask, changed-files] + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.12 + if: fromJSON(needs.changed-files.outputs.changed_file_groups).test_python + with: + build_type: pull-request + script: ci/test_wheel_raft_dask.sh + devcontainer: + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/build-in-devcontainer.yaml@branch-24.12 + with: + arch: '["amd64"]' + cuda: '["12.5"]' + build_command: | + sccache -z; + build-all -DBUILD_PRIMS_BENCH=ON -DBUILD_ANN_BENCH=ON --verbose; + sccache -s; diff --git a/ci/build_wheel_pylibraft.sh b/ci/build_wheel_pylibraft.sh index 3e4be0f97f..ce9f0ed172 100755 --- a/ci/build_wheel_pylibraft.sh +++ b/ci/build_wheel_pylibraft.sh @@ -3,27 +3,6 @@ set -euo pipefail -package_name="pylibraft" - -rapids-logger "Generating build requirements" - -rapids-dependency-file-generator \ - --output requirements \ - --file-key "py_build_${package_name}" \ - --file-key "py_rapids_build_${package_name}" \ - --matrix "cuda=${RAPIDS_CUDA_VERSION%.*};arch=$(arch);py=${RAPIDS_PY_VERSION};cuda_suffixed=true" \ -| tee /tmp/requirements-build.txt - -rapids-logger "Installing build requirements" -python -m pip install \ - -v \ - --prefer-binary \ - -r /tmp/requirements-build.txt - -# build with '--no-build-isolation', for better sccache hit rate -# 0 really means "add --no-build-isolation" (ref: https://github.com/pypa/pip/issues/5735) -export PIP_NO_BUILD_ISOLATION=0 - case "${RAPIDS_CUDA_VERSION}" in 12.*) EXTRA_CMAKE_ARGS=";-DUSE_CUDA_MATH_WHEELS=ON" @@ -36,4 +15,4 @@ esac # Set up skbuild options. Enable sccache in skbuild config options export SKBUILD_CMAKE_ARGS="-DDETECT_CONDA_ENV=OFF;-DFIND_RAFT_CPP=OFF${EXTRA_CMAKE_ARGS}" -ci/build_wheel.sh "${package_name}" python/pylibraft +ci/build_wheel.sh pylibraft python/pylibraft From 406375602d2064f740006702f078c5baafdc88ed Mon Sep 17 00:00:00 2001 From: James Lamb Date: Thu, 31 Oct 2024 14:01:09 -0500 Subject: [PATCH 10/10] move zero-stats inside if statement --- ci/build_python.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ci/build_python.sh b/ci/build_python.sh index 47c95ee969..dc303de4f5 100755 --- a/ci/build_python.sh +++ b/ci/build_python.sh @@ -51,12 +51,13 @@ rapids-conda-retry mambabuild \ conda/recipes/raft-ann-bench sccache --show-adv-stats -sccache --zero-stats # Build ann-bench-cpu only in CUDA 11 jobs since it only depends on python # version RAPIDS_CUDA_MAJOR="${RAPIDS_CUDA_VERSION%%.*}" if [[ ${RAPIDS_CUDA_MAJOR} == "11" ]]; then + sccache --zero-stats + rapids-conda-retry mambabuild \ --no-test \ --channel "${CPP_CHANNEL}" \