From 21736cda2b1dd460d7b8349c011d74a22ae1f0a7 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Thu, 11 Apr 2024 20:02:32 +0000 Subject: [PATCH 01/10] move to workflow_dispatch and trigger it from branch.yaml --- .github/workflows/build.yaml | 19 +++++++++++++++ .github/workflows/pandas-tests.yml | 27 ++++++++++++++++++++++ .github/workflows/test.yaml | 11 --------- ci/cudf_pandas_scripts/pandas-tests/run.sh | 7 +++--- 4 files changed, 50 insertions(+), 14 deletions(-) create mode 100644 .github/workflows/pandas-tests.yml diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index 67c451fbd6e..c1cf876f4c7 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -108,3 +108,22 @@ jobs: sha: ${{ inputs.sha }} date: ${{ inputs.date }} package-name: dask_cudf + trigger-pandas-tests: + if: inputs.build_type == 'nightly' + needs: wheel-build-cudf + runs-on: ubuntu-latest + steps: + - name: Checkout code repo + uses: actions/checkout@v3 + with: + ref: ${{ inputs.sha }} + fetch-depth: 0 # unshallow fetch for setuptools-scm + persist-credentials: false + - name: Trigger pandas-tests + env: + GH_TOKEN: ${{ github.token }} + run: | + gh workflow run pandas-tests.yaml \ + -f branch=${{ inputs.branch }} \ + -f sha=${{ inputs.sha }} \ + -f date=${{ inputs.date }} diff --git a/.github/workflows/pandas-tests.yml b/.github/workflows/pandas-tests.yml new file mode 100644 index 00000000000..4cbd243e762 --- /dev/null +++ b/.github/workflows/pandas-tests.yml @@ -0,0 +1,27 @@ +name: Pandas Test Job + +on: + workflow_dispatch: + inputs: + branch: + required: true + type: string + date: + required: true + type: string + sha: + required: true + type: string + +jobs: + pandas-tests: + # run the Pandas unit tests + secrets: inherit + uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 + with: + matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(min_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) + build_type: nightly + branch: ${{ inputs.branch }} + date: ${{ inputs.date }} + sha: ${{ inputs.sha }} + script: ci/cudf_pandas_scripts/pandas-tests/run.sh main diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 65aef37697e..c5ae2f3b5a8 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -125,14 +125,3 @@ jobs: date: ${{ inputs.date }} sha: ${{ inputs.sha }} script: ci/cudf_pandas_scripts/run_tests.sh - pandas-tests: - # run the Pandas unit tests - secrets: inherit - uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 - with: - matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(min_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) - build_type: nightly - branch: ${{ inputs.branch }} - date: ${{ inputs.date }} - sha: ${{ inputs.sha }} - script: ci/cudf_pandas_scripts/pandas-tests/run.sh main diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index 1f70ca78c41..52e7e78c336 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -27,9 +27,10 @@ bash python/cudf/cudf/pandas/scripts/run-pandas-tests.sh \ --dist worksteal \ --report-log=${PANDAS_TESTS_BRANCH}.json 2>&1 +SUMMARY_FILE_NAME=${PANDAS_TESTS_BRANCH}-24.06-results.json # summarize the results and save them to artifacts: -python python/cudf/cudf/pandas/scripts/summarize-test-results.py --output json pandas-testing/${PANDAS_TESTS_BRANCH}.json > pandas-testing/${PANDAS_TESTS_BRANCH}-results.json +python python/cudf/cudf/pandas/scripts/summarize-test-results.py --output json pandas-testing/${PANDAS_TESTS_BRANCH}.json > pandas-testing/${SUMMARY_FILE_NAME} RAPIDS_ARTIFACTS_DIR=${RAPIDS_ARTIFACTS_DIR:-"${PWD}/artifacts"} mkdir -p "${RAPIDS_ARTIFACTS_DIR}" -mv pandas-testing/${PANDAS_TESTS_BRANCH}-results.json ${RAPIDS_ARTIFACTS_DIR}/ -rapids-upload-to-s3 ${RAPIDS_ARTIFACTS_DIR}/${PANDAS_TESTS_BRANCH}-results.json "${RAPIDS_ARTIFACTS_DIR}" +mv pandas-testing/${SUMMARY_FILE_NAME} ${RAPIDS_ARTIFACTS_DIR}/ +rapids-upload-to-s3 ${RAPIDS_ARTIFACTS_DIR}/${SUMMARY_FILE_NAME} "${RAPIDS_ARTIFACTS_DIR}" From 6858f99974beaf4e7be777e9633838005bbc0f20 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 11 Apr 2024 15:06:16 -0500 Subject: [PATCH 02/10] Update build.yaml --- .github/workflows/build.yaml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index c1cf876f4c7..d2b56c1227f 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -117,7 +117,6 @@ jobs: uses: actions/checkout@v3 with: ref: ${{ inputs.sha }} - fetch-depth: 0 # unshallow fetch for setuptools-scm persist-credentials: false - name: Trigger pandas-tests env: From 4a499f98e79d695fd7c6fcb687ab446325997900 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Thu, 11 Apr 2024 20:12:01 +0000 Subject: [PATCH 03/10] rename extension --- .github/workflows/{pandas-tests.yml => pandas-tests.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{pandas-tests.yml => pandas-tests.yaml} (100%) diff --git a/.github/workflows/pandas-tests.yml b/.github/workflows/pandas-tests.yaml similarity index 100% rename from .github/workflows/pandas-tests.yml rename to .github/workflows/pandas-tests.yaml From f844e68623dd0c9b2f6853a8cd7d68eaa50c4dad Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 11 Apr 2024 15:12:10 -0500 Subject: [PATCH 04/10] Update .github/workflows/build.yaml Co-authored-by: AJ Schmidt --- .github/workflows/build.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index d2b56c1227f..6942ef0009d 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -114,7 +114,7 @@ jobs: runs-on: ubuntu-latest steps: - name: Checkout code repo - uses: actions/checkout@v3 + uses: actions/checkout@v4 with: ref: ${{ inputs.sha }} persist-credentials: false From bce7bd4ea66a0546344882bc407d943569d74c8c Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Thu, 11 Apr 2024 20:28:30 +0000 Subject: [PATCH 05/10] address reviews --- .github/workflows/pandas-tests.yaml | 2 +- .github/workflows/pr.yaml | 2 +- ci/release/update-version.sh | 3 +++ 3 files changed, 5 insertions(+), 2 deletions(-) diff --git a/.github/workflows/pandas-tests.yaml b/.github/workflows/pandas-tests.yaml index 4cbd243e762..60544294809 100644 --- a/.github/workflows/pandas-tests.yaml +++ b/.github/workflows/pandas-tests.yaml @@ -19,7 +19,7 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: - matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(min_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.9" and .CUDA_VER == "12.2.2" )) build_type: nightly branch: ${{ inputs.branch }} date: ${{ inputs.date }} diff --git a/.github/workflows/pr.yaml b/.github/workflows/pr.yaml index 345ccbea45b..f84b1f42928 100644 --- a/.github/workflows/pr.yaml +++ b/.github/workflows/pr.yaml @@ -174,7 +174,7 @@ jobs: secrets: inherit uses: rapidsai/shared-workflows/.github/workflows/wheels-test.yaml@branch-24.06 with: - matrix_filter: map(select(.ARCH == "amd64")) | group_by(.CUDA_VER|split(".")|map(tonumber)|.[0]) | map(max_by([(.PY_VER|split(".")|map(tonumber)), (.CUDA_VER|split(".")|map(tonumber))])) + matrix_filter: map(select(.ARCH == "amd64" and .PY_VER == "3.9" and .CUDA_VER == "12.2.2" )) build_type: pull-request script: ci/cudf_pandas_scripts/pandas-tests/run.sh pr # Hide test failures because they exceed the GITHUB_STEP_SUMMARY output limit. diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 7cacdfd39c3..201e39ec47f 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -89,3 +89,6 @@ find .devcontainer/ -type f -name devcontainer.json -print0 | while IFS= read -r sed_runner "s@rapidsai/devcontainers:[0-9.]*@rapidsai/devcontainers:${NEXT_SHORT_TAG}@g" "${filename}" sed_runner "s@rapidsai/devcontainers/features/rapids-build-utils:[0-9.]*@rapidsai/devcontainers/features/rapids-build-utils:${NEXT_SHORT_TAG_PEP440}@" "${filename}" done + +# Update ci/cudf_pandas_scripts/pandas-tests/run.sh +sed -i "s/${CURRENT_SHORT_TAG}-results.json/${NEXT_SHORT_TAG}-results.json/g" ci/cudf_pandas_scripts/pandas-tests/run.sh From f13a3c72018d0f17198526a9881bdff50fc826fe Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 11 Apr 2024 15:35:48 -0500 Subject: [PATCH 06/10] Update ci/release/update-version.sh Co-authored-by: Ray Douglass <3107146+raydouglass@users.noreply.github.com> --- ci/release/update-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 201e39ec47f..31984c67c99 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -91,4 +91,4 @@ find .devcontainer/ -type f -name devcontainer.json -print0 | while IFS= read -r done # Update ci/cudf_pandas_scripts/pandas-tests/run.sh -sed -i "s/${CURRENT_SHORT_TAG}-results.json/${NEXT_SHORT_TAG}-results.json/g" ci/cudf_pandas_scripts/pandas-tests/run.sh +sed -i "s/SUMMARY_FILE_NAME=${PANDAS_TESTS_BRANCH}-.*-results.json/SUMMARY_FILE_NAME=${PANDAS_TESTS_BRANCH}-${NEXT_SHORT_TAG}-results.json/g" ci/cudf_pandas_scripts/pandas-tests/run.sh From 0470bf6a8840943d0c5a20a7a1269c55a51a252a Mon Sep 17 00:00:00 2001 From: Ray Douglass <3107146+raydouglass@users.noreply.github.com> Date: Thu, 11 Apr 2024 16:42:02 -0400 Subject: [PATCH 07/10] Escape quotes --- ci/release/update-version.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 31984c67c99..f694059bdec 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -91,4 +91,4 @@ find .devcontainer/ -type f -name devcontainer.json -print0 | while IFS= read -r done # Update ci/cudf_pandas_scripts/pandas-tests/run.sh -sed -i "s/SUMMARY_FILE_NAME=${PANDAS_TESTS_BRANCH}-.*-results.json/SUMMARY_FILE_NAME=${PANDAS_TESTS_BRANCH}-${NEXT_SHORT_TAG}-results.json/g" ci/cudf_pandas_scripts/pandas-tests/run.sh +sed -i "s/SUMMARY_FILE_NAME=\${PANDAS_TESTS_BRANCH}-.*-results.json/SUMMARY_FILE_NAME=\${PANDAS_TESTS_BRANCH}-${NEXT_SHORT_TAG}-results.json/g" ci/cudf_pandas_scripts/pandas-tests/run.sh From 4e543ecad589c9bdacbee9dc23c7c271336e1fdd Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Thu, 11 Apr 2024 20:43:52 +0000 Subject: [PATCH 08/10] don't hard-code version --- ci/cudf_pandas_scripts/pandas-tests/diff.sh | 4 +++- ci/cudf_pandas_scripts/pandas-tests/run.sh | 4 ++-- ci/release/update-version.sh | 3 --- 3 files changed, 5 insertions(+), 6 deletions(-) diff --git a/ci/cudf_pandas_scripts/pandas-tests/diff.sh b/ci/cudf_pandas_scripts/pandas-tests/diff.sh index cf80f383db4..6c9bc84ab0e 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/diff.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/diff.sh @@ -8,14 +8,16 @@ # Hard-coded needs to match the version deduced by rapids-upload-artifacts-dir GH_JOB_NAME="pandas-tests-diff / build" +RAPIDS_FULL_VERSION=$(./VERSION) rapids-logger "Github job name: ${GH_JOB_NAME}" +rapids-logger "Rapids version: ${RAPIDS_FULL_VERSION}" PY_VER="39" MAIN_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.main-results.json PR_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.pr-results.json rapids-logger "Fetching latest available results from nightly" -aws s3api list-objects-v2 --bucket rapids-downloads --prefix "nightly/" --query "sort_by(Contents[?ends_with(Key, '_py${PY_VER}.main-results.json')], &LastModified)[::-1].[Key]" --output text > s3_output.txt +aws s3api list-objects-v2 --bucket rapids-downloads --prefix "nightly/" --query "sort_by(Contents[?ends_with(Key, '_py${PY_VER}.main-${RAPIDS_FULL_VERSION}-results.json')], &LastModified)[::-1].[Key]" --output text > s3_output.txt read -r COMPARE_ENV < s3_output.txt export COMPARE_ENV diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index 52e7e78c336..713673eb6ac 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -6,8 +6,8 @@ set -euo pipefail PANDAS_TESTS_BRANCH=${1} - -rapids-logger "Running Pandas tests using $PANDAS_TESTS_BRANCH branch" +RAPIDS_FULL_VERSION=$(./VERSION) +rapids-logger "Running Pandas tests using $PANDAS_TESTS_BRANCH branch and rapids-version $RAPIDS_FULL_VERSION" rapids-logger "PR number: ${RAPIDS_REF_NAME:-"unknown"}" RAPIDS_PY_CUDA_SUFFIX="$(rapids-wheel-ctk-name-gen ${RAPIDS_CUDA_VERSION})" diff --git a/ci/release/update-version.sh b/ci/release/update-version.sh index 31984c67c99..7cacdfd39c3 100755 --- a/ci/release/update-version.sh +++ b/ci/release/update-version.sh @@ -89,6 +89,3 @@ find .devcontainer/ -type f -name devcontainer.json -print0 | while IFS= read -r sed_runner "s@rapidsai/devcontainers:[0-9.]*@rapidsai/devcontainers:${NEXT_SHORT_TAG}@g" "${filename}" sed_runner "s@rapidsai/devcontainers/features/rapids-build-utils:[0-9.]*@rapidsai/devcontainers/features/rapids-build-utils:${NEXT_SHORT_TAG_PEP440}@" "${filename}" done - -# Update ci/cudf_pandas_scripts/pandas-tests/run.sh -sed -i "s/SUMMARY_FILE_NAME=${PANDAS_TESTS_BRANCH}-.*-results.json/SUMMARY_FILE_NAME=${PANDAS_TESTS_BRANCH}-${NEXT_SHORT_TAG}-results.json/g" ci/cudf_pandas_scripts/pandas-tests/run.sh From 8bccd8856efcca51ae4bb4b86fef2d5abe127978 Mon Sep 17 00:00:00 2001 From: galipremsagar Date: Thu, 11 Apr 2024 20:45:26 +0000 Subject: [PATCH 09/10] fix --- ci/cudf_pandas_scripts/pandas-tests/diff.sh | 2 +- ci/cudf_pandas_scripts/pandas-tests/run.sh | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/cudf_pandas_scripts/pandas-tests/diff.sh b/ci/cudf_pandas_scripts/pandas-tests/diff.sh index 6c9bc84ab0e..b2a94f02770 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/diff.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/diff.sh @@ -8,7 +8,7 @@ # Hard-coded needs to match the version deduced by rapids-upload-artifacts-dir GH_JOB_NAME="pandas-tests-diff / build" -RAPIDS_FULL_VERSION=$(./VERSION) +RAPIDS_FULL_VERSION=$(<./VERSION) rapids-logger "Github job name: ${GH_JOB_NAME}" rapids-logger "Rapids version: ${RAPIDS_FULL_VERSION}" diff --git a/ci/cudf_pandas_scripts/pandas-tests/run.sh b/ci/cudf_pandas_scripts/pandas-tests/run.sh index 713673eb6ac..d13d31ad09f 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/run.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/run.sh @@ -6,7 +6,7 @@ set -euo pipefail PANDAS_TESTS_BRANCH=${1} -RAPIDS_FULL_VERSION=$(./VERSION) +RAPIDS_FULL_VERSION=$(<./VERSION) rapids-logger "Running Pandas tests using $PANDAS_TESTS_BRANCH branch and rapids-version $RAPIDS_FULL_VERSION" rapids-logger "PR number: ${RAPIDS_REF_NAME:-"unknown"}" From c8e834a89ff09edcd33bdf2865eaba3b98fdd4c1 Mon Sep 17 00:00:00 2001 From: GALI PREM SAGAR Date: Thu, 11 Apr 2024 17:00:56 -0500 Subject: [PATCH 10/10] Update diff.sh --- ci/cudf_pandas_scripts/pandas-tests/diff.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ci/cudf_pandas_scripts/pandas-tests/diff.sh b/ci/cudf_pandas_scripts/pandas-tests/diff.sh index b2a94f02770..f87a3a36fcc 100755 --- a/ci/cudf_pandas_scripts/pandas-tests/diff.sh +++ b/ci/cudf_pandas_scripts/pandas-tests/diff.sh @@ -13,8 +13,8 @@ rapids-logger "Github job name: ${GH_JOB_NAME}" rapids-logger "Rapids version: ${RAPIDS_FULL_VERSION}" PY_VER="39" -MAIN_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.main-results.json -PR_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.pr-results.json +MAIN_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.main-${RAPIDS_FULL_VERSION}-results.json +PR_ARTIFACT=$(rapids-s3-path)cuda12_$(arch)_py${PY_VER}.pr-${RAPIDS_FULL_VERSION}-results.json rapids-logger "Fetching latest available results from nightly" aws s3api list-objects-v2 --bucket rapids-downloads --prefix "nightly/" --query "sort_by(Contents[?ends_with(Key, '_py${PY_VER}.main-${RAPIDS_FULL_VERSION}-results.json')], &LastModified)[::-1].[Key]" --output text > s3_output.txt