diff --git a/.github/workflows/check_license.yml b/.github/workflows/check_license.yml index 338397dbd6cb..6664b50cf5db 100644 --- a/.github/workflows/check_license.yml +++ b/.github/workflows/check_license.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: license header check +name: License Header Check on: pull_request concurrency: @@ -21,8 +21,7 @@ concurrency: cancel-in-progress: true jobs: - license-check: - name: License Header Check + check: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/code_style.yml b/.github/workflows/code_format.yml similarity index 92% rename from .github/workflows/code_style.yml rename to .github/workflows/code_format.yml index c6ba7c647c17..ddabff6e9f8d 100644 --- a/.github/workflows/code_style.yml +++ b/.github/workflows/code_format.yml @@ -13,20 +13,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Code style checks +name: Code Format Check on: pull_request: paths: - - '.github/workflows/code_style.yml' + - '.github/workflows/code_format.yml' + - '.github/workflows/**/*.py' + - 'tools/**/*.py' + - 'dev/**/*.py' - 'cpp/**' + concurrency: group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} cancel-in-progress: true jobs: - Code-format-check: - name: Code Format Check(cpp) + format-check: runs-on: ubuntu-latest container: ubuntu:22.04 steps: @@ -40,7 +43,7 @@ jobs: - uses: actions/checkout@v4 with: fetch-depth: 0 - - name: Check CPP format + - name: Check run: | git config --global --add safe.directory $GITHUB_WORKSPACE cd $GITHUB_WORKSPACE/ diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index d2ddaec8f835..e18670d16e5e 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -13,12 +13,11 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: "Label pull requests" +name: Label Pull Requests on: pull_request_target jobs: label: - name: Label pull requests runs-on: ubuntu-latest permissions: contents: read diff --git a/.github/workflows/scala_code_format.yml b/.github/workflows/scala_code_format.yml new file mode 100644 index 000000000000..1d72be94fa01 --- /dev/null +++ b/.github/workflows/scala_code_format.yml @@ -0,0 +1,58 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +name: Code Format Check +on: + pull_request: + paths: + - '.github/workflows/scala_code_format.yml' + - 'backends-velox/**' + - 'gluten-uniffle/**' + - 'gluten-celeborn/**' + - 'gluten-ras/**' + - 'gluten-core/**' + - 'gluten-substrait/**' + - 'gluten-arrow/**' + - 'gluten-delta/**' + - 'gluten-iceberg/**' + - 'gluten-hudi/**' + - 'gluten-paimon/**' + - 'gluten-ut/**' + - 'shims/**' + - 'tools/gluten-it/**' + +concurrency: + group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }} + cancel-in-progress: true + +env: + SETUP: 'bash .github/workflows/util/setup_helper.sh' + +jobs: + + scala-format-check: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + - name: Check + run: | + ./dev/format-scala-code.sh check + - name: Fix + if: failure() + run: | + ./dev/format-scala-code.sh + git show diff --git a/.github/workflows/util/setup_helper.sh b/.github/workflows/util/setup_helper.sh index 86bb43c3d2f9..4679d1a8c174 100644 --- a/.github/workflows/util/setup_helper.sh +++ b/.github/workflows/util/setup_helper.sh @@ -26,7 +26,12 @@ function install_maven { wget -nv -O ${local_binary} ${url} tar -xvf ${local_binary} && mv apache-maven-${maven_version} /usr/lib/maven ) - echo "PATH=${PATH}:/usr/lib/maven/bin" >> $GITHUB_ENV + export PATH=/usr/lib/maven/bin:$PATH + if [ -n "$GITHUB_ENV" ]; then + echo "PATH=/usr/lib/maven/bin:$PATH" >> $GITHUB_ENV + else + echo "Warning: GITHUB_ENV is not set. Skipping environment variable export." + fi } for cmd in "$@" diff --git a/.github/workflows/velox_backend_arm.yml b/.github/workflows/velox_backend_arm.yml index 9ae827809760..5608097f9792 100644 --- a/.github/workflows/velox_backend_arm.yml +++ b/.github/workflows/velox_backend_arm.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Velox backend Github Runner (ARM) +name: Velox Backend (ARM) on: pull_request: diff --git a/.github/workflows/velox_backend_enhanced_features.yml b/.github/workflows/velox_backend_enhanced.yml similarity index 98% rename from .github/workflows/velox_backend_enhanced_features.yml rename to .github/workflows/velox_backend_enhanced.yml index 5a000db6ebfb..554d13aaddd7 100644 --- a/.github/workflows/velox_backend_enhanced_features.yml +++ b/.github/workflows/velox_backend_enhanced.yml @@ -13,12 +13,12 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Velox backend Github Runner (Enhanced Features) +name: Velox Backend (Enhanced) on: pull_request: paths: - - '.github/workflows/velox_backend_enhanced_features.yml' + - '.github/workflows/velox_backend_enhanced.yml' - 'pom.xml' - 'backends-velox/**' - 'gluten-uniffle/**' diff --git a/.github/workflows/velox_backend_x86.yml b/.github/workflows/velox_backend_x86.yml index 7e5ba406476b..18c5a5d90d08 100644 --- a/.github/workflows/velox_backend_x86.yml +++ b/.github/workflows/velox_backend_x86.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Velox backend Github Runner (x86) +name: Velox Backend (x86) on: pull_request: @@ -43,7 +43,7 @@ env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true MVN_CMD: 'mvn -ntp' WGET_CMD: 'wget -nv' - SETUP: 'bash .github/workflows/util/setup_helper.sh' + SETUP: 'source .github/workflows/util/setup_helper.sh' CCACHE_DIR: "${{ github.workspace }}/.ccache" # spark.sql.ansi.enabled defaults to false. SPARK_ANSI_SQL_MODE: false @@ -306,10 +306,7 @@ jobs: # Setup java and maven yum update -y && yum install -y java-1.8.0-openjdk-devel wget tzdata python3-pip - $SETUP install_maven - export PATH=${PATH}:/usr/lib/maven/bin - # Set environment variables export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk diff --git a/.github/workflows/velox_weekly.yml b/.github/workflows/velox_weekly.yml index 5091fb1905b5..68b8c7eb9b4f 100644 --- a/.github/workflows/velox_weekly.yml +++ b/.github/workflows/velox_weekly.yml @@ -13,7 +13,7 @@ # See the License for the specific language governing permissions and # limitations under the License. -name: Velox backend weekly job +name: Velox Backend Weekly Job on: pull_request: @@ -25,7 +25,7 @@ on: env: ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true MVN_CMD: 'mvn -ntp' - SETUP: 'bash .github/workflows/util/setup_helper.sh' + SETUP: 'source .github/workflows/util/setup_helper.sh' TPCH_TEST: "env GLUTEN_IT_JVM_ARGS=-Xmx5G sbin/gluten-it.sh queries-compare --local --preset=velox --benchmark-type=h --error-on-memleak --off-heap-size=10g -s=1.0 --threads=16 --iterations=1" INSTALL_PREFIX: /usr/local @@ -62,17 +62,11 @@ jobs: yum install -y java-1.8.0-openjdk-devel patch wget git perl python3-pip export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \ export PATH=$JAVA_HOME/bin:$PATH - wget --no-check-certificate https://downloads.apache.org/maven/maven-3/3.8.9/binaries/apache-maven-3.8.9-bin.tar.gz && \ - tar -xvf apache-maven*bin.tar.gz && \ - mv apache-maven-3.8.9 /usr/lib/maven && \ - export MAVEN_HOME=/usr/lib/maven && \ - export PATH=${PATH}:${MAVEN_HOME}/bin && \ - ln -s /usr/lib/maven/bin /usr/bin/mvn - ls -l /usr/bin/mvn + # The upstream main branch is cloned. Please temporarily switch to the dev branch for PR validation. git clone -b main --depth=1 https://github.com/apache/incubator-gluten.git && cd incubator-gluten/ + $SETUP install_maven ./dev/package.sh --spark_version=3.5 - build-on-centos: strategy: @@ -112,10 +106,7 @@ jobs: yum install -y java-1.8.0-openjdk-devel patch wget git perl export JAVA_HOME=/usr/lib/jvm/java-1.8.0-openjdk && \ export PATH=$JAVA_HOME/bin:$PATH - $SETUP install_maven - export MAVEN_HOME=/usr/lib/maven && \ - export PATH=${PATH}:${MAVEN_HOME}/bin cd $GITHUB_WORKSPACE/ && ./dev/package.sh --spark_version=3.5 diff --git a/dev/formatcppcode.sh b/dev/format-cpp-code.sh similarity index 100% rename from dev/formatcppcode.sh rename to dev/format-cpp-code.sh diff --git a/dev/format-scala-code.sh b/dev/format-scala-code.sh new file mode 100755 index 000000000000..9dc7f2f220b0 --- /dev/null +++ b/dev/format-scala-code.sh @@ -0,0 +1,34 @@ +#!/bin/bash + +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# If a new profile is introduced for new modules, please add it here to ensure +# the new modules are covered. +PROFILES="-Pbackends-velox -Pceleborn,uniffle -Piceberg,delta,hudi,paimon \ + -Pspark-3.2,spark-3.3,spark-3.4,spark-3.5,spark-4.0 -Pspark-ut" + +COMMAND=$1 + +if [[ "$COMMAND" == "check" ]]; then + echo "Checking Scala code style.." + mvn -q spotless:check $PROFILES +elif [[ "$COMMAND" == "apply" ]] || [[ "$COMMAND" == "" ]]; then + echo "Fixing Scala code style.." + mvn -q spotless:apply $PROFILES +else + echo "Unrecognized option." + exit 1 +fi