Skip to content

Commit

Permalink
[fix](shellcheck) fix hive-metastore and enable shellcheck in docker (#…
Browse files Browse the repository at this point in the history
…46496)

Problem Summary:
This pull request includes changes to the
`.github/workflows/code-checks.yml` and
`docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh`
files. The most important changes involve updating the shell checker
exclusions and modifying the script execution behavior for the Hive
metastore.
  • Loading branch information
suxiaogang223 authored Jan 7, 2025
1 parent c9f19a6 commit bcf08cd
Show file tree
Hide file tree
Showing 2 changed files with 17 additions and 17 deletions.
2 changes: 1 addition & 1 deletion .github/workflows/code-checks.yml
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,7 @@ jobs:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
with:
sh_checker_comment: true
sh_checker_exclude: .git .github ^docker ^thirdparty/src ^thirdparty/installed ^ui ^docs/node_modules ^tools/clickbench-tools ^extension ^output ^fs_brokers/apache_hdfs_broker/output (^|.*/)Dockerfile$ ^be/src/apache-orc ^be/src/clucene ^pytest ^samples
sh_checker_exclude: .git .github ^docker/compilation ^docker/runtime ^thirdparty/src ^thirdparty/installed ^ui ^docs/node_modules ^tools/clickbench-tools ^extension ^output ^fs_brokers/apache_hdfs_broker/output (^|.*/)Dockerfile$ ^be/src/apache-orc ^be/src/clucene ^pytest ^samples

preparation:
name: "Clang Tidy Preparation"
Expand Down
32 changes: 16 additions & 16 deletions docker/thirdparties/docker-compose/hive/scripts/hive-metastore.sh
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@
# specific language governing permissions and limitations
# under the License.

set -x
set -e -x

nohup /opt/hive/bin/hive --service metastore &

Expand All @@ -27,14 +27,14 @@ sleep 10s
# new cases should use separate dir
hadoop fs -mkdir -p /user/doris/suites/

lockfile1 = "mnt/scripts/run-data.lock"
lockfile1="/mnt/scripts/run-data.lock"

# wait lockfile
while [ -f "$lockfile1" ]; do
while [[ -f "${lockfile1}" ]]; do
sleep 10
done

touch "$lockfile1"
touch "${lockfile1}"

DATA_DIR="/mnt/scripts/data/"
find "${DATA_DIR}" -type f -name "run.sh" -print0 | xargs -0 -n 1 -P 10 -I {} sh -c '
Expand All @@ -45,16 +45,16 @@ find "${DATA_DIR}" -type f -name "run.sh" -print0 | xargs -0 -n 1 -P 10 -I {} sh
echo "Script: {} executed in $EXECUTION_TIME seconds"
'

rm -f "$lockfile1"
rm -f "${lockfile1}"

lockfile2 = "mnt/scripts/download-data.lock"
lockfile2="/mnt/scripts/download-data.lock"

# wait lockfile
while [ -f "$lockfile2" ]; do
while [[ -f "${lockfile2}" ]]; do
sleep 10
done

touch "$lockfile2"
touch "${lockfile2}"

# if you test in your local,better use # to annotation section about tpch1.db
if [[ ! -d "/mnt/scripts/tpch1.db" ]]; then
Expand Down Expand Up @@ -89,39 +89,39 @@ else
echo "/mnt/scripts/tvf_data exist, continue !"
fi

rm -f "$lockfile2"
rm -f "${lockfile2}"

# put data file
## put tpch1
if [ -z "$(ls /mnt/scripts/tpch1.db)" ]; then
if [[ -z "$(ls /mnt/scripts/tpch1.db)" ]]; then
echo "tpch1.db does not exist"
exit 1
fi
hadoop fs -mkdir -p /user/doris/
hadoop fs -put /mnt/scripts/tpch1.db /user/doris/
if [ -z "$(hadoop fs -ls /user/doris/tpch1.db)" ]; then
if [[ -z "$(hadoop fs -ls /user/doris/tpch1.db)" ]]; then
echo "tpch1.db put failed"
exit 1
fi

## put paimon1
if [ -z "$(ls /mnt/scripts/paimon1)" ]; then
if [[ -z "$(ls /mnt/scripts/paimon1)" ]]; then
echo "paimon1 does not exist"
exit 1
fi
hadoop fs -put /mnt/scripts/paimon1 /user/doris/
if [ -z "$(hadoop fs -ls /user/doris/paimon1)" ]; then
if [[ -z "$(hadoop fs -ls /user/doris/paimon1)" ]]; then
echo "paimon1 put failed"
exit 1
fi

## put tvf_data
if [ -z "$(ls /mnt/scripts/tvf_data)" ]; then
if [[ -z "$(ls /mnt/scripts/tvf_data)" ]]; then
echo "tvf_data does not exist"
exit 1
fi
hadoop fs -put /mnt/scripts/tvf_data /user/doris/
if [ -z "$(hadoop fs -ls /user/doris/tvf_data)" ]; then
if [[ -z "$(hadoop fs -ls /user/doris/tvf_data)" ]]; then
echo "tvf_data put failed"
exit 1
fi
Expand All @@ -143,7 +143,7 @@ START_TIME=$(date +%s)
hive -f /mnt/scripts/create_view_scripts/create_view.hql
END_TIME=$(date +%s)
EXECUTION_TIME=$((END_TIME - START_TIME))
echo "Script: create_view.hql executed in $EXECUTION_TIME seconds"
echo "Script: create_view.hql executed in ${EXECUTION_TIME} seconds"

touch /mnt/SUCCESS

Expand Down

0 comments on commit bcf08cd

Please sign in to comment.