diff --git a/.github/script/seekdb/collect_result.sh b/.github/script/seekdb/collect_result.sh new file mode 100644 index 000000000..7459768bb --- /dev/null +++ b/.github/script/seekdb/collect_result.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Collect result: read fail_cases.output and write seekdb_result.json. +# Usage: collect_result.sh +# Required env: GITHUB_WORKSPACE, GITHUB_RUN_ID, SEEKDB_TASK_DIR +set -e + +OUT_PATH="${1:-$GITHUB_WORKSPACE/seekdb_result.json}" +TASK_DIR="${SEEKDB_TASK_DIR:?}" +WORKSPACE="${GITHUB_WORKSPACE:-.}" + +FAIL_FILE="$TASK_DIR/fail_cases.output" + +failed_cases=() +if [[ -f "$FAIL_FILE" ]] && [[ -s "$FAIL_FILE" ]]; then + while IFS= read -r line; do + [[ -n "$line" ]] && failed_cases+=("$line") + done < "$FAIL_FILE" +fi + +success="true" +[[ ${#failed_cases[@]} -gt 0 ]] && success="false" + +# Build JSON (escape quotes in case names) +cs="" +for c in "${failed_cases[@]}"; do + esc="${c//\"/\\\"}" + [[ -n "$cs" ]] && cs="$cs," + cs="$cs\"$esc\"" +done +echo "{\"success\":$success,\"run_id\":\"${GITHUB_RUN_ID:-0}\",\"failed_cases\":[$cs]}" > "$OUT_PATH" + +echo "[collect_result.sh] Result written to $OUT_PATH (success=$success, failed=${#failed_cases[@]})."c \ No newline at end of file diff --git a/.github/script/seekdb/compile.sh b/.github/script/seekdb/compile.sh new file mode 100644 index 000000000..83592d305 --- /dev/null +++ b/.github/script/seekdb/compile.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Compile: 步骤与 .github/workflows/buildbase 一致(init → build.sh $TARGET → cd build_* && make)。 +# Required env: GITHUB_WORKSPACE, SEEKDB_TASK_DIR +# Optional: RELEASE_MODE, FORWARDING_HOST, MAKE, MAKE_ARGS +set -e + +WORKSPACE="${GITHUB_WORKSPACE:?}" +TASK_DIR="${SEEKDB_TASK_DIR:?}" + +# 调试:便于排查 k8s/container 下 No build.sh +echo "[compile.sh] WORKSPACE=$WORKSPACE" +echo "[compile.sh] pwd=$(pwd)" +ls -la "$WORKSPACE/" 2>/dev/null | head -20 || true +echo "[compile.sh] build.sh: -f=$([[ -f "$WORKSPACE/build.sh" ]] && echo 1 || echo 0) -x=$([[ -x "$WORKSPACE/build.sh" ]] && echo 1 || echo 0)" + +export GITHUB_WORKSPACE="$WORKSPACE" +export SEEKDB_TASK_DIR="$TASK_DIR" +export PACKAGE_TYPE="${RELEASE_MODE:+release}" +export PACKAGE_TYPE="${PACKAGE_TYPE:-debug}" +export MAKE="${MAKE:-make}" +export MAKE_ARGS="${MAKE_ARGS:--j32}" +export PATH="$WORKSPACE/deps/3rd/usr/local/oceanbase/devtools/bin:$PATH" +[[ -n "$FORWARDING_HOST" ]] && echo "$FORWARDING_HOST mirrors.oceanbase.com" >> /etc/hosts 2>/dev/null || true + +cd "$WORKSPACE" +mkdir -p "$TASK_DIR" + +BUILD_TARGET="${PACKAGE_TYPE:-debug}" +BUILD_DIR="build_${BUILD_TARGET}" +compile_ret=0 + +# 存在即可(不要求 -x),用 bash 执行 +if [[ ! -f "$WORKSPACE/build.sh" ]]; then + echo "[compile.sh] No build.sh at $WORKSPACE/build.sh, skip." +else + # Step 1: Build init(与 buildbase 一致,只传 init,先拉取/安装 deps 再才能用 cmake) + bash "$WORKSPACE/build.sh" init 2>&1 | tee "$TASK_DIR/compile_init.output" + [[ ${PIPESTATUS[0]} -ne 0 ]] && exit 1 + bash "$WORKSPACE/build.sh" "$BUILD_TARGET" -DOB_USE_CCACHE=ON -DNEED_PARSER_CACHE=OFF 2>&1 | tee "$TASK_DIR/compile_configure.output" + [[ ${PIPESTATUS[0]} -ne 0 ]] && exit 1 + set +e + command -v ccache >/dev/null 2>&1 && ccache -z || true + (cd "$WORKSPACE/$BUILD_DIR" && $MAKE $MAKE_ARGS observer) 2>&1 | tee "$TASK_DIR/compile.output" + compile_ret=${PIPESTATUS[0]} + command -v ccache >/dev/null 2>&1 && ccache -s || true + set -e +fi + +# 产物落到 build_*,打包 observer/obproxy 为 zst 并拷贝到任务目录 +for binary in observer obproxy; do + for base in . build_debug build_release build; do + if [[ -f "$WORKSPACE/$base/$binary" ]]; then + cp -f "$WORKSPACE/$base/$binary" "$WORKSPACE/$binary" 2>/dev/null || true + break + fi + done + if [[ -f "$WORKSPACE/$binary" ]]; then + command -v zstd >/dev/null 2>&1 && zstd -f "$WORKSPACE/$binary" || true + [[ -f "$WORKSPACE/$binary.zst" ]] && cp -f "$WORKSPACE/$binary.zst" "$TASK_DIR/" || true + fi +done + +exit "$compile_ret" diff --git a/.github/script/seekdb/mysqltest_slice.sh b/.github/script/seekdb/mysqltest_slice.sh new file mode 100644 index 000000000..d1ce0a849 --- /dev/null +++ b/.github/script/seekdb/mysqltest_slice.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Run one mysqltest slice (slice index from SLICE_IDX). +# Required env: GITHUB_WORKSPACE, SEEKDB_TASK_DIR, SLICE_IDX, SLICES, BRANCH +# Optional: FORWARDING_HOST +set -e + +WORKSPACE="${GITHUB_WORKSPACE:?}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPTS_DIR="$SCRIPT_DIR/scripts" + +export GITHUB_WORKSPACE="$WORKSPACE" +export SEEKDB_TASK_DIR="${SEEKDB_TASK_DIR:?}" +export SLICE_IDX="${SLICE_IDX:-0}" +export SLICES="${SLICES:-4}" +export BRANCH="${BRANCH:-master}" + + +# Copy compile artifacts from task dir to workspace if running in container +for f in observer.zst obproxy.zst; do + if [[ -f "$SEEKDB_TASK_DIR/$f" ]] && [[ ! -f "$WORKSPACE/$f" ]]; then + cp -f "$SEEKDB_TASK_DIR/$f" "$WORKSPACE/" || true + fi +done + +if [[ -f "$SCRIPTS_DIR/mysqltest_for_seekdb.sh" ]]; then + bash "$SCRIPTS_DIR/mysqltest_for_seekdb.sh" "$@" +else + echo "[mysqltest_slice.sh] No mysqltest_for_seekdb.sh, skip slice $SLICE_IDX." +fi \ No newline at end of file diff --git a/.github/script/seekdb/prepare.sh b/.github/script/seekdb/prepare.sh new file mode 100644 index 000000000..29955e0b6 --- /dev/null +++ b/.github/script/seekdb/prepare.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Prepare: create task dir and generate jobargs.output / run_jobs.output +# Required env: GITHUB_WORKSPACE, GITHUB_RUN_ID, MYSQLTEST_SLICES, SEEKDB_TASK_DIR +set -e + +WORKSPACE="${GITHUB_WORKSPACE:?}" +RUN_ID="${GITHUB_RUN_ID:?}" +SLICES="${MYSQLTEST_SLICES:-4}" +TASK_DIR="${SEEKDB_TASK_DIR:?}" + +mkdir -p "$TASK_DIR" + +# run_jobs.output: compile + N mysqltest slices (align with seekdb.groovy) +echo '++compile++' > "$TASK_DIR/run_jobs.output" +for i in $(seq 0 $((SLICES - 1))); do + echo "++mysqltest++${i}++" >> "$TASK_DIR/run_jobs.output" +done + +# jobargs.output: build options (align with seekdb.groovy) +{ + echo '++is_cmake++' + echo '++need_agentserver++0' + echo '++need_libobserver_so++0' + echo '++need_liboblog++0' +} > "$TASK_DIR/jobargs.output" + +echo "[prepare.sh] SEEKDB_TASK_DIR=$TASK_DIR run_jobs and jobargs written." \ No newline at end of file diff --git a/.github/script/seekdb/scripts/farm_compile.sh b/.github/script/seekdb/scripts/farm_compile.sh new file mode 100644 index 000000000..f5a085b37 --- /dev/null +++ b/.github/script/seekdb/scripts/farm_compile.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Farm compile: build observer / obproxy in workspace. +# Expects: WORKSPACE, SEEKDB_TASK_DIR, PACKAGE_TYPE (debug|release), REPO=server +set -e + +cd "$WORKSPACE" +# Use repo build.sh; debug or release from PACKAGE_TYPE +build_type="${PACKAGE_TYPE:-debug}" +if [[ -x "$WORKSPACE/build.sh" ]]; then + bash "$WORKSPACE/build.sh" "$build_type" --init --make +else + echo "[farm_compile.sh] No build.sh in $WORKSPACE, skip." +fi \ No newline at end of file diff --git a/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh b/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh new file mode 100644 index 000000000..fc58b6b8f --- /dev/null +++ b/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh @@ -0,0 +1,805 @@ +#!/bin/bash + +set -x + +# 脚本所在目录,末尾 source frame.sh 用 +SEEKDB_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# seekdb native:GITHUB_WORKSPACE=仓库根,SLICE_IDX=slice 序号(0,1,2,3) +# 脚本在 .github/script/seekdb/scripts/,仓库根为 4 级上级 +REPO_ROOT_BY_SCRIPT="$(cd "$SEEKDB_SCRIPT_DIR/../../../.." && pwd)" +export GITHUB_WORKSPACE="${GITHUB_WORKSPACE:-$REPO_ROOT_BY_SCRIPT}" +# 若传入的 GITHUB_WORKSPACE 在容器内无 build.sh(如 runner 传主机路径),改用实际存在的仓库根 +if [[ ! -f "$GITHUB_WORKSPACE/build.sh" ]]; then + if [[ -f "$REPO_ROOT_BY_SCRIPT/build.sh" ]]; then + export GITHUB_WORKSPACE="$REPO_ROOT_BY_SCRIPT" + elif [[ -f "$(pwd)/build.sh" ]]; then + export GITHUB_WORKSPACE="$(pwd)" + fi +fi +export USER="${USER:-$(whoami)}" +SEEKDB_HOME="${GITHUB_WORKSPACE}/seekdb_home_${SLICE_IDX:-0}" +mkdir -p "$SEEKDB_HOME" +ln -sf "$GITHUB_WORKSPACE" "$SEEKDB_HOME/seekdb" +export HOME="$SEEKDB_HOME" +export SLOT_ID="${SLICE_IDX:-0}" +# 保证 HOME/observer 存在:优先用 WORKSPACE 下 observer,否则解压 observer.zst +if [[ -x "$GITHUB_WORKSPACE/observer" ]]; then + cp -f "$GITHUB_WORKSPACE/observer" "$HOME/observer" && chmod +x "$HOME/observer" +elif [[ -f "$GITHUB_WORKSPACE/observer.zst" ]]; then + (command -v zstd >/dev/null 2>&1 && zstd -d -f -o "$HOME/observer" "$GITHUB_WORKSPACE/observer.zst" && chmod +x "$HOME/observer") || true +fi + +export PATH=/bin:/usr/bin +HOST=$(hostname -i) +DOWNLOAD_DIR=$HOME/seekdb/tools/deploy + +function prepare_config { + if [[ "$MINI" == "1" ]] || ([[ "$MINI" == "-1" ]] && [[ -f $HOME/seekdb/tools/deploy/enable_mini_mode ]]) + then + [[ -f $HOME/seekdb/tools/deploy/enable_mini_mode ]] && MINI_SIZE=$(cat $HOME/seekdb/tools/deploy/enable_mini_mode) + [[ $MINI_SIZE =~ ^[0-9]+G ]] || MINI_SIZE="8G" + MINI_CONFIG_ITEM="ObCfg.init_config['memory_limit']='$MINI_SIZE'" + fi + + + cd $HOME/seekdb/tools/deploy + if [ "$WITH_PROXY" ] && [ "$WITH_PROXY" != "0" ] + then + if [ "$CLUSTER_SPEC" == '2x1' ] + then + SPEC="[$HOST,proxy@$HOST]@zone1 [$HOST]@zone2" + else + SPEC="[$HOST,proxy@$HOST]@zone1" + fi + else + if [[ "$CLUSTER_SPEC" == '2x1' ]] || [[ "$SLAVE" == "1" ]] + then + SPEC="[$HOST]@zone1 [$HOST]@zone2" + else + SPEC="[$HOST]@zone1" + fi + fi + # farm中由于机器变小了,需要设置下 + CPU_COUNT_CONFIG="ObCfg.init_config['cpu_count']='24'" + cat >config7.py < test.fifo 2>&1 + elif [[ "$SLAVE" == "1" ]] + then + cat test.fifo | tee result.out & ./hap.py $USER.obs1.mysqltest collect_all slices=$SLICES slice_idx=$SLICE_IDX $slb `echo $ARGV` 1> test.fifo 2>&1 + else + cat test.fifo | tee result.out & ./hap.py $USER.mysqltest collect_all slices=$SLICES slice_idx=$SLICE_IDX $slb `echo $ARGV` 1> test.fifo 2>&1 + fi + + # check if there is error + if [ "$?" == "0" ] + then + grep -E 'PASSED' $HOME/seekdb/tools/deploy/result.out > /dev/null && ! grep -E "FAIL LST" $HOME/seekdb/tools/deploy/result.out > /dev/null || return 1 + else + return 1 + fi +} + +function collect_log { + cd $HOME/seekdb/tools/deploy || return 1 + if [ -d "collected_log" ] + then + mv mysql_test/var/log/* collected_log + mv $HOME/$USER.*/core[.-]* collected_log + mv collected_log collected_log_$SLICE_IDX || return 1 + tar cvfz $HOME/collected_log_$SLICE_IDX.tar.gz collected_log_$SLICE_IDX + fi +} + +export -f run_mysqltest + +function obd_prepare_obd { + if [[ -f $HOME/seekdb/deps/init/dep_create.sh ]] + then + if [[ "$IS_CE" == "1" ]] + then + cd $HOME/seekdb && ./build.sh init --ce || return 3 + else + cd $HOME/seekdb && ./build.sh init || return 3 + fi + else + if grep 'dep_create.sh' $HOME/seekdb/build.sh + then + cd $HOME/seekdb/deps/3rd && bash dep_create.sh all || return 4 + else + cd $HOME/seekdb && ./build.sh init || return 3 + fi + fi + + $obd devmode enable + $obd env set OBD_DEPLOY_BASE_DIR $HOME/seekdb/tools/deploy + $obd --version + if [[ "$IS_CE" == "1" && ! -f $OBD_HOME/.obd/mirror/remote/taobao.repo ]] + then + mkdir -p $OBD_HOME/.obd/mirror/remote && echo -e "[taobao]\nname=taobao\nbaseurl=http://yum.tbsite.net/taobao/7/x86_64/current\nenabled=1\n\n[taobao-test]\nname=taobao-test\nbaseurl=http://yum.tbsite.net/taobao/7/x86_64/test\nenabled=0" > $OBD_HOME/.obd/mirror/remote/taobao.repo + fi +} + +export -f obd_prepare_obd + +function obd_prepare_global { + export LANG=en_US.UTF-8 + HOST=$(hostname -i) + DOWNLOAD_DIR=$HOME/downloads + export SLOT_ID="${SLOT_ID:-0}" + PORT_NUM=$((5000 + SLOT_ID * 100)) + + # SeekDB CI:固定按 CE 路径走 OBD/init(原先用 observer -V 判 CE,SeekDB 输出不含 CE 字样会被判成 0) + COMPONENT="seekdb" + # 根据build.sh中的内容判断依赖安装路径 + if grep 'dep_create.sh' $HOME/seekdb/build.sh + then + DEP_PATH=$HOME/seekdb/deps/3rd + else + DEP_PATH=$HOME/seekdb/rpm/.dep_create/var + fi + + if [[ -f "$HOME/seekdb/tools/deploy/mysqltest_config.yaml" ]] + then + export WITH_MYSQLTEST_CONFIG_YAML=1 + else + export WITH_MYSQLTEST_CONFIG_YAML=0 + fi + + ob_name=obcluster$SLOT_ID + app_name=$ob_name.$USER.$HOST + ocp_config_server='http://ocp-cfg.alibaba.net:8080/services?User_ID=alibaba&UID=test' + cfg_url=${ocp_config_server} + + export obd=$DEP_PATH/usr/bin/obd + export OBD_HOME=$HOME + export OBD_INSTALL_PRE=$DEP_PATH + export DATA_PATH=$HOME/data + [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh + +} + +function obd_prepare_config { + + MINI_SIZE="10G" + if [[ "$MINI" == "1" ]] || ([[ "$MINI" == "-1" ]] && [[ -f $HOME/seekdb/tools/deploy/enable_mini_mode ]]) + then + [[ -f $HOME/seekdb/tools/deploy/enable_mini_mode ]] && MINI_SIZE=$(cat $HOME/seekdb/tools/deploy/enable_mini_mode) + [[ $MINI_SIZE =~ ^[0-9]+G ]] || MINI_SIZE="8G" + fi + if [[ "$CLUSTER_SPEC" == '2x1' ]] || [[ "$SLAVE" == "1" ]] + then + mysql_port=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1` + rpc_port1=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1` + mysql_port2=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1` + rpc_port2=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1` + SERVERS=$(cat <<-EOF + servers: + - name: server1 + ip: 127.0.0.1 + - name: server2 + ip: 127.0.0.1 + server1: + mysql_port: $mysql_port + rpc_port: $rpc_port1 + home_path: $DATA_PATH/observer1 + zone: zone1 + server2: + mysql_port: $mysql_port2 + rpc_port: $rpc_port2 + home_path: $DATA_PATH/observer2 + zone: zone2 +EOF +) + else + mysql_port=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1` + rpc_port=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1` + SERVERS=$(cat <<-EOF + servers: + - name: server1 + ip: 127.0.0.1 + server1: + mysql_port: $mysql_port + rpc_port: $rpc_port + home_path: $DATA_PATH/observer1 + zone: zone1 +EOF +) + fi + export MYSQL_PORT=$mysql_port + + if [[ "$MINI" == "1" && -f $HOME/seekdb/tools/deploy/obd/core-test.yaml.template ]] + then + conf=$(cat $HOME/seekdb/tools/deploy/obd/core-test.yaml.template) + else + conf=$(cat $HOME/seekdb/tools/deploy/obd/config.yaml.template) + fi + # cgroup support + if [[ -f $HOME/seekdb/tools/deploy/obd/.use_cgroup || $(echo " $ARGV " | grep " use-cgroup ") ]] + then + CGROUP_CONFIG="cgroup_dir: /sys/fs/cgroup/cpu/$USER" + fi + extra_conf="""$MYSQLRTEST_ARGS + $CGROUP_CONFIG +""" + conf=${conf//'{{%% PROXY_CONF %%}}'/"$extra_conf"} + conf=${conf//'{{%% DEPLOY_PATH %%}}'/"$HOME/seekdb/tools/deploy"} + conf=${conf//'{{%% COMPONENT %%}}'/$COMPONENT} + conf=${conf//'{{%% SERVERS %%}}'/$SERVERS} + conf=${conf//'{{%% TAG %%}}'/'latest'} + conf=${conf//'{{%% MINI_SIZE %%}}'/$MINI_SIZE} + conf=${conf//'{{%% OBCONFIG_URL %%}}'/$cfg_url} + conf=${conf//'{{%% APPNAME %%}}'/$app_name} + conf=${conf//'{{%% EXTRA_PARAM %%}}'/$MYSQLRTEST_ARGS} + if [[ "$IS_CE" == "1" ]] + then + conf=$(echo "$conf" | sed 's/oceanbase:/oceanbase\-ce:/g' | sed 's/\- oceanbase$/- oceanbase-ce/g') + fi + echo "$conf" > $HOME/seekdb/tools/deploy/config.yaml + + if [[ $WITH_SHARED_STORAGE == "1" ]] + then + # shared_storage 需要在global修改相关的mode + share_storage_workdir="$(date +%s)-$(uuidgen)" + first_global_line_count=`sed -n '/global:/=' $HOME/seekdb/tools/deploy/config.yaml | head -n 1` + set +x + sed -i "${first_global_line_count}a\ mode: 'shared_storage'\n\ + shared_storage: \"oss://oss-436751-0701-all-test/$FARM2_RUN_USER/$share_storage_workdir?host=oss-cn-hangzhou.aliyuncs.com&access_id=${SENSITIVE_TEST_OSS_ID_FOR_OBJECT_STORAGE_FOR_TOTAL}&access_key=${SENSITIVE_TEST_OSS_KEY_FOR_OBJECT_STORAGE_FOR_TOTAL}&max_iops=0&max_bandwidth=0B&scope=region\"" $HOME/seekdb/tools/deploy/config.yaml + sed -i "s/datafile_size: '20G'/datafile_size: '200G'/g" $HOME/seekdb/tools/deploy/config.yaml + set -x + fi + + if [[ $SHARED_STORAGE_MODE == "1" ]] + then + # shared_storage 需要在global修改相关的mode + MINIO_IP="100.88.99.213" + if [[ $TEST_NAME == "farm2" ]] + then + MINIO_IP="100.88.99.213" + fi + share_storage_workdir="$(date +%s)-$(uuidgen)" + first_global_line_count=`sed -n '/global:/=' $HOME/seekdb/tools/deploy/config.yaml | head -n 1` + sed -i "${first_global_line_count}a\ mode: 'shared_storage'\n\ + shared_storage: \"s3://farm-test/mysqltest/$share_storage_workdir/clog_and_data?host=http://$MINIO_IP:9000&access_id=minioadmin&access_key=minioadmin&s3_region=us-east-1&max_iops=10000&max_bandwidth=1GB&scope=region\"" $HOME/seekdb/tools/deploy/config.yaml + fi + + if [[ ! $WITH_SHARED_STORAGE == "1" ]] + then + cat $HOME/seekdb/tools/deploy/config.yaml + fi +} + +function obd_prepare_bin { + cd $HOME + mkdir -p $DOWNLOAD_DIR/{bin,etc,admin} || return 1 + + if [ ! -x observer ] + then + ( + cd $DOWNLOAD_DIR/bin && + wget http://6.12.233.4:5000/observer -O observer && + chmod +x observer + ) || return 1 + else + cp observer $DOWNLOAD_DIR/bin/ || return 2 + fi + + if [[ "$COMPONENT" == "seekdb" ]] && [[ -x $DOWNLOAD_DIR/bin/observer ]] + then + cp $DOWNLOAD_DIR/bin/observer $DOWNLOAD_DIR/bin/seekdb && chmod +x $DOWNLOAD_DIR/bin/seekdb + fi + + + cd $HOME/seekdb/tools/deploy + [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh + [[ -x $HOME/seekdb/tools/deploy/obd.sh ]] && ./obd.sh prepare + cp $DEP_PATH/u01/obclient/bin/obclient ./obclient && chmod 777 obclient + cp $DEP_PATH/u01/obclient/bin/mysqltest ./mysqltest && chmod 777 mysqltest + + if [ "$WITH_DEPS" ] && [ "$WITH_DEPS" != "0" ] + then + cd $HOME/seekdb/tools/deploy && [[ -f copy.sh ]] && sh copy.sh + fi + if [[ -f "$HOME/seekdb/tools/deploy/obd/.observer_obd_plugin_version" ]] + then + obs_version=$(cat $HOME/seekdb/tools/deploy/obd/.observer_obd_plugin_version) + else + $DOWNLOAD_DIR/bin/observer -V + obs_version=`$DOWNLOAD_DIR/bin/observer -V 2>&1 | grep -E "(observer|seekdb) \(OceanBase([ \_]CE|[ \_]Lite|[ \_]SeekDB|[ \_]seekdb)? [0-9]" | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?' | head -1` + fi + + mkdir -p $HOME/seekdb/tools/deploy/admin + if [[ -d $HOME/seekdb/src/share/inner_table/sys_package ]] + then + cp $HOME/seekdb/src/share/inner_table/sys_package/*.sql $HOME/seekdb/tools/deploy/admin/ + fi + + $obd mirror create -n $COMPONENT -p $DOWNLOAD_DIR -t latest -V $obs_version || return 2 + +} + +function obd_init_cluster { + retries=$reboot_retries + while (( $retries > 0 )) + do + if [[ "$retries" == "$reboot_retries" ]] + then + [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh + $obd cluster deploy $ob_name -c $HOME/seekdb/tools/deploy/config.yaml -f && $obd cluster start $ob_name -f && $obd cluster display $ob_name + else + [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh + $obd cluster redeploy $ob_name -f + fi + retries=`expr $retries - 1` + ./obclient -h 127.1 -P $MYSQL_PORT -u root -A -e "alter system set_tp tp_no = 509, error_code = 4016, frequency = 1;" + ret=$? + if [[ $ret == 0 ]] + then + init_files=('init.sql' 'init_mini.sql' 'init_for_ce.sql') + for init_file in ${init_files[*]} + do + if [[ -f $HOME/seekdb/tools/deploy/$init_file ]] + then + if ! grep "alter system set_tp tp_no = 509, error_code = 4016, frequency = 1;" $HOME/seekdb/tools/deploy/$init_file + then + echo -e "\nalter system set_tp tp_no = 509, error_code = 4016, frequency = 1;" >> $HOME/seekdb/tools/deploy/$init_file + fi + fi + done + fi + $obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient --init-only --init-sql-dir=$HOME/seekdb/tools/deploy $INIT_FLIES --log-dir=$HOME/init_case $EXTRA_ARGS -v > ./init_sql_log && init_seccess=1 && break + done +} + +function is_case_selector_arg() +{ + local arg + arg=$1 + case_selector_patterns=('test-set=*' 'testset=*' 'suite=*' 'tag=*' 'regress_suite=*' 'all' 'psmall') + for p in ${case_selector_patterns[@]} + do + [[ "$(echo $arg | grep -Eo $p)" != "" ]] && return 0 + done + return 1 +} + +function obd_run_mysqltest { + set -x + obd_prepare_global + + cd $HOME/seekdb/tools/deploy + [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh + if [[ "$SLB" != "" ]] && [[ "$EXECID" != "" ]] + then + SCHE_ARGS="--slb-host=$SLB --exec-id=$EXECID " + else + SCHE_ARGS="--slices=$SLICES --slice-idx=$SLICE_IDX " + fi + + + if [[ "$MINI" == "1" && -f core-test.init_mini.sql ]] + then + INIT_FLIES="--init-sql-files=core-test.init_mini.sql,init_user.sql|root@mysql|test" + [ -f init_user_oracle.sql ] && INIT_FLIES="${INIT_FLIES},init_user_oracle.sql|SYS@oracle|SYS" + else + if [[ "$IS_CE" == "1" && -f $HOME/seekdb/.ce ]] + then + INIT_FLIES="--init-sql-files=init_mini.sql,init_user.sql|root@mysql|test " + fi + fi + CLUSTER_MODE="c" + if [[ "$SLAVE" == "1" ]] + then + SERVER_ARGS="--test-server=server2" + CLUSTER_MODE="slave" + else + SERVER_ARGS="" + fi + if [ "$WITH_PROXY" ] && [ "$WITH_PROXY" != "0" ] + then + CLUSTER_MODE="proxy" + fi + reboot_retries=2 + if [[ "$WITH_MYSQLTEST_CONFIG_YAML" == "1" ]] + then + EXTRA_ARGS="--test-mode=$CLUSTER_MODE " + else + EXTRA_ARGS="--cluster-mode=$CLUSTER_MODE " + fi + if [[ -f $HOME/seekdb/tools/deploy/obd/.fast-reboot ]] + then + EXTRA_ARGS="${EXTRA_ARGS}--fast-reboot " + fi + + if [[ "$IS_CE" == "1" ]] + then + EXTRA_ARGS="${EXTRA_ARGS}--mode=mysql " + fi + + EXTRA_ARGS_WITHOUT_CASE=$EXTRA_ARGS + + if [[ "$ARGV" != "" ]] + then + for arg in $ARGV + do + [[ "${arg%%=*}" == "testset" ]] && arg="${arg/testset=/test-set=}" + [[ "${arg%%=*}" == "reboot-timeout" ]] && has_reboot_timeout=1 + [[ "${arg%%=*}" == "reboot-retries" ]] && reboot_retries=${arg#*=} + [[ "${arg%%=*}" == "disable-collect" ]] && disable_collect=1 && continue + [[ "${arg%%=*}" == "use-cgroup" ]] && continue + EXTRA_ARGS="${EXTRA_ARGS}--${arg} " + if ! is_case_selector_arg "${arg}" + then + EXTRA_ARGS_WITHOUT_CASE="${EXTRA_ARGS_WITHOUT_CASE}--${arg} " + fi + done + fi + + if [[ "$has_reboot_timeout" != "1" ]] + then + REBOOT_TIMEOUT="--reboot-timeout=600" + fi + if [[ "$disable_collect" != "1" ]] + then + COLLECT_ARG="--collect-all" + if [[ "$WITH_MYSQLTEST_CONFIG_YAML" == "0" ]] && [ "$WITH_PROXY" ] && [ "$WITH_PROXY" != "0" ] && [[ -f "$HOME/seekdb/tools/deploy/obd/.collect_proxy_log" ]] + then + COLLECT_ARG="$COLLECT_ARG --collect-components=$COMPONENT,obproxy" + fi + fi + obd_init_cluster + if [[ "$init_seccess" != "1" ]] + then + cat ./init_sql_log + echo "Failed to init sql, see more log at ./collected_log/_test_init" + mkdir -p $HOME/collected_log/_test_init/ + [[ -d "$DATA_PATH/observer1/log" ]] && mkdir -p $HOME/collected_log/_test_init/observer1 && mv $DATA_PATH/observer1/log/* $HOME/collected_log/_test_init/observer1 + [[ -d "$DATA_PATH/observer2/log" ]] && mkdir -p $HOME/collected_log/_test_init/observer2 && mv $DATA_PATH/observer2/log/* $HOME/collected_log/_test_init/observer2 + [[ -d "$DATA_PATH/obproxy/log" ]] && mkdir -p $HOME/collected_log/_test_init/obproxy && mv $DATA_PATH/obproxy/log/* $HOME/collected_log/_test_init/obproxy + exit 1 + fi + if [[ "$SPECIAL_RUN" == "1" ]] + then + ret=255 + result_dir=$HOME/record + mkdir -p $result_dir + RECORD_ARGS="--record-dir=$result_dir --record --log-dir=./var/rec_log" + RESULT_ARGS="--result-dir=$result_dir --result-file-suffix=.record --log-dir=./var/log" + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient --init-sql-dir=$HOME/seekdb/tools/deploy --special-run --sort-result $COLLECT_ARG $REBOOT_TIMEOUT $VERBOSE_ARG" + # $mysqltest_cmd --all $INIT_FLIES $SCHE_ARGS $RECORD_ARGS 2>&1 > record.out + $mysqltest_cmd $INIT_FLIES $SCHE_ARGS $RECORD_ARGS $EXTRA_ARGS 2>&1 | tee record.out && ( exit ${PIPESTATUS[0]}) + rec_ret=$? + cat record.out | grep -E '\|\s+(PASSED|FAILED)|s+\|' > record_result + TEST_CASES=`cat record_result | grep PASSED | awk '{print \$2}' | tr '\n' ',' | head -c -1` + ret=0 + if [[ "$TEST_CASES" != "" ]] + then + if [[ "$SP_CHANGE" != "" ]] + then + arr=(${SP_CHANGE//,/ }) + for line in ${arr[@]} + do + MYSQLRTEST_ARGS=$MYSQLRTEST_ARGS$'\n'" ${line%%=*}: ${line##*=}" + done + obd_prepare_config + echo "config after change:" + cat $HOME/seekdb/tools/deploy/config.yaml + $obd cluster destroy $ob_name && $obd cluster deploy $ob_name -c $HOME/seekdb/tools/deploy/config.yaml && $obd cluster start $ob_name -f && $obd cluster display $ob_name + else + $obd cluster redeploy $ob_name + fi + obd_init_cluster + $mysqltest_cmd $INIT_FLIES --test-set=$TEST_CASES --sp-hint="$SP_HINT" $RESULT_ARGS $EXTRA_ARGS_WITHOUT_CASE 2>&1 | tee compare.out && ( exit ${PIPESTATUS[0]}) + ret=$? + else + echo 'No case succeed?!' + fi + if [[ $JOBNAME == 'mysqltest_opensource' ]] + then + submarker="_opensource" + else + submarker="" + fi + mv record.out $HOME/mysqltest${submarker}_record_output.$SLICE_IDX + mv compare.out $HOME/mysqltest${submarker}_compare_output${JOB_SUFFIX}.$SLICE_IDX + echo "finish!" + [[ "$rec_ret" != "0" ]] && return $rec_ret + [[ "$ret" != "0" ]] && return $ret + return 0 + elif [[ $CROSS_VALIDATION == "1" ]] + then + ret=255 + # 先checkout commit + # todo: 修改为把tools deploy下的init sql都带入过去 + mkdir $HOME/tmp_init_sql + mkdir $HOME/obd_tmp + find $HOME/seekdb/tools/deploy -name 'init_*.sql' | xargs -i cp {} $HOME/tmp_init_sql + cp $HOME/seekdb/tools/deploy/obd/* $HOME/obd_tmp + cd $HOME && tar -cvf tmp_init_sql.tar.gz $HOME/tmp_init_sql + cd $HOME && tar -cvf origin_obd_file.tar.gz $HOME/obd_tmp + + cd $HOME/seekdb + # 针对多个分支进行处理 + ret=0 + for commit in $CROSS_VALIDATION_COMMIT + do + git checkout -f $commit || ret=2 + # 然后重新针对obd 进行重新构建路径 + cd $HOME && tar -xvf tmp_init_sql.tar.gz && cp tmp_init_sql/* $HOME/seekdb/tools/deploy + cd $HOME && tar -xvf origin_obd_file.tar.gz && cp obd_tmp/* $HOME/seekdb/tools/deploy/obd + + if [[ -f $HOME/seekdb/deps/init/dep_create.sh ]] + then + if [[ "$IS_CE" == "1" ]] + then + cd $HOME/seekdb && ./build.sh init --ce || ret=3 + else + cd $HOME/seekdb && ./build.sh init || ret=3 + fi + else + if grep 'dep_create.sh' $HOME/seekdb/build.sh + then + cd $HOME/seekdb/deps/3rd && bash dep_create.sh all || ret=4 + else + cd $HOME/seekdb && ./build.sh init || ret=4 + fi + fi + # 根据build.sh中的内容判断依赖安装路径 + if grep 'dep_create.sh' $HOME/seekdb/build.sh + then + DEP_PATH=$HOME/seekdb/deps/3rd + else + DEP_PATH=$HOME/seekdb/rpm/.dep_create/var + fi + extra_ignore_cmd="" + if [[ `$DEP_PATH/u01/obclient/bin/mysqltest --help | grep "cmp-ignore-explain"` ]] + then + extra_ignore_cmd=" --ignore-explain=enable" + fi + + if [[ `$DEP_PATH/u01/obclient/bin/mysqltest --help | grep "disable-explain"` ]] + then + extra_ignore_cmd=" $extra_ignore_cmd --special-run" + fi + + cd $HOME/seekdb/tools/deploy + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=$DEP_PATH/u01/obclient/bin/mysqltest --obclient-bin=$DEP_PATH/u01/obclient/bin/obclient $COLLECT_ARG --init-sql-dir=$HOME/seekdb/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS_WITHOUT_CASE --suite=cross_validation $extra_ignore_cmd" + $mysqltest_cmd $INIT_FLIES $SCHE_ARGS 2>&1 | tee compare.out && ( exit ${PIPESTATUS[0]}) + test_suite_ret=$? + + FILTER=`ls -ltr $HOME/seekdb/tools/deploy/mysql_test/test_suite/cross_validation/t | grep -v total | awk '{gsub(".test","",$9);print "cross_validation."$9}' | tr '\n' ',' | head -c -1` + if [[ $FILTER == "" ]] + then + echo "without testsuite cases" + fi + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=$DEP_PATH/u01/obclient/bin/mysqltest --obclient-bin=$DEP_PATH/u01/obclient/bin/obclient $COLLECT_ARG --init-sql-dir=$HOME/seekdb/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS_WITHOUT_CASE --exclude=$FILTER --tag=cross_validation $extra_ignore_cmd" + $mysqltest_cmd $INIT_FLIES $SCHE_ARGS 2>&1 | tee compare.out && ( exit ${PIPESTATUS[0]}) + tag_ret=$? + [[ $test_suite_ret = 0 && $tag_ret = 0 ]] && current_ret=0 || current=1 + [[ $ret = 0 && $current_ret = 0 ]] && ret=0 || ret=1 + done + + if [[ $JOBNAME == 'mysqltest_opensource' ]] + then + submarker="_opensource" + else + submarker="" + fi + + mv compare.out $HOME/mysqltest${submarker}_compare_output.$SLICE_IDX + echo "finish!" + return $ret + elif [[ $WITH_SHARED_STORAGE == "1" ]] + then + # 把sensitive_test目录下的测试集合移动到tools/deploy/mysql_test + # 并且创建一个新的目录用来表示单独的测试 + if [[ ! -d $HOME/seekdb/sensitive_test/mysql_test ]] + then + return 0 + fi + need_mv_dirs=`ls -1 $HOME/seekdb/sensitive_test/mysql_test/test_suite|xargs` + run_suites="" + for need_mv_dir in $need_mv_dirs + do + new_dir=$HOME/seekdb/tools/deploy/mysql_test/test_suite/shared_storage__${need_mv_dir} + run_suites="${run_suites}shared_storage__${need_mv_dir}," + mkdir -p $new_dir + cp -r $HOME/seekdb/sensitive_test/mysql_test/test_suite/$need_mv_dir/* $new_dir + done + run_suites=`echo $run_suites | awk '{print substr($0, 1, length($0)-1)}'` + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient $COLLECT_ARG --init-sql-dir=$HOME/seekdb/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS_WITHOUT_CASE --suite=$run_suites" + $mysqltest_cmd $INIT_FLIES $SCHE_ARGS 2>&1 | tee compare.out && ( exit ${PIPESTATUS[0]}) + else + if [[ -f $HOME/seekdb/tools/deploy/error_log_filter.json && ( $BRANCH == 'master' || $BRANCH == "4_2_x_release" ) && "$FROM_FARM" == '1' ]] + then + echo "python $HOME/common/analyse_the_observer_log.py collect -p ${DATA_PATH}/observer1/log -p ${DATA_PATH}/observer2/log -i $GID -j ${JOBNAME}.${SLICE_IDX} -o $HOME -F $HOME/seekdb/tools/deploy/error_log_filter.json" > $HOME/seekdb/tools/deploy/analyse_the_observer_log.sh + chmod a+x $HOME/seekdb/tools/deploy/analyse_the_observer_log.sh + RUN_EXTRA_CHECK_CMD='--extra-script-after-case="$HOME/seekdb/tools/deploy/./analyse_the_observer_log.sh"' + fi + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient $COLLECT_ARG --init-sql-dir=$HOME/seekdb/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS" + $mysqltest_cmd $RUN_EXTRA_CHECK_CMD $INIT_FLIES $SCHE_ARGS 2>&1 | tee compare.out && ( exit ${PIPESTATUS[0]}) + ret=$? + if [[ $JOBNAME == 'mysqltest_opensource' ]] + then + submarker="_opensource" + else + submarker="" + fi + mv compare.out $HOME/mysqltest${submarker}_compare_output.$SLICE_IDX + echo "finish!" + return $ret + fi +} + +function obd_prepare_env { + obd_prepare_global + obd_prepare_obd + obd_prepare_config + obd_prepare_bin +} + +function obd_collect_log { + + echo "collect log" + cd $HOME/ + mkdir -p collected_log + mkdir -p collected_log/obd_log + mkdir -p collected_log/mysqltest_log + mkdir -p collected_log/mysqltest_rec_log + find $DATA_PATH -name 'core[.-]*' | xargs -i cp {} collected_log + [[ "$OBD_HOME" != "" ]] && mv $OBD_HOME/.obd/log/* collected_log/obd_log/ + set +x + if [[ $WITH_SHARED_STORAGE == "1" ]] + then + sed -i "s/${SENSITIVE_TEST_OSS_KEY_FOR_OBJECT_STORAGE_FOR_TOTAL}//" collected_log/obd_log/obd + fi + set -x + mv seekdb/tools/deploy/var/log/* collected_log/mysqltest_log/ + mv seekdb/tools/deploy/var/rec_log/* collected_log/mysqltest_rec_log/ + mv collected_log collected_log_$SLICE_IDX + tar zcvf collected_log_${SLICE_IDX}${JOB_SUFFIX}.tar.gz collected_log_$SLICE_IDX +} + +function collect_obd_case_log { + + echo "collect obd case log" + cd $HOME/ + if [[ $JOBNAME == 'mysqltest_opensource' ]] + then + submarker="_opensource" + else + submarker="" + fi + [[ "$OBD_HOME" != "" ]] && cat $OBD_HOME/.obd/log/* | grep '\[INFO\]' > mysqltest_cases_log${submarker}_$SLICE_IDX.output + +} + + +export -f obd_prepare_global +export -f obd_prepare_config +export -f obd_run_mysqltest +export -f obd_init_cluster +export -f is_case_selector_arg + +function run { + if [[ -f /root/tmp/.ssh/id_rsa ]] + then + mv /root/tmp/.ssh/id_rsa /root/.ssh/ + fi + if ([[ -f $HOME/seekdb/rpm/oceanbase.deps ]] && [[ "$(grep 'ob-deploy' $HOME/seekdb/rpm/oceanbase.deps )" != "" ]]) || + ([[ -f $HOME/seekdb/deps/3rd/oceanbase.el7.x86_64.deps ]] && [[ "$(grep 'ob-deploy' $HOME/seekdb/deps/3rd/oceanbase.el7.x86_64.deps )" != "" ]]) || + ([[ -f $HOME/seekdb/deps/init/oceanbase.el7.x86_64.deps ]] && [[ "$(grep 'ob-deploy' $HOME/seekdb/deps/init/oceanbase.el7.x86_64.deps )" != "" ]]) + then + if [[ $WITH_SHARED_STORAGE == "1" ]] + then + # 单独判断下是否有对应的case集合 + if [[ ! -d $HOME/seekdb/sensitive_test/mysql_test ]] + then + return 0 + fi + fi + timeout=18000 + [[ "$SPECIAL_RUN" == "1" ]] && timeout=72000 + obd_prepare_env && + timeout $timeout bash -c "obd_run_mysqltest" + test_ret=$? + error_log_ret=0 + if [[ -f $HOME/seekdb/tools/deploy/error_log_filter.json && $BRANCH == 'master' ]] + then + # notice the core + python $HOME/common/analyse_the_observer_core.py collect -p ${DATA_PATH} -j ${JOBNAME}.${SLICE_IDX} -o $HOME + error_log_ret=0 + collect_obd_case_log + fi + if [[ $test_ret -ne 0 || $error_log_ret -ne 0 ]] + then + obd_collect_log + fi + return `[[ $test_ret = 0 && $error_log_ret = 0 ]]` + else + prepare_env && + timeout 18000 bash -c "run_mysqltest" + test_ret=$? + error_log_ret=0 + if [[ -f $HOME/seekdb/tools/deploy/error_log_filter.json && ( $BRANCH == 'master' || $BRANCH == "4_2_x_release" ) ]] + then + python $HOME/common/analyse_the_observer_log.py collect -p ${DATA_PATH}/observer1/log -p ${DATA_PATH}/observer2/log -i $GID -j ${JOBNAME}.${SLICE_IDX} -o $HOME -F $HOME/seekdb/tools/deploy/error_log_filter.json + error_log_ret=0 + fi + if [[ $test_ret -ne 0 || $error_log_ret -ne 0 ]] + then + collect_log + fi + return `[[ $test_ret = 0 && $error_log_ret = 0 ]]` + fi + +} + +run "$@" \ No newline at end of file diff --git a/.github/script/seekdb_wait_result.py b/.github/script/seekdb_wait_result.py new file mode 100644 index 000000000..60a66ef3c --- /dev/null +++ b/.github/script/seekdb_wait_result.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +在 GitHub Actions 中用于「等待 SeekDB Farm2 结果」的脚本。 +配合 seekdb_robot_for_ob_for_retry.py:robot 发现 workflow 里 seekdb3 步骤 in_progress 后会发 Farm2 任务, +任务结束后把结果写到 OSS;本脚本轮询该 OSS 结果文件,读到后根据 status 退出并写 GitHub output。 + +用法(在 workflow 的 job seekdb3、step 名以 "action by seekdb3" 开头): + export GITHUB_RUN_ID=123456789 + export GITHUB_RUN_ATTEMPT=1 + python3 seekdb_wait_result.py [--timeout 14400] [--poll-interval 15] + +结果文件 URL: https://obfarm-ce.oss-cn-hongkong.aliyuncs.com/farm/seekdb_results/{run_id}-{attempt}.json +status: 4=success, 其他为失败。 +""" + +import os +import sys +import json +import time +import argparse + +try: + import requests +except ImportError: + print("需要 requests: pip install requests", file=sys.stderr) + sys.exit(2) + +# 与 seekdb_robot 里 RESULT_FILE_KEY 及 bucket 一致 +OSS_RESULT_BASE = "https://obfarm-ce.oss-cn-hongkong.aliyuncs.com/farm/seekdb_results" +# Farm2 TaskStatusEnum.success = 4 +STATUS_SUCCESS = 4 + + +def main(): + parser = argparse.ArgumentParser(description="轮询 OSS 上的 SeekDB Farm2 结果,用于 GitHub Actions 回调") + parser.add_argument("--timeout", type=int, default=24 * 3600, help="轮询超时秒数,默认 24h") + parser.add_argument("--poll-interval", type=int, default=15, help="轮询间隔秒数") + parser.add_argument("--run-id", default=None, help="GitHub run id,默认用环境变量 GITHUB_RUN_ID") + parser.add_argument("--attempt", default=None, help="run attempt,默认用环境变量 GITHUB_RUN_ATTEMPT 或 1") + args = parser.parse_args() + + run_id = args.run_id or os.environ.get("GITHUB_RUN_ID") + attempt = args.attempt or os.environ.get("GITHUB_RUN_ATTEMPT", "1") + if not run_id: + print("[ERROR] 需要 GITHUB_RUN_ID 或 --run-id", file=sys.stderr) + sys.exit(2) + + result_key = "{}-{}.json".format(run_id, attempt) + url = "{}/{}".format(OSS_RESULT_BASE.rstrip("/"), result_key) + end_time = time.time() + args.timeout + + print("[INFO] 等待 Farm2 结果: {}".format(url)) + while time.time() <= end_time: + try: + r = requests.get(url, timeout=30) + if r.status_code == 200: + data = r.json() + status = data.get("status") + task_id = data.get("task_id", "") + _write_github_output(task_id=task_id, status=status, success=(status == STATUS_SUCCESS)) + if status == STATUS_SUCCESS: + print("[INFO] SeekDB Farm2 任务成功, task_id={}".format(task_id)) + sys.exit(0) + print("[INFO] SeekDB Farm2 任务未通过, status={}, task_id={}".format(status, task_id)) + sys.exit(1) + except requests.RequestException: + pass + except json.JSONDecodeError: + pass + time.sleep(args.poll_interval) + + _write_github_output(task_id="", status="", success=False) + print("[ERROR] 等待 Farm2 结果超时", file=sys.stderr) + sys.exit(1) + + +def _write_github_output(task_id="", status="", success=False): + out = os.environ.get("GITHUB_OUTPUT") + if not out: + return + with open(out, "a") as f: + f.write("task_id={}\n".format(task_id)) + f.write("status={}\n".format(status)) + f.write("success={}\n".format("true" if success else "false")) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/seekdb.yml b/.github/workflows/seekdb.yml new file mode 100644 index 000000000..c4879db1e --- /dev/null +++ b/.github/workflows/seekdb.yml @@ -0,0 +1,249 @@ +# SeekDB CI - 执行下沉版(Native) +# setup 准备 observer → 4 个并行 mysqltest job(各独立容器)→ collect 合并结果 +name: SeekDB CI + +on: + pull_request: + branches: [master, develop] + types: [opened, reopened, synchronize, ready_for_review] + paths-ignore: + - 'docs/**' + - '.github/ISSUE_TEMPLATE/**' + - '.github/pull_request_template.md' + - 'README.md' + - 'README_CN.md' + - 'CONTRIBUTING.md' + schedule: + - cron: '30 9 * * *' + workflow_dispatch: + +env: + MYSQLTEST_SLICES: "4" + cache-name: cache-deps + SEEKDB_TASK_DIR: ${{ github.workspace }}/seekdb_build/${{ github.run_id }} + SEEKDB_RESULT_NFS_ROOT: /data/storage/jenkins_on_k8s/results/seekdb + +jobs: + seekdb-setup: + name: setup + if: ${{ github.repository_owner == 'oceanbase' && (github.event_name != 'pull_request' || !github.event.pull_request.draft) }} + runs-on: arc-runner-k8s-ci-org + container: + image: harbor.oceanbase-dev.com/obrde-test/ob-actions-runner-worker:0.0.2 + steps: + - name: Checkout + uses: actions/checkout@v4 + + + - name: Cache deps + uses: actions/cache@v4 + with: + key: ${{ runner.os }}-build-${{ env.cache-name }}-el9.x86_64-${{ hashFiles('deps/init/oceanbase.el9.x86_64.deps') }} + path: deps/3rd + + - name: Prepare (jobargs, run_jobs) + shell: bash + run: | + chmod +x .github/script/seekdb/prepare.sh + export GITHUB_RUN_ID="${{ github.run_id }}" + export GITHUB_WORKSPACE="${{ github.workspace }}" + export MYSQLTEST_SLICES="${{ env.MYSQLTEST_SLICES }}" + export SEEKDB_TASK_DIR="${{ env.SEEKDB_TASK_DIR }}" + .github/script/seekdb/prepare.sh + - name: Setup ccache + uses: hendrikmuhs/ccache-action@v1.2.11 + with: + max-size: 800M + save: ${{ github.event_name == 'push' }} + key: seekdb + + - name: Compile + id: compile + shell: bash + run: | + chmod +x .github/script/seekdb/compile.sh + export GITHUB_RUN_ID="${{ github.run_id }}" + WS="${{ github.workspace }}" + if [[ ! -f "$WS/build.sh" ]] && [[ -f "$(pwd)/build.sh" ]]; then + export GITHUB_WORKSPACE="$(pwd)" + else + export GITHUB_WORKSPACE="$WS" + fi + export SEEKDB_TASK_DIR="${{ env.SEEKDB_TASK_DIR }}" + export RELEASE_MODE="${{ vars.RELEASE_MODE || '' }}" + export FORWARDING_HOST="${{ vars.FORWARDING_HOST || '' }}" + export PATH="$GITHUB_WORKSPACE/deps/3rd/usr/local/oceanbase/devtools/bin:$PATH" + .github/script/seekdb/compile.sh + - name: Prepare observer + shell: bash + run: | + if [[ ! -x observer ]]; then + for base in build_debug build_release build; do + for cand in "$base/observer" "$base/src/observer/observer"; do + if [[ -f "$cand" ]] && [[ -x "$cand" ]]; then + cp -f "$cand" observer && chmod +x observer + echo "observer copied from $cand" + break 2 + fi + done + done + fi + if [[ ! -x observer ]]; then echo "observer missing or not executable"; exit 1; fi + echo "observer size: $(stat -c%s observer 2>/dev/null || stat -f%z observer 2>/dev/null) bytes" + - name: Compress observer + shell: bash + run: | + zstd -f -q -T0 -o observer.zst observer + echo "observer.zst size: $(stat -c%s observer.zst 2>/dev/null || stat -f%z observer.zst 2>/dev/null) bytes" + # observer 与 run_attempt 无关:仅重跑失败 job 时 setup 可能跳过,若路径含 attempt-N + # 会去找 attempt-2/setup 而产物仍在 attempt-1/setup,导致「observer not found on NFS」。 + - name: Persist observer to NFS + shell: bash + run: | + RUN_ROOT="${{ env.SEEKDB_RESULT_NFS_ROOT }}/${{ github.run_id }}" + DEST_DIR="$RUN_ROOT/setup" + mkdir -p "$DEST_DIR" + cp -f observer.zst "$DEST_DIR/" + echo "observer.zst saved to $DEST_DIR/observer.zst" + seekdb-mysqltest: + name: mysqltest-slice-${{ matrix.slice }} + needs: seekdb-setup + runs-on: arc-runner-k8s-ci-org + container: + image: harbor.oceanbase-dev.com/obrde-test/ob-actions-runner-worker:0.0.2 + strategy: + fail-fast: false + matrix: + slice: [0, 1, 2, 3] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Place observer (from NFS) + shell: bash + run: | + set -eo pipefail + RUN_ROOT="${{ env.SEEKDB_RESULT_NFS_ROOT }}/${{ github.run_id }}" + ATT="${{ github.run_attempt }}" + OBSERVER_ZST="" + if [[ -f "$RUN_ROOT/setup/observer.zst" ]]; then + OBSERVER_ZST="$RUN_ROOT/setup/observer.zst" + elif [[ -f "$RUN_ROOT/attempt-${ATT}/setup/observer.zst" ]]; then + OBSERVER_ZST="$RUN_ROOT/attempt-${ATT}/setup/observer.zst" + else + shopt -s nullglob + _fb=( "$RUN_ROOT"/attempt-*/setup/observer.zst ) + shopt -u nullglob + if (( ${#_fb[@]} > 0 )); then + OBSERVER_ZST=$(printf '%s\n' "${_fb[@]}" | xargs ls -t | head -1) + fi + fi + if [[ -n "$OBSERVER_ZST" && -f "$OBSERVER_ZST" ]]; then + echo "Using observer.zst: $OBSERVER_ZST" + zstd -d -f -q -o observer "$OBSERVER_ZST" && chmod +x observer + else + echo "observer not found on NFS under $RUN_ROOT (tried setup/ and attempt-*/setup/)" + exit 1 + fi + ls -la observer + - name: Run mysqltest slice + shell: bash + run: | + chmod +x .github/script/seekdb/mysqltest_slice.sh + export GITHUB_RUN_ID="${{ github.run_id }}" GITHUB_WORKSPACE="${{ github.workspace }}" + export SEEKDB_TASK_DIR="${{ github.workspace }}/seekdb_build/${{ github.run_id }}/slice_${{ matrix.slice }}" + export SLICE_IDX=${{ matrix.slice }} SLICES="${{ env.MYSQLTEST_SLICES }}" + export BRANCH="${{ github.ref_name }}" FORWARDING_HOST="${{ vars.FORWARDING_HOST || '' }}" + mkdir -p "$SEEKDB_TASK_DIR" + .github/script/seekdb/mysqltest_slice.sh + - name: Collect slice result + if: success() || failure() + shell: bash + run: | + chmod +x .github/script/seekdb/collect_result.sh + export GITHUB_RUN_ID="${{ github.run_id }}" GITHUB_WORKSPACE="${{ github.workspace }}" + export SEEKDB_TASK_DIR="${{ github.workspace }}/seekdb_build/${{ github.run_id }}/slice_${{ matrix.slice }}" + .github/script/seekdb/collect_result.sh seekdb_result.json + continue-on-error: true + + - name: Persist slice result to NFS + if: success() || failure() + shell: bash + run: | + DEST_DIR="${{ env.SEEKDB_RESULT_NFS_ROOT }}/${{ github.run_id }}/attempt-${{ github.run_attempt }}/slices/slice_${{ matrix.slice }}" + mkdir -p "$DEST_DIR" + if [[ -f seekdb_result.json ]]; then + cp -f seekdb_result.json "$DEST_DIR/" + else + echo "skip copying slice result: seekdb_result.json not found" + fi + if [[ -d "${{ github.workspace }}/seekdb_build/${{ github.run_id }}/slice_${{ matrix.slice }}" ]]; then + cp -a "${{ github.workspace }}/seekdb_build/${{ github.run_id }}/slice_${{ matrix.slice }}" "$DEST_DIR/seekdb_task_dir" + fi + continue-on-error: true + + # 失败时保存 collected_log 到 NFS 便于排查:obd_log、mysqltest_log、mysqltest_rec_log、core + - name: Persist collected_log to NFS (on failure) + if: failure() + shell: bash + run: | + SRC="seekdb_home_${{ matrix.slice }}/collected_log_${{ matrix.slice }}.tar.gz" + DEST_DIR="${{ env.SEEKDB_RESULT_NFS_ROOT }}/${{ github.run_id }}/attempt-${{ github.run_attempt }}/logs/slice_${{ matrix.slice }}" + mkdir -p "$DEST_DIR" + if [[ -f "$SRC" ]]; then + cp -f "$SRC" "$DEST_DIR/" + echo "collected log saved to $DEST_DIR/" + else + echo "skip copying collected log: $SRC not found" + fi + continue-on-error: true + + - name: Job summary (logs hint) + if: always() + run: | + if [[ -f "seekdb_home_${{ matrix.slice }}/collected_log_${{ matrix.slice }}.tar.gz" ]]; then + echo "### Slice ${{ matrix.slice }} collected_log saved" >> "$GITHUB_STEP_SUMMARY" + echo "Saved to \`${{ env.SEEKDB_RESULT_NFS_ROOT }}/${{ github.run_id }}/attempt-${{ github.run_attempt }}/logs/slice_${{ matrix.slice }}/collected_log_${{ matrix.slice }}.tar.gz\`. Contents: \`obd_log/\`, \`mysqltest_log/\`, \`mysqltest_rec_log/\`, core dumps." >> "$GITHUB_STEP_SUMMARY" + fi + seekdb-collect: + # 仅在 4 个 mysqltest slice 都跑完(成功或失败)时执行;若 mysqltest 被 skip 则不跑 + if: ${{ always() && needs.seekdb-mysqltest.result != 'skipped' }} + name: collect + needs: seekdb-mysqltest + runs-on: arc-runner-k8s-ci-org + container: + image: harbor.oceanbase-dev.com/obrde-test/ob-actions-runner-worker:0.0.2 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Collect slice results from NFS + shell: bash + run: | + SRC_ROOT="${{ env.SEEKDB_RESULT_NFS_ROOT }}/${{ github.run_id }}/attempt-${{ github.run_attempt }}/slices" + mkdir -p slice_results + if [[ -d "$SRC_ROOT" ]]; then + cp -a "$SRC_ROOT/." slice_results/ + else + echo "slice result directory not found on NFS: $SRC_ROOT" + fi + - name: Merge results and persist + shell: bash + run: | + RUN_ID="${{ github.run_id }}" + FAILED="" + SUCCESS="true" + while IFS= read -r f; do + [[ -f "$f" ]] || continue + grep -q '"success":false' "$f" 2>/dev/null && SUCCESS="false" + while IFS= read -r c; do + [[ -n "$c" ]] && FAILED="${FAILED}${FAILED:+,}\"${c//\"/\\\"}\"" + done < <(grep -oP '"failed_cases":\[\K[^\]]*' "$f" 2>/dev/null | tr ',' '\n' | sed 's/^"\|"$//g') + done < <(find slice_results -name 'seekdb_result.json' 2>/dev/null) + echo "{\"success\":$SUCCESS,\"run_id\":\"$RUN_ID\",\"failed_cases\":[${FAILED:-}]}" > seekdb_result.json + echo "=== SeekDB native result ===" + cat seekdb_result.json + DEST_DIR="${{ env.SEEKDB_RESULT_NFS_ROOT }}/${{ github.run_id }}/attempt-${{ github.run_attempt }}/collect" + mkdir -p "$DEST_DIR" + cp -f seekdb_result.json "$DEST_DIR/" + echo "seekdb_result.json saved to $DEST_DIR/seekdb_result.json" \ No newline at end of file