From b9638710a75fd3d22e522d4179174204b01278cc Mon Sep 17 00:00:00 2001 From: guoyan Date: Thu, 5 Feb 2026 15:55:19 +0800 Subject: [PATCH 01/11] test ci --- .github/script/seekdb_wait_result.py | 89 ++++++++++++++++++++++++++++ .github/workflows/seekdb-farm2.yml | 40 +++++++++++++ test/test_ci | 1 + 3 files changed, 130 insertions(+) create mode 100644 .github/script/seekdb_wait_result.py create mode 100644 .github/workflows/seekdb-farm2.yml create mode 100644 test/test_ci diff --git a/.github/script/seekdb_wait_result.py b/.github/script/seekdb_wait_result.py new file mode 100644 index 000000000..60a66ef3c --- /dev/null +++ b/.github/script/seekdb_wait_result.py @@ -0,0 +1,89 @@ +#!/usr/bin/env python3 +# -*- coding: utf-8 -*- +""" +在 GitHub Actions 中用于「等待 SeekDB Farm2 结果」的脚本。 +配合 seekdb_robot_for_ob_for_retry.py：robot 发现 workflow 里 seekdb3 步骤 in_progress 后会发 Farm2 任务， +任务结束后把结果写到 OSS；本脚本轮询该 OSS 结果文件，读到后根据 status 退出并写 GitHub output。 + +用法（在 workflow 的 job seekdb3、step 名以 "action by seekdb3" 开头）: + export GITHUB_RUN_ID=123456789 + export GITHUB_RUN_ATTEMPT=1 + python3 seekdb_wait_result.py [--timeout 14400] [--poll-interval 15] + +结果文件 URL: https://obfarm-ce.oss-cn-hongkong.aliyuncs.com/farm/seekdb_results/{run_id}-{attempt}.json +status: 4=success, 其他为失败。 +""" + +import os +import sys +import json +import time +import argparse + +try: + import requests +except ImportError: + print("需要 requests: pip install requests", file=sys.stderr) + sys.exit(2) + +# 与 seekdb_robot 里 RESULT_FILE_KEY 及 bucket 一致 +OSS_RESULT_BASE = "https://obfarm-ce.oss-cn-hongkong.aliyuncs.com/farm/seekdb_results" +# Farm2 TaskStatusEnum.success = 4 +STATUS_SUCCESS = 4 + + +def main(): + parser = argparse.ArgumentParser(description="轮询 OSS 上的 SeekDB Farm2 结果，用于 GitHub Actions 回调") + parser.add_argument("--timeout", type=int, default=24 * 3600, help="轮询超时秒数，默认 24h") + parser.add_argument("--poll-interval", type=int, default=15, help="轮询间隔秒数") + parser.add_argument("--run-id", default=None, help="GitHub run id，默认用环境变量 GITHUB_RUN_ID") + parser.add_argument("--attempt", default=None, help="run attempt，默认用环境变量 GITHUB_RUN_ATTEMPT 或 1") + args = parser.parse_args() + + run_id = args.run_id or os.environ.get("GITHUB_RUN_ID") + attempt = args.attempt or os.environ.get("GITHUB_RUN_ATTEMPT", "1") + if not run_id: + print("[ERROR] 需要 GITHUB_RUN_ID 或 --run-id", file=sys.stderr) + sys.exit(2) + + result_key = "{}-{}.json".format(run_id, attempt) + url = "{}/{}".format(OSS_RESULT_BASE.rstrip("/"), result_key) + end_time = time.time() + args.timeout + + print("[INFO] 等待 Farm2 结果: {}".format(url)) + while time.time() <= end_time: + try: + r = requests.get(url, timeout=30) + if r.status_code == 200: + data = r.json() + status = data.get("status") + task_id = data.get("task_id", "") + _write_github_output(task_id=task_id, status=status, success=(status == STATUS_SUCCESS)) + if status == STATUS_SUCCESS: + print("[INFO] SeekDB Farm2 任务成功, task_id={}".format(task_id)) + sys.exit(0) + print("[INFO] SeekDB Farm2 任务未通过, status={}, task_id={}".format(status, task_id)) + sys.exit(1) + except requests.RequestException: + pass + except json.JSONDecodeError: + pass + time.sleep(args.poll_interval) + + _write_github_output(task_id="", status="", success=False) + print("[ERROR] 等待 Farm2 结果超时", file=sys.stderr) + sys.exit(1) + + +def _write_github_output(task_id="", status="", success=False): + out = os.environ.get("GITHUB_OUTPUT") + if not out: + return + with open(out, "a") as f: + f.write("task_id={}\n".format(task_id)) + f.write("status={}\n".format(status)) + f.write("success={}\n".format("true" if success else "false")) + + +if __name__ == "__main__": + main() diff --git a/.github/workflows/seekdb-farm2.yml b/.github/workflows/seekdb-farm2.yml new file mode 100644 index 000000000..28b0206b1 --- /dev/null +++ b/.github/workflows/seekdb-farm2.yml @@ -0,0 +1,40 @@ +# 示例：在 GitHub 上触发 SeekDB 测试并由 Farm2 执行，结果回写到 OSS 后本 workflow 轮询并回调 +# +# 使用方式： +# 1. 将本文件复制到 oceanbase/seekdb 的 .github/workflows/，并把 farm2-client 的 bin/seekdb_wait_result.py 复制到该仓库 .github/scripts/seekdb_wait_result.py +# 2. 确保 seekdb_robot_for_ob_for_retry.py 在服务器上常驻运行（轮询 GitHub 发现本 job 的 step in_progress 后发 Farm2 任务，结束写 OSS） +# 3. job 名必须为 seekdb3，且有一个 step 名以 "action by seekdb3" 开头 +# +name: SeekDB Farm2 + +on: + push: + branches: [ develop, master ] + pull_request: + branches: [ develop, master ] + workflow_dispatch: + +jobs: + seekdb3: + name: seekdb3 + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: action by seekdb3 + id: seekdb_farm2 + env: + GITHUB_RUN_ID: ${{ github.run_id }} + GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} + run: | + pip install -q requests + python3 .github/script/seekdb_wait_result.py --timeout 14400 --poll-interval 15 + continue-on-error: true + + - name: Farm2 result + if: always() + run: | + echo "task_id: ${{ steps.seekdb_farm2.outputs.task_id }}" + echo "success: ${{ steps.seekdb_farm2.outputs.success }}" + if [ "${{ steps.seekdb_farm2.outputs.success }}" != "true" ]; then exit 1; fi diff --git a/test/test_ci b/test/test_ci new file mode 100644 index 000000000..c79d0df0e --- /dev/null +++ b/test/test_ci @@ -0,0 +1 @@ +1111111111111testtttttt \ No newline at end of file From c130690ff28a0acc4657d0e6257c92d82c270cdc Mon Sep 17 00:00:00 2001 From: guoyan1996 <1028109394@qq.com> Date: Fri, 3 Apr 2026 17:37:05 +0800 Subject: [PATCH 02/11] add CI --- .github/script/seekdb/collect_result.sh | 32 + .github/script/seekdb/compile.sh | 63 ++ .github/script/seekdb/mysqltest_slice.sh | 29 + .github/script/seekdb/prepare.sh | 27 + .github/script/seekdb/scripts/farm_compile.sh | 13 + .../seekdb/scripts/mysqltest_for_seekdb.sh | 812 ++++++++++++++++++ .github/workflows/buildbase/seekdb.yml | 229 +++++ 7 files changed, 1205 insertions(+) create mode 100644 .github/script/seekdb/collect_result.sh create mode 100644 .github/script/seekdb/compile.sh create mode 100644 .github/script/seekdb/mysqltest_slice.sh create mode 100644 .github/script/seekdb/prepare.sh create mode 100644 .github/script/seekdb/scripts/farm_compile.sh create mode 100644 .github/script/seekdb/scripts/mysqltest_for_seekdb.sh create mode 100644 .github/workflows/buildbase/seekdb.yml diff --git a/.github/script/seekdb/collect_result.sh b/.github/script/seekdb/collect_result.sh new file mode 100644 index 000000000..7459768bb --- /dev/null +++ b/.github/script/seekdb/collect_result.sh @@ -0,0 +1,32 @@ +#!/usr/bin/env bash +# Collect result: read fail_cases.output and write seekdb_result.json. +# Usage: collect_result.sh +# Required env: GITHUB_WORKSPACE, GITHUB_RUN_ID, SEEKDB_TASK_DIR +set -e + +OUT_PATH="${1:-$GITHUB_WORKSPACE/seekdb_result.json}" +TASK_DIR="${SEEKDB_TASK_DIR:?}" +WORKSPACE="${GITHUB_WORKSPACE:-.}" + +FAIL_FILE="$TASK_DIR/fail_cases.output" + +failed_cases=() +if [[ -f "$FAIL_FILE" ]] && [[ -s "$FAIL_FILE" ]]; then + while IFS= read -r line; do + [[ -n "$line" ]] && failed_cases+=("$line") + done < "$FAIL_FILE" +fi + +success="true" +[[ ${#failed_cases[@]} -gt 0 ]] && success="false" + +# Build JSON (escape quotes in case names) +cs="" +for c in "${failed_cases[@]}"; do + esc="${c//\"/\\\"}" + [[ -n "$cs" ]] && cs="$cs," + cs="$cs\"$esc\"" +done +echo "{\"success\":$success,\"run_id\":\"${GITHUB_RUN_ID:-0}\",\"failed_cases\":[$cs]}" > "$OUT_PATH" + +echo "[collect_result.sh] Result written to $OUT_PATH (success=$success, failed=${#failed_cases[@]})."c \ No newline at end of file diff --git a/.github/script/seekdb/compile.sh b/.github/script/seekdb/compile.sh new file mode 100644 index 000000000..83592d305 --- /dev/null +++ b/.github/script/seekdb/compile.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Compile: 步骤与 .github/workflows/buildbase 一致（init → build.sh $TARGET → cd build_* && make）。 +# Required env: GITHUB_WORKSPACE, SEEKDB_TASK_DIR +# Optional: RELEASE_MODE, FORWARDING_HOST, MAKE, MAKE_ARGS +set -e + +WORKSPACE="${GITHUB_WORKSPACE:?}" +TASK_DIR="${SEEKDB_TASK_DIR:?}" + +# 调试：便于排查 k8s/container 下 No build.sh +echo "[compile.sh] WORKSPACE=$WORKSPACE" +echo "[compile.sh] pwd=$(pwd)" +ls -la "$WORKSPACE/" 2>/dev/null | head -20 || true +echo "[compile.sh] build.sh: -f=$([[ -f "$WORKSPACE/build.sh" ]] && echo 1 || echo 0) -x=$([[ -x "$WORKSPACE/build.sh" ]] && echo 1 || echo 0)" + +export GITHUB_WORKSPACE="$WORKSPACE" +export SEEKDB_TASK_DIR="$TASK_DIR" +export PACKAGE_TYPE="${RELEASE_MODE:+release}" +export PACKAGE_TYPE="${PACKAGE_TYPE:-debug}" +export MAKE="${MAKE:-make}" +export MAKE_ARGS="${MAKE_ARGS:--j32}" +export PATH="$WORKSPACE/deps/3rd/usr/local/oceanbase/devtools/bin:$PATH" +[[ -n "$FORWARDING_HOST" ]] && echo "$FORWARDING_HOST mirrors.oceanbase.com" >> /etc/hosts 2>/dev/null || true + +cd "$WORKSPACE" +mkdir -p "$TASK_DIR" + +BUILD_TARGET="${PACKAGE_TYPE:-debug}" +BUILD_DIR="build_${BUILD_TARGET}" +compile_ret=0 + +# 存在即可（不要求 -x），用 bash 执行 +if [[ ! -f "$WORKSPACE/build.sh" ]]; then + echo "[compile.sh] No build.sh at $WORKSPACE/build.sh, skip." +else + # Step 1: Build init（与 buildbase 一致，只传 init，先拉取/安装 deps 再才能用 cmake） + bash "$WORKSPACE/build.sh" init 2>&1 | tee "$TASK_DIR/compile_init.output" + [[ ${PIPESTATUS[0]} -ne 0 ]] && exit 1 + bash "$WORKSPACE/build.sh" "$BUILD_TARGET" -DOB_USE_CCACHE=ON -DNEED_PARSER_CACHE=OFF 2>&1 | tee "$TASK_DIR/compile_configure.output" + [[ ${PIPESTATUS[0]} -ne 0 ]] && exit 1 + set +e + command -v ccache >/dev/null 2>&1 && ccache -z || true + (cd "$WORKSPACE/$BUILD_DIR" && $MAKE $MAKE_ARGS observer) 2>&1 | tee "$TASK_DIR/compile.output" + compile_ret=${PIPESTATUS[0]} + command -v ccache >/dev/null 2>&1 && ccache -s || true + set -e +fi + +# 产物落到 build_*，打包 observer/obproxy 为 zst 并拷贝到任务目录 +for binary in observer obproxy; do + for base in . build_debug build_release build; do + if [[ -f "$WORKSPACE/$base/$binary" ]]; then + cp -f "$WORKSPACE/$base/$binary" "$WORKSPACE/$binary" 2>/dev/null || true + break + fi + done + if [[ -f "$WORKSPACE/$binary" ]]; then + command -v zstd >/dev/null 2>&1 && zstd -f "$WORKSPACE/$binary" || true + [[ -f "$WORKSPACE/$binary.zst" ]] && cp -f "$WORKSPACE/$binary.zst" "$TASK_DIR/" || true + fi +done + +exit "$compile_ret" diff --git a/.github/script/seekdb/mysqltest_slice.sh b/.github/script/seekdb/mysqltest_slice.sh new file mode 100644 index 000000000..d1ce0a849 --- /dev/null +++ b/.github/script/seekdb/mysqltest_slice.sh @@ -0,0 +1,29 @@ +#!/usr/bin/env bash +# Run one mysqltest slice (slice index from SLICE_IDX). +# Required env: GITHUB_WORKSPACE, SEEKDB_TASK_DIR, SLICE_IDX, SLICES, BRANCH +# Optional: FORWARDING_HOST +set -e + +WORKSPACE="${GITHUB_WORKSPACE:?}" +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +SCRIPTS_DIR="$SCRIPT_DIR/scripts" + +export GITHUB_WORKSPACE="$WORKSPACE" +export SEEKDB_TASK_DIR="${SEEKDB_TASK_DIR:?}" +export SLICE_IDX="${SLICE_IDX:-0}" +export SLICES="${SLICES:-4}" +export BRANCH="${BRANCH:-master}" + + +# Copy compile artifacts from task dir to workspace if running in container +for f in observer.zst obproxy.zst; do + if [[ -f "$SEEKDB_TASK_DIR/$f" ]] && [[ ! -f "$WORKSPACE/$f" ]]; then + cp -f "$SEEKDB_TASK_DIR/$f" "$WORKSPACE/" || true + fi +done + +if [[ -f "$SCRIPTS_DIR/mysqltest_for_seekdb.sh" ]]; then + bash "$SCRIPTS_DIR/mysqltest_for_seekdb.sh" "$@" +else + echo "[mysqltest_slice.sh] No mysqltest_for_seekdb.sh, skip slice $SLICE_IDX." +fi \ No newline at end of file diff --git a/.github/script/seekdb/prepare.sh b/.github/script/seekdb/prepare.sh new file mode 100644 index 000000000..29955e0b6 --- /dev/null +++ b/.github/script/seekdb/prepare.sh @@ -0,0 +1,27 @@ +#!/usr/bin/env bash +# Prepare: create task dir and generate jobargs.output / run_jobs.output +# Required env: GITHUB_WORKSPACE, GITHUB_RUN_ID, MYSQLTEST_SLICES, SEEKDB_TASK_DIR +set -e + +WORKSPACE="${GITHUB_WORKSPACE:?}" +RUN_ID="${GITHUB_RUN_ID:?}" +SLICES="${MYSQLTEST_SLICES:-4}" +TASK_DIR="${SEEKDB_TASK_DIR:?}" + +mkdir -p "$TASK_DIR" + +# run_jobs.output: compile + N mysqltest slices (align with seekdb.groovy) +echo '++compile++' > "$TASK_DIR/run_jobs.output" +for i in $(seq 0 $((SLICES - 1))); do + echo "++mysqltest++${i}++" >> "$TASK_DIR/run_jobs.output" +done + +# jobargs.output: build options (align with seekdb.groovy) +{ + echo '++is_cmake++' + echo '++need_agentserver++0' + echo '++need_libobserver_so++0' + echo '++need_liboblog++0' +} > "$TASK_DIR/jobargs.output" + +echo "[prepare.sh] SEEKDB_TASK_DIR=$TASK_DIR run_jobs and jobargs written." \ No newline at end of file diff --git a/.github/script/seekdb/scripts/farm_compile.sh b/.github/script/seekdb/scripts/farm_compile.sh new file mode 100644 index 000000000..f5a085b37 --- /dev/null +++ b/.github/script/seekdb/scripts/farm_compile.sh @@ -0,0 +1,13 @@ +#!/usr/bin/env bash +# Farm compile: build observer / obproxy in workspace. +# Expects: WORKSPACE, SEEKDB_TASK_DIR, PACKAGE_TYPE (debug|release), REPO=server +set -e + +cd "$WORKSPACE" +# Use repo build.sh; debug or release from PACKAGE_TYPE +build_type="${PACKAGE_TYPE:-debug}" +if [[ -x "$WORKSPACE/build.sh" ]]; then + bash "$WORKSPACE/build.sh" "$build_type" --init --make +else + echo "[farm_compile.sh] No build.sh in $WORKSPACE, skip." +fi \ No newline at end of file diff --git a/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh b/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh new file mode 100644 index 000000000..be0694524 --- /dev/null +++ b/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh @@ -0,0 +1,812 @@ +#!/bin/bash + +set -x + +# 脚本所在目录，末尾 source frame.sh 用 +SEEKDB_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# seekdb native：GITHUB_WORKSPACE=仓库根，SLICE_IDX=slice 序号（0,1,2,3） +# 脚本在 .github/script/seekdb/scripts/，仓库根为 4 级上级 +REPO_ROOT_BY_SCRIPT="$(cd "$SEEKDB_SCRIPT_DIR/../../../.." && pwd)" +export GITHUB_WORKSPACE="${GITHUB_WORKSPACE:-$REPO_ROOT_BY_SCRIPT}" +# 若传入的 GITHUB_WORKSPACE 在容器内无 build.sh（如 runner 传主机路径），改用实际存在的仓库根 +if [[ ! -f "$GITHUB_WORKSPACE/build.sh" ]]; then + if [[ -f "$REPO_ROOT_BY_SCRIPT/build.sh" ]]; then + export GITHUB_WORKSPACE="$REPO_ROOT_BY_SCRIPT" + elif [[ -f "$(pwd)/build.sh" ]]; then + export GITHUB_WORKSPACE="$(pwd)" + fi +fi +export USER="${USER:-$(whoami)}" +SEEKDB_HOME="${GITHUB_WORKSPACE}/seekdb_home_${SLICE_IDX:-0}" +mkdir -p "$SEEKDB_HOME" +ln -sf "$GITHUB_WORKSPACE" "$SEEKDB_HOME/seekdb" +export HOME="$SEEKDB_HOME" +export SLOT_ID="${SLICE_IDX:-0}" +# 保证 HOME/observer 存在：优先用 WORKSPACE 下 observer，否则解压 observer.zst +if [[ -x "$GITHUB_WORKSPACE/observer" ]]; then + cp -f "$GITHUB_WORKSPACE/observer" "$HOME/observer" && chmod +x "$HOME/observer" +elif [[ -f "$GITHUB_WORKSPACE/observer.zst" ]]; then + (command -v zstd >/dev/null 2>&1 && zstd -d -f -o "$HOME/observer" "$GITHUB_WORKSPACE/observer.zst" && chmod +x "$HOME/observer") || true +fi + +export PATH=/bin:/usr/bin +HOST=$(hostname -i) +DOWNLOAD_DIR=$HOME/seekdb/tools/deploy + +function prepare_config { + if [[ "$MINI" == "1" ]] || ([[ "$MINI" == "-1" ]] && [[ -f $HOME/seekdb/tools/deploy/enable_mini_mode ]]) + then + [[ -f $HOME/seekdb/tools/deploy/enable_mini_mode ]] && MINI_SIZE=$(cat $HOME/seekdb/tools/deploy/enable_mini_mode) + [[ $MINI_SIZE =~ ^[0-9]+G ]] || MINI_SIZE="8G" + MINI_CONFIG_ITEM="ObCfg.init_config['memory_limit']='$MINI_SIZE'" + fi + + + cd $HOME/seekdb/tools/deploy + if [ "$WITH_PROXY" ] && [ "$WITH_PROXY" != "0" ] + then + if [ "$CLUSTER_SPEC" == '2x1' ] + then + SPEC="[$HOST,proxy@$HOST]@zone1 [$HOST]@zone2" + else + SPEC="[$HOST,proxy@$HOST]@zone1" + fi + else + if [[ "$CLUSTER_SPEC" == '2x1' ]] || [[ "$SLAVE" == "1" ]] + then + SPEC="[$HOST]@zone1 [$HOST]@zone2" + else + SPEC="[$HOST]@zone1" + fi + fi + # farm中由于机器变小了，需要设置下 + CPU_COUNT_CONFIG="ObCfg.init_config['cpu_count']='24'" + cat >config7.py < test.fifo 2>&1 + elif [[ "$SLAVE" == "1" ]] + then + cat test.fifo | tee result.out & ./hap.py $USER.obs1.mysqltest collect_all slices=$SLICES slice_idx=$SLICE_IDX $slb `echo $ARGV` 1> test.fifo 2>&1 + else + cat test.fifo | tee result.out & ./hap.py $USER.mysqltest collect_all slices=$SLICES slice_idx=$SLICE_IDX $slb `echo $ARGV` 1> test.fifo 2>&1 + fi + + # check if there is error + if [ "$?" == "0" ] + then + grep -E 'PASSED' $HOME/seekdb/tools/deploy/result.out > /dev/null && ! grep -E "FAIL LST" $HOME/seekdb/tools/deploy/result.out > /dev/null || return 1 + else + return 1 + fi +} + +function collect_log { + cd $HOME/seekdb/tools/deploy || return 1 + if [ -d "collected_log" ] + then + mv mysql_test/var/log/* collected_log + mv $HOME/$USER.*/core[.-]* collected_log + mv collected_log collected_log_$SLICE_IDX || return 1 + tar cvfz $HOME/collected_log_$SLICE_IDX.tar.gz collected_log_$SLICE_IDX + fi +} + +export -f run_mysqltest + +function obd_prepare_obd { + if [[ -f $HOME/seekdb/deps/init/dep_create.sh ]] + then + if [[ "$IS_CE" == "1" ]] + then + cd $HOME/seekdb && ./build.sh init --ce || return 3 + else + cd $HOME/seekdb && ./build.sh init || return 3 + fi + else + if grep 'dep_create.sh' $HOME/seekdb/build.sh + then + cd $HOME/seekdb/deps/3rd && bash dep_create.sh all || return 4 + else + cd $HOME/seekdb && ./build.sh init || return 3 + fi + fi + + $obd devmode enable + $obd env set OBD_DEPLOY_BASE_DIR $HOME/seekdb/tools/deploy + $obd --version + if [[ "$IS_CE" == "1" && ! -f $OBD_HOME/.obd/mirror/remote/taobao.repo ]] + then + mkdir -p $OBD_HOME/.obd/mirror/remote && echo -e "[taobao]\nname=taobao\nbaseurl=http://yum.tbsite.net/taobao/7/x86_64/current\nenabled=1\n\n[taobao-test]\nname=taobao-test\nbaseurl=http://yum.tbsite.net/taobao/7/x86_64/test\nenabled=0" > $OBD_HOME/.obd/mirror/remote/taobao.repo + fi +} + +export -f obd_prepare_obd + +function obd_prepare_global { + export LANG=en_US.UTF-8 + HOST=$(hostname -i) + DOWNLOAD_DIR=$HOME/downloads + export SLOT_ID="${SLOT_ID:-0}" + PORT_NUM=$((5000 + SLOT_ID * 100)) + + # 根据传入的observer binary判断是否开源版 + if [[ `$HOME/observer -V 2>&1 | grep -E '(OceanBase CE|OceanBase_CE)'` ]] + then + COMPONENT="seekdb" + export IS_CE=1 + else + COMPONENT="seekdb" + export IS_CE=0 + fi + # 根据build.sh中的内容判断依赖安装路径 + if grep 'dep_create.sh' $HOME/seekdb/build.sh + then + DEP_PATH=$HOME/seekdb/deps/3rd + else + DEP_PATH=$HOME/seekdb/rpm/.dep_create/var + fi + + if [[ -f "$HOME/seekdb/tools/deploy/mysqltest_config.yaml" ]] + then + export WITH_MYSQLTEST_CONFIG_YAML=1 + else + export WITH_MYSQLTEST_CONFIG_YAML=0 + fi + + ob_name=obcluster$SLOT_ID + app_name=$ob_name.$USER.$HOST + ocp_config_server='http://ocp-cfg.alibaba.net:8080/services?User_ID=alibaba&UID=test' + cfg_url=${ocp_config_server} + + export obd=$DEP_PATH/usr/bin/obd + export OBD_HOME=$HOME + export OBD_INSTALL_PRE=$DEP_PATH + export DATA_PATH=$HOME/data + [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh + +} + +function obd_prepare_config { + + MINI_SIZE="10G" + if [[ "$MINI" == "1" ]] || ([[ "$MINI" == "-1" ]] && [[ -f $HOME/seekdb/tools/deploy/enable_mini_mode ]]) + then + [[ -f $HOME/seekdb/tools/deploy/enable_mini_mode ]] && MINI_SIZE=$(cat $HOME/seekdb/tools/deploy/enable_mini_mode) + [[ $MINI_SIZE =~ ^[0-9]+G ]] || MINI_SIZE="8G" + fi + if [[ "$CLUSTER_SPEC" == '2x1' ]] || [[ "$SLAVE" == "1" ]] + then + mysql_port=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1` + rpc_port1=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1` + mysql_port2=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1` + rpc_port2=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1` + SERVERS=$(cat <<-EOF + servers: + - name: server1 + ip: 127.0.0.1 + - name: server2 + ip: 127.0.0.1 + server1: + mysql_port: $mysql_port + rpc_port: $rpc_port1 + home_path: $DATA_PATH/observer1 + zone: zone1 + server2: + mysql_port: $mysql_port2 + rpc_port: $rpc_port2 + home_path: $DATA_PATH/observer2 + zone: zone2 +EOF +) + else + mysql_port=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1` + rpc_port=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1` + SERVERS=$(cat <<-EOF + servers: + - name: server1 + ip: 127.0.0.1 + server1: + mysql_port: $mysql_port + rpc_port: $rpc_port + home_path: $DATA_PATH/observer1 + zone: zone1 +EOF +) + fi + export MYSQL_PORT=$mysql_port + + if [[ "$MINI" == "1" && -f $HOME/seekdb/tools/deploy/obd/core-test.yaml.template ]] + then + conf=$(cat $HOME/seekdb/tools/deploy/obd/core-test.yaml.template) + else + conf=$(cat $HOME/seekdb/tools/deploy/obd/config.yaml.template) + fi + # cgroup support + if [[ -f $HOME/seekdb/tools/deploy/obd/.use_cgroup || $(echo " $ARGV " | grep " use-cgroup ") ]] + then + CGROUP_CONFIG="cgroup_dir: /sys/fs/cgroup/cpu/$USER" + fi + extra_conf="""$MYSQLRTEST_ARGS + $CGROUP_CONFIG +""" + conf=${conf//'{{%% PROXY_CONF %%}}'/"$extra_conf"} + conf=${conf//'{{%% DEPLOY_PATH %%}}'/"$HOME/seekdb/tools/deploy"} + conf=${conf//'{{%% COMPONENT %%}}'/$COMPONENT} + conf=${conf//'{{%% SERVERS %%}}'/$SERVERS} + conf=${conf//'{{%% TAG %%}}'/'latest'} + conf=${conf//'{{%% MINI_SIZE %%}}'/$MINI_SIZE} + conf=${conf//'{{%% OBCONFIG_URL %%}}'/$cfg_url} + conf=${conf//'{{%% APPNAME %%}}'/$app_name} + conf=${conf//'{{%% EXTRA_PARAM %%}}'/$MYSQLRTEST_ARGS} + if [[ "$IS_CE" == "1" ]] + then + conf=$(echo "$conf" | sed 's/oceanbase:/oceanbase\-ce:/g' | sed 's/\- oceanbase$/- oceanbase-ce/g') + fi + echo "$conf" > $HOME/seekdb/tools/deploy/config.yaml + + if [[ $WITH_SHARED_STORAGE == "1" ]] + then + # shared_storage 需要在global修改相关的mode + share_storage_workdir="$(date +%s)-$(uuidgen)" + first_global_line_count=`sed -n '/global:/=' $HOME/seekdb/tools/deploy/config.yaml | head -n 1` + set +x + sed -i "${first_global_line_count}a\ mode: 'shared_storage'\n\ + shared_storage: \"oss://oss-436751-0701-all-test/$FARM2_RUN_USER/$share_storage_workdir?host=oss-cn-hangzhou.aliyuncs.com&access_id=${SENSITIVE_TEST_OSS_ID_FOR_OBJECT_STORAGE_FOR_TOTAL}&access_key=${SENSITIVE_TEST_OSS_KEY_FOR_OBJECT_STORAGE_FOR_TOTAL}&max_iops=0&max_bandwidth=0B&scope=region\"" $HOME/seekdb/tools/deploy/config.yaml + sed -i "s/datafile_size: '20G'/datafile_size: '200G'/g" $HOME/seekdb/tools/deploy/config.yaml + set -x + fi + + if [[ $SHARED_STORAGE_MODE == "1" ]] + then + # shared_storage 需要在global修改相关的mode + MINIO_IP="100.88.99.213" + if [[ $TEST_NAME == "farm2" ]] + then + MINIO_IP="100.88.99.213" + fi + share_storage_workdir="$(date +%s)-$(uuidgen)" + first_global_line_count=`sed -n '/global:/=' $HOME/seekdb/tools/deploy/config.yaml | head -n 1` + sed -i "${first_global_line_count}a\ mode: 'shared_storage'\n\ + shared_storage: \"s3://farm-test/mysqltest/$share_storage_workdir/clog_and_data?host=http://$MINIO_IP:9000&access_id=minioadmin&access_key=minioadmin&s3_region=us-east-1&max_iops=10000&max_bandwidth=1GB&scope=region\"" $HOME/seekdb/tools/deploy/config.yaml + fi + + if [[ ! $WITH_SHARED_STORAGE == "1" ]] + then + cat $HOME/seekdb/tools/deploy/config.yaml + fi +} + +function obd_prepare_bin { + cd $HOME + mkdir -p $DOWNLOAD_DIR/{bin,etc,admin} || return 1 + + if [ ! -x observer ] + then + ( + cd $DOWNLOAD_DIR/bin && + wget http://6.12.233.4:5000/observer -O observer && + chmod +x observer + ) || return 1 + else + cp observer $DOWNLOAD_DIR/bin/ || return 2 + fi + + if [[ "$COMPONENT" == "seekdb" ]] && [[ -x $DOWNLOAD_DIR/bin/observer ]] + then + cp $DOWNLOAD_DIR/bin/observer $DOWNLOAD_DIR/bin/seekdb && chmod +x $DOWNLOAD_DIR/bin/seekdb + fi + + + cd $HOME/seekdb/tools/deploy + [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh + [[ -x $HOME/seekdb/tools/deploy/obd.sh ]] && ./obd.sh prepare + cp $DEP_PATH/u01/obclient/bin/obclient ./obclient && chmod 777 obclient + cp $DEP_PATH/u01/obclient/bin/mysqltest ./mysqltest && chmod 777 mysqltest + + if [ "$WITH_DEPS" ] && [ "$WITH_DEPS" != "0" ] + then + cd $HOME/seekdb/tools/deploy && [[ -f copy.sh ]] && sh copy.sh + fi + if [[ -f "$HOME/seekdb/tools/deploy/obd/.observer_obd_plugin_version" ]] + then + obs_version=$(cat $HOME/seekdb/tools/deploy/obd/.observer_obd_plugin_version) + else + $DOWNLOAD_DIR/bin/observer -V + obs_version=`$DOWNLOAD_DIR/bin/observer -V 2>&1 | grep -E "(observer|seekdb) \(OceanBase([ \_]CE|[ \_]Lite|[ \_]SeekDB|[ \_]seekdb)? [0-9]" | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?' | head -1` + fi + + mkdir $HOME/seekdb/tools/deploy/admin + if [[ -d $HOME/seekdb/src/share/inner_table/sys_package ]] + then + cp $HOME/seekdb/src/share/inner_table/sys_package/*.sql $HOME/seekdb/tools/deploy/admin/ + fi + + $obd mirror create -n $COMPONENT -p $DOWNLOAD_DIR -t latest -V $obs_version || return 2 + +} + +function obd_init_cluster { + retries=$reboot_retries + while (( $retries > 0 )) + do + if [[ "$retries" == "$reboot_retries" ]] + then + [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh + $obd cluster deploy $ob_name -c $HOME/seekdb/tools/deploy/config.yaml -f && $obd cluster start $ob_name -f && $obd cluster display $ob_name + else + [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh + $obd cluster redeploy $ob_name -f + fi + retries=`expr $retries - 1` + ./obclient -h 127.1 -P $MYSQL_PORT -u root -A -e "alter system set_tp tp_no = 509, error_code = 4016, frequency = 1;" + ret=$? + if [[ $ret == 0 ]] + then + init_files=('init.sql' 'init_mini.sql' 'init_for_ce.sql') + for init_file in ${init_files[*]} + do + if [[ -f $HOME/seekdb/tools/deploy/$init_file ]] + then + if ! grep "alter system set_tp tp_no = 509, error_code = 4016, frequency = 1;" $HOME/seekdb/tools/deploy/$init_file + then + echo -e "\nalter system set_tp tp_no = 509, error_code = 4016, frequency = 1;" >> $HOME/seekdb/tools/deploy/$init_file + fi + fi + done + fi + $obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient --init-only --init-sql-dir=$HOME/seekdb/tools/deploy $INIT_FLIES --log-dir=$HOME/init_case $EXTRA_ARGS -v > ./init_sql_log && init_seccess=1 && break + done +} + +function is_case_selector_arg() +{ + local arg + arg=$1 + case_selector_patterns=('test-set=*' 'testset=*' 'suite=*' 'tag=*' 'regress_suite=*' 'all' 'psmall') + for p in ${case_selector_patterns[@]} + do + [[ "$(echo $arg | grep -Eo $p)" != "" ]] && return 0 + done + return 1 +} + +function obd_run_mysqltest { + set -x + obd_prepare_global + + cd $HOME/seekdb/tools/deploy + [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh + if [[ "$SLB" != "" ]] && [[ "$EXECID" != "" ]] + then + SCHE_ARGS="--slb-host=$SLB --exec-id=$EXECID " + else + SCHE_ARGS="--slices=$SLICES --slice-idx=$SLICE_IDX " + fi + + + if [[ "$MINI" == "1" && -f core-test.init_mini.sql ]] + then + INIT_FLIES="--init-sql-files=core-test.init_mini.sql,init_user.sql|root@mysql|test" + [ -f init_user_oracle.sql ] && INIT_FLIES="${INIT_FLIES},init_user_oracle.sql|SYS@oracle|SYS" + else + if [[ "$IS_CE" == "1" && -f $HOME/seekdb/.ce ]] + then + INIT_FLIES="--init-sql-files=init_mini.sql,init_user.sql|root@mysql|test " + fi + fi + CLUSTER_MODE="c" + if [[ "$SLAVE" == "1" ]] + then + SERVER_ARGS="--test-server=server2" + CLUSTER_MODE="slave" + else + SERVER_ARGS="" + fi + if [ "$WITH_PROXY" ] && [ "$WITH_PROXY" != "0" ] + then + CLUSTER_MODE="proxy" + fi + reboot_retries=2 + if [[ "$WITH_MYSQLTEST_CONFIG_YAML" == "1" ]] + then + EXTRA_ARGS="--test-mode=$CLUSTER_MODE " + else + EXTRA_ARGS="--cluster-mode=$CLUSTER_MODE " + fi + if [[ -f $HOME/seekdb/tools/deploy/obd/.fast-reboot ]] + then + EXTRA_ARGS="${EXTRA_ARGS}--fast-reboot " + fi + + if [[ "$IS_CE" == "1" ]] + then + EXTRA_ARGS="${EXTRA_ARGS}--mode=mysql " + fi + + EXTRA_ARGS_WITHOUT_CASE=$EXTRA_ARGS + + if [[ "$ARGV" != "" ]] + then + for arg in $ARGV + do + [[ "${arg%%=*}" == "testset" ]] && arg="${arg/testset=/test-set=}" + [[ "${arg%%=*}" == "reboot-timeout" ]] && has_reboot_timeout=1 + [[ "${arg%%=*}" == "reboot-retries" ]] && reboot_retries=${arg#*=} + [[ "${arg%%=*}" == "disable-collect" ]] && disable_collect=1 && continue + [[ "${arg%%=*}" == "use-cgroup" ]] && continue + EXTRA_ARGS="${EXTRA_ARGS}--${arg} " + if ! is_case_selector_arg "${arg}" + then + EXTRA_ARGS_WITHOUT_CASE="${EXTRA_ARGS_WITHOUT_CASE}--${arg} " + fi + done + fi + + if [[ "$has_reboot_timeout" != "1" ]] + then + REBOOT_TIMEOUT="--reboot-timeout=600" + fi + if [[ "$disable_collect" != "1" ]] + then + COLLECT_ARG="--collect-all" + if [[ "$WITH_MYSQLTEST_CONFIG_YAML" == "0" ]] && [ "$WITH_PROXY" ] && [ "$WITH_PROXY" != "0" ] && [[ -f "$HOME/seekdb/tools/deploy/obd/.collect_proxy_log" ]] + then + COLLECT_ARG="$COLLECT_ARG --collect-components=$COMPONENT,obproxy" + fi + fi + obd_init_cluster + if [[ "$init_seccess" != "1" ]] + then + cat ./init_sql_log + echo "Failed to init sql, see more log at ./collected_log/_test_init" + mkdir -p $HOME/collected_log/_test_init/ + [[ -d "$DATA_PATH/observer1/log" ]] && mkdir -p $HOME/collected_log/_test_init/observer1 && mv $DATA_PATH/observer1/log/* $HOME/collected_log/_test_init/observer1 + [[ -d "$DATA_PATH/observer2/log" ]] && mkdir -p $HOME/collected_log/_test_init/observer2 && mv $DATA_PATH/observer2/log/* $HOME/collected_log/_test_init/observer2 + [[ -d "$DATA_PATH/obproxy/log" ]] && mkdir -p $HOME/collected_log/_test_init/obproxy && mv $DATA_PATH/obproxy/log/* $HOME/collected_log/_test_init/obproxy + exit 1 + fi + if [[ "$SPECIAL_RUN" == "1" ]] + then + ret=255 + result_dir=$HOME/record + mkdir -p $result_dir + RECORD_ARGS="--record-dir=$result_dir --record --log-dir=./var/rec_log" + RESULT_ARGS="--result-dir=$result_dir --result-file-suffix=.record --log-dir=./var/log" + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient --init-sql-dir=$HOME/seekdb/tools/deploy --special-run --sort-result $COLLECT_ARG $REBOOT_TIMEOUT $VERBOSE_ARG" + # $mysqltest_cmd --all $INIT_FLIES $SCHE_ARGS $RECORD_ARGS 2>&1 > record.out + $mysqltest_cmd $INIT_FLIES $SCHE_ARGS $RECORD_ARGS $EXTRA_ARGS 2>&1 | tee record.out && ( exit ${PIPESTATUS[0]}) + rec_ret=$? + cat record.out | grep -E '\|\s+(PASSED|FAILED)|s+\|' > record_result + TEST_CASES=`cat record_result | grep PASSED | awk '{print \$2}' | tr '\n' ',' | head -c -1` + ret=0 + if [[ "$TEST_CASES" != "" ]] + then + if [[ "$SP_CHANGE" != "" ]] + then + arr=(${SP_CHANGE//,/ }) + for line in ${arr[@]} + do + MYSQLRTEST_ARGS=$MYSQLRTEST_ARGS$'\n'" ${line%%=*}: ${line##*=}" + done + obd_prepare_config + echo "config after change:" + cat $HOME/seekdb/tools/deploy/config.yaml + $obd cluster destroy $ob_name && $obd cluster deploy $ob_name -c $HOME/seekdb/tools/deploy/config.yaml && $obd cluster start $ob_name -f && $obd cluster display $ob_name + else + $obd cluster redeploy $ob_name + fi + obd_init_cluster + $mysqltest_cmd $INIT_FLIES --test-set=$TEST_CASES --sp-hint="$SP_HINT" $RESULT_ARGS $EXTRA_ARGS_WITHOUT_CASE 2>&1 | tee compare.out && ( exit ${PIPESTATUS[0]}) + ret=$? + else + echo 'No case succeed?!' + fi + if [[ $JOBNAME == 'mysqltest_opensource' ]] + then + submarker="_opensource" + else + submarker="" + fi + mv record.out $HOME/mysqltest${submarker}_record_output.$SLICE_IDX + mv compare.out $HOME/mysqltest${submarker}_compare_output${JOB_SUFFIX}.$SLICE_IDX + echo "finish!" + [[ "$rec_ret" != "0" ]] && return $rec_ret + [[ "$ret" != "0" ]] && return $ret + return 0 + elif [[ $CROSS_VALIDATION == "1" ]] + then + ret=255 + # 先checkout commit + # todo: 修改为把tools deploy下的init sql都带入过去 + mkdir $HOME/tmp_init_sql + mkdir $HOME/obd_tmp + find $HOME/seekdb/tools/deploy -name 'init_*.sql' | xargs -i cp {} $HOME/tmp_init_sql + cp $HOME/seekdb/tools/deploy/obd/* $HOME/obd_tmp + cd $HOME && tar -cvf tmp_init_sql.tar.gz $HOME/tmp_init_sql + cd $HOME && tar -cvf origin_obd_file.tar.gz $HOME/obd_tmp + + cd $HOME/seekdb + # 针对多个分支进行处理 + ret=0 + for commit in $CROSS_VALIDATION_COMMIT + do + git checkout -f $commit || ret=2 + # 然后重新针对obd 进行重新构建路径 + cd $HOME && tar -xvf tmp_init_sql.tar.gz && cp tmp_init_sql/* $HOME/seekdb/tools/deploy + cd $HOME && tar -xvf origin_obd_file.tar.gz && cp obd_tmp/* $HOME/seekdb/tools/deploy/obd + + if [[ -f $HOME/seekdb/deps/init/dep_create.sh ]] + then + if [[ "$IS_CE" == "1" ]] + then + cd $HOME/seekdb && ./build.sh init --ce || ret=3 + else + cd $HOME/seekdb && ./build.sh init || ret=3 + fi + else + if grep 'dep_create.sh' $HOME/seekdb/build.sh + then + cd $HOME/seekdb/deps/3rd && bash dep_create.sh all || ret=4 + else + cd $HOME/seekdb && ./build.sh init || ret=4 + fi + fi + # 根据build.sh中的内容判断依赖安装路径 + if grep 'dep_create.sh' $HOME/seekdb/build.sh + then + DEP_PATH=$HOME/seekdb/deps/3rd + else + DEP_PATH=$HOME/seekdb/rpm/.dep_create/var + fi + extra_ignore_cmd="" + if [[ `$DEP_PATH/u01/obclient/bin/mysqltest --help | grep "cmp-ignore-explain"` ]] + then + extra_ignore_cmd=" --ignore-explain=enable" + fi + + if [[ `$DEP_PATH/u01/obclient/bin/mysqltest --help | grep "disable-explain"` ]] + then + extra_ignore_cmd=" $extra_ignore_cmd --special-run" + fi + + cd $HOME/seekdb/tools/deploy + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=$DEP_PATH/u01/obclient/bin/mysqltest --obclient-bin=$DEP_PATH/u01/obclient/bin/obclient $COLLECT_ARG --init-sql-dir=$HOME/seekdb/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS_WITHOUT_CASE --suite=cross_validation $extra_ignore_cmd" + $mysqltest_cmd $INIT_FLIES $SCHE_ARGS 2>&1 | tee compare.out && ( exit ${PIPESTATUS[0]}) + test_suite_ret=$? + + FILTER=`ls -ltr $HOME/seekdb/tools/deploy/mysql_test/test_suite/cross_validation/t | grep -v total | awk '{gsub(".test","",$9);print "cross_validation."$9}' | tr '\n' ',' | head -c -1` + if [[ $FILTER == "" ]] + then + echo "without testsuite cases" + fi + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=$DEP_PATH/u01/obclient/bin/mysqltest --obclient-bin=$DEP_PATH/u01/obclient/bin/obclient $COLLECT_ARG --init-sql-dir=$HOME/seekdb/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS_WITHOUT_CASE --exclude=$FILTER --tag=cross_validation $extra_ignore_cmd" + $mysqltest_cmd $INIT_FLIES $SCHE_ARGS 2>&1 | tee compare.out && ( exit ${PIPESTATUS[0]}) + tag_ret=$? + [[ $test_suite_ret = 0 && $tag_ret = 0 ]] && current_ret=0 || current=1 + [[ $ret = 0 && $current_ret = 0 ]] && ret=0 || ret=1 + done + + if [[ $JOBNAME == 'mysqltest_opensource' ]] + then + submarker="_opensource" + else + submarker="" + fi + + mv compare.out $HOME/mysqltest${submarker}_compare_output.$SLICE_IDX + echo "finish!" + return $ret + elif [[ $WITH_SHARED_STORAGE == "1" ]] + then + # 把sensitive_test目录下的测试集合移动到tools/deploy/mysql_test + # 并且创建一个新的目录用来表示单独的测试 + if [[ ! -d $HOME/seekdb/sensitive_test/mysql_test ]] + then + return 0 + fi + need_mv_dirs=`ls -1 $HOME/seekdb/sensitive_test/mysql_test/test_suite|xargs` + run_suites="" + for need_mv_dir in $need_mv_dirs + do + new_dir=$HOME/seekdb/tools/deploy/mysql_test/test_suite/shared_storage__${need_mv_dir} + run_suites="${run_suites}shared_storage__${need_mv_dir}," + mkdir -p $new_dir + cp -r $HOME/seekdb/sensitive_test/mysql_test/test_suite/$need_mv_dir/* $new_dir + done + run_suites=`echo $run_suites | awk '{print substr($0, 1, length($0)-1)}'` + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient $COLLECT_ARG --init-sql-dir=$HOME/seekdb/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS_WITHOUT_CASE --suite=$run_suites" + $mysqltest_cmd $INIT_FLIES $SCHE_ARGS 2>&1 | tee compare.out && ( exit ${PIPESTATUS[0]}) + else + if [[ -f $HOME/seekdb/tools/deploy/error_log_filter.json && ( $BRANCH == 'master' || $BRANCH == "4_2_x_release" ) && "$FROM_FARM" == '1' ]] + then + echo "python $HOME/common/analyse_the_observer_log.py collect -p ${DATA_PATH}/observer1/log -p ${DATA_PATH}/observer2/log -i $GID -j ${JOBNAME}.${SLICE_IDX} -o $HOME -F $HOME/seekdb/tools/deploy/error_log_filter.json" > $HOME/seekdb/tools/deploy/analyse_the_observer_log.sh + chmod a+x $HOME/seekdb/tools/deploy/analyse_the_observer_log.sh + RUN_EXTRA_CHECK_CMD='--extra-script-after-case="$HOME/seekdb/tools/deploy/./analyse_the_observer_log.sh"' + fi + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient $COLLECT_ARG --init-sql-dir=$HOME/seekdb/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS" + $mysqltest_cmd $RUN_EXTRA_CHECK_CMD $INIT_FLIES $SCHE_ARGS 2>&1 | tee compare.out && ( exit ${PIPESTATUS[0]}) + ret=$? + if [[ $JOBNAME == 'mysqltest_opensource' ]] + then + submarker="_opensource" + else + submarker="" + fi + mv compare.out $HOME/mysqltest${submarker}_compare_output.$SLICE_IDX + echo "finish!" + return $ret + fi +} + +function obd_prepare_env { + obd_prepare_global + obd_prepare_obd + obd_prepare_config + obd_prepare_bin +} + +function obd_collect_log { + + echo "collect log" + cd $HOME/ + mkdir -p collected_log + mkdir -p collected_log/obd_log + mkdir -p collected_log/mysqltest_log + mkdir -p collected_log/mysqltest_rec_log + find $DATA_PATH -name 'core[.-]*' | xargs -i cp {} collected_log + [[ "$OBD_HOME" != "" ]] && mv $OBD_HOME/.obd/log/* collected_log/obd_log/ + set +x + if [[ $WITH_SHARED_STORAGE == "1" ]] + then + sed -i "s/${SENSITIVE_TEST_OSS_KEY_FOR_OBJECT_STORAGE_FOR_TOTAL}//" collected_log/obd_log/obd + fi + set -x + mv seekdb/tools/deploy/var/log/* collected_log/mysqltest_log/ + mv seekdb/tools/deploy/var/rec_log/* collected_log/mysqltest_rec_log/ + mv collected_log collected_log_$SLICE_IDX + tar zcvf collected_log_${SLICE_IDX}${JOB_SUFFIX}.tar.gz collected_log_$SLICE_IDX +} + +function collect_obd_case_log { + + echo "collect obd case log" + cd $HOME/ + if [[ $JOBNAME == 'mysqltest_opensource' ]] + then + submarker="_opensource" + else + submarker="" + fi + [[ "$OBD_HOME" != "" ]] && cat $OBD_HOME/.obd/log/* | grep '\[INFO\]' > mysqltest_cases_log${submarker}_$SLICE_IDX.output + +} + + +export -f obd_prepare_global +export -f obd_prepare_config +export -f obd_run_mysqltest +export -f obd_init_cluster +export -f is_case_selector_arg + +function run { + if [[ -f /root/tmp/.ssh/id_rsa ]] + then + mv /root/tmp/.ssh/id_rsa /root/.ssh/ + fi + if ([[ -f $HOME/seekdb/rpm/oceanbase.deps ]] && [[ "$(grep 'ob-deploy' $HOME/seekdb/rpm/oceanbase.deps )" != "" ]]) || + ([[ -f $HOME/seekdb/deps/3rd/oceanbase.el7.x86_64.deps ]] && [[ "$(grep 'ob-deploy' $HOME/seekdb/deps/3rd/oceanbase.el7.x86_64.deps )" != "" ]]) || + ([[ -f $HOME/seekdb/deps/init/oceanbase.el7.x86_64.deps ]] && [[ "$(grep 'ob-deploy' $HOME/seekdb/deps/init/oceanbase.el7.x86_64.deps )" != "" ]]) + then + if [[ $WITH_SHARED_STORAGE == "1" ]] + then + # 单独判断下是否有对应的case集合 + if [[ ! -d $HOME/seekdb/sensitive_test/mysql_test ]] + then + return 0 + fi + fi + timeout=18000 + [[ "$SPECIAL_RUN" == "1" ]] && timeout=72000 + obd_prepare_env && + timeout $timeout bash -c "obd_run_mysqltest" + test_ret=$? + error_log_ret=0 + if [[ -f $HOME/seekdb/tools/deploy/error_log_filter.json && $BRANCH == 'master' ]] + then + # notice the core + python $HOME/common/analyse_the_observer_core.py collect -p ${DATA_PATH} -j ${JOBNAME}.${SLICE_IDX} -o $HOME + error_log_ret=0 + collect_obd_case_log + fi + if [[ $test_ret -ne 0 || $error_log_ret -ne 0 ]] + then + obd_collect_log + fi + return `[[ $test_ret = 0 && $error_log_ret = 0 ]]` + else + prepare_env && + timeout 18000 bash -c "run_mysqltest" + test_ret=$? + error_log_ret=0 + if [[ -f $HOME/seekdb/tools/deploy/error_log_filter.json && ( $BRANCH == 'master' || $BRANCH == "4_2_x_release" ) ]] + then + python $HOME/common/analyse_the_observer_log.py collect -p ${DATA_PATH}/observer1/log -p ${DATA_PATH}/observer2/log -i $GID -j ${JOBNAME}.${SLICE_IDX} -o $HOME -F $HOME/seekdb/tools/deploy/error_log_filter.json + error_log_ret=0 + fi + if [[ $test_ret -ne 0 || $error_log_ret -ne 0 ]] + then + collect_log + fi + return `[[ $test_ret = 0 && $error_log_ret = 0 ]]` + fi + +} + +run "$@" \ No newline at end of file diff --git a/.github/workflows/buildbase/seekdb.yml b/.github/workflows/buildbase/seekdb.yml new file mode 100644 index 000000000..6aa3e447e --- /dev/null +++ b/.github/workflows/buildbase/seekdb.yml @@ -0,0 +1,229 @@ +# SeekDB CI - 执行下沉版（Native） +# setup 准备 observer → 4 个并行 mysqltest job（各独立容器）→ collect 合并结果 +name: SeekDB CI + +on: + pull_request: + branches: [master, develop] + types: [opened, reopened, synchronize, ready_for_review] + paths-ignore: + - 'docs/**' + - '.github/ISSUE_TEMPLATE/**' + - '.github/pull_request_template.md' + - 'README.md' + - 'README_CN.md' + - 'CONTRIBUTING.md' + schedule: + - cron: '30 9 * * *' + workflow_dispatch: + +env: + MYSQLTEST_SLICES: "4" + cache-name: cache-deps + SEEKDB_TASK_DIR: ${{ github.workspace }}/seekdb_build/${{ github.run_id }} + SEEKDB_RESULT_NFS_ROOT: /data/storage/jenkins_on_k8s/results/seekdb + +jobs: + seekdb-setup: + name: setup + if: ${{ github.repository_owner == 'oceanbase' && (github.event_name != 'pull_request' || !github.event.pull_request.draft) }} + runs-on: arc-runner-k8s-ci-org + container: + image: harbor.oceanbase-dev.com/obrde-test/ob-actions-runner-worker:0.0.3 + steps: + - name: Checkout + uses: actions/checkout@v4 + + + - name: Cache deps + uses: actions/cache@v4 + with: + key: ${{ runner.os }}-build-${{ env.cache-name }}-el9.x86_64-${{ hashFiles('deps/init/oceanbase.el9.x86_64.deps') }} + path: deps/3rd + + - name: Prepare (jobargs, run_jobs) + shell: bash + run: | + chmod +x .github/script/seekdb/prepare.sh + export GITHUB_RUN_ID="${{ github.run_id }}" + export GITHUB_WORKSPACE="${{ github.workspace }}" + export MYSQLTEST_SLICES="${{ env.MYSQLTEST_SLICES }}" + export SEEKDB_TASK_DIR="${{ env.SEEKDB_TASK_DIR }}" + .github/script/seekdb/prepare.sh + - name: Setup ccache + uses: hendrikmuhs/ccache-action@v1.2.11 + with: + max-size: 800M + save: ${{ github.event_name == 'push' }} + key: seekdb + + - name: Compile + id: compile + shell: bash + run: | + chmod +x .github/script/seekdb/compile.sh + export GITHUB_RUN_ID="${{ github.run_id }}" + WS="${{ github.workspace }}" + if [[ ! -f "$WS/build.sh" ]] && [[ -f "$(pwd)/build.sh" ]]; then + export GITHUB_WORKSPACE="$(pwd)" + else + export GITHUB_WORKSPACE="$WS" + fi + export SEEKDB_TASK_DIR="${{ env.SEEKDB_TASK_DIR }}" + export RELEASE_MODE="${{ vars.RELEASE_MODE || '' }}" + export FORWARDING_HOST="${{ vars.FORWARDING_HOST || '' }}" + export PATH="$GITHUB_WORKSPACE/deps/3rd/usr/local/oceanbase/devtools/bin:$PATH" + .github/script/seekdb/compile.sh + - name: Prepare observer + shell: bash + run: | + if [[ ! -x observer ]]; then + for base in build_debug build_release build; do + for cand in "$base/observer" "$base/src/observer/observer"; do + if [[ -f "$cand" ]] && [[ -x "$cand" ]]; then + cp -f "$cand" observer && chmod +x observer + echo "observer copied from $cand" + break 2 + fi + done + done + fi + if [[ ! -x observer ]]; then echo "observer missing or not executable"; exit 1; fi + echo "observer size: $(stat -c%s observer 2>/dev/null || stat -f%z observer 2>/dev/null) bytes" + - name: Compress observer + shell: bash + run: | + zstd -f -q -T0 -o observer.zst observer + echo "observer.zst size: $(stat -c%s observer.zst 2>/dev/null || stat -f%z observer.zst 2>/dev/null) bytes" + - name: Persist observer to NFS + shell: bash + run: | + DEST_DIR="${{ env.SEEKDB_RESULT_NFS_ROOT }}/${{ github.run_id }}/attempt-${{ github.run_attempt }}/setup" + mkdir -p "$DEST_DIR" + cp -f observer.zst "$DEST_DIR/" + echo "observer.zst saved to $DEST_DIR/observer.zst" + seekdb-mysqltest: + name: mysqltest-slice-${{ matrix.slice }} + needs: seekdb-setup + runs-on: arc-runner-k8s-ci-org + container: + image: harbor.oceanbase-dev.com/obrde-test/ob-actions-runner-worker:0.0.3 + strategy: + fail-fast: false + matrix: + slice: [0, 1, 2, 3] + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Place observer (from NFS) + shell: bash + run: | + OBSERVER_ZST="${{ env.SEEKDB_RESULT_NFS_ROOT }}/${{ github.run_id }}/attempt-${{ github.run_attempt }}/setup/observer.zst" + if [[ -f "$OBSERVER_ZST" ]]; then + zstd -d -f -q -o observer "$OBSERVER_ZST" && chmod +x observer + else + echo "observer not found on NFS" + fi + ls -la observer + - name: Run mysqltest slice + shell: bash + run: | + chmod +x .github/script/seekdb/mysqltest_slice.sh + export GITHUB_RUN_ID="${{ github.run_id }}" GITHUB_WORKSPACE="${{ github.workspace }}" + export SEEKDB_TASK_DIR="${{ github.workspace }}/seekdb_build/${{ github.run_id }}/slice_${{ matrix.slice }}" + export SLICE_IDX=${{ matrix.slice }} SLICES="${{ env.MYSQLTEST_SLICES }}" + export BRANCH="${{ github.ref_name }}" FORWARDING_HOST="${{ vars.FORWARDING_HOST || '' }}" + mkdir -p "$SEEKDB_TASK_DIR" + .github/script/seekdb/mysqltest_slice.sh + - name: Collect slice result + if: success() || failure() + shell: bash + run: | + chmod +x .github/script/seekdb/collect_result.sh + export GITHUB_RUN_ID="${{ github.run_id }}" GITHUB_WORKSPACE="${{ github.workspace }}" + export SEEKDB_TASK_DIR="${{ github.workspace }}/seekdb_build/${{ github.run_id }}/slice_${{ matrix.slice }}" + .github/script/seekdb/collect_result.sh seekdb_result.json + continue-on-error: true + + - name: Persist slice result to NFS + if: success() || failure() + shell: bash + run: | + DEST_DIR="${{ env.SEEKDB_RESULT_NFS_ROOT }}/${{ github.run_id }}/attempt-${{ github.run_attempt }}/slices/slice_${{ matrix.slice }}" + mkdir -p "$DEST_DIR" + if [[ -f seekdb_result.json ]]; then + cp -f seekdb_result.json "$DEST_DIR/" + else + echo "skip copying slice result: seekdb_result.json not found" + fi + if [[ -d "${{ github.workspace }}/seekdb_build/${{ github.run_id }}/slice_${{ matrix.slice }}" ]]; then + cp -a "${{ github.workspace }}/seekdb_build/${{ github.run_id }}/slice_${{ matrix.slice }}" "$DEST_DIR/seekdb_task_dir" + fi + continue-on-error: true + + # 失败时保存 collected_log 到 NFS 便于排查：obd_log、mysqltest_log、mysqltest_rec_log、core + - name: Persist collected_log to NFS (on failure) + if: failure() + shell: bash + run: | + SRC="seekdb_home_${{ matrix.slice }}/collected_log_${{ matrix.slice }}.tar.gz" + DEST_DIR="${{ env.SEEKDB_RESULT_NFS_ROOT }}/${{ github.run_id }}/attempt-${{ github.run_attempt }}/logs/slice_${{ matrix.slice }}" + mkdir -p "$DEST_DIR" + if [[ -f "$SRC" ]]; then + cp -f "$SRC" "$DEST_DIR/" + echo "collected log saved to $DEST_DIR/" + else + echo "skip copying collected log: $SRC not found" + fi + continue-on-error: true + + - name: Job summary (logs hint) + if: always() + run: | + if [[ -f "seekdb_home_${{ matrix.slice }}/collected_log_${{ matrix.slice }}.tar.gz" ]]; then + echo "### Slice ${{ matrix.slice }} collected_log saved" >> "$GITHUB_STEP_SUMMARY" + echo "Saved to \`${{ env.SEEKDB_RESULT_NFS_ROOT }}/${{ github.run_id }}/attempt-${{ github.run_attempt }}/logs/slice_${{ matrix.slice }}/collected_log_${{ matrix.slice }}.tar.gz\`. Contents: \`obd_log/\`, \`mysqltest_log/\`, \`mysqltest_rec_log/\`, core dumps." >> "$GITHUB_STEP_SUMMARY" + fi + seekdb-collect: + # 仅在 4 个 mysqltest slice 都跑完（成功或失败）时执行；若 mysqltest 被 skip 则不跑 + if: ${{ always() && needs.seekdb-mysqltest.result != 'skipped' }} + name: collect + needs: seekdb-mysqltest + runs-on: arc-runner-k8s-ci-org + container: + image: harbor.oceanbase-dev.com/obrde-test/ob-actions-runner-worker:0.0.3 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Collect slice results from NFS + shell: bash + run: | + SRC_ROOT="${{ env.SEEKDB_RESULT_NFS_ROOT }}/${{ github.run_id }}/attempt-${{ github.run_attempt }}/slices" + mkdir -p slice_results + if [[ -d "$SRC_ROOT" ]]; then + cp -a "$SRC_ROOT/." slice_results/ + else + echo "slice result directory not found on NFS: $SRC_ROOT" + fi + - name: Merge results and persist + shell: bash + run: | + RUN_ID="${{ github.run_id }}" + FAILED="" + SUCCESS="true" + while IFS= read -r f; do + [[ -f "$f" ]] || continue + grep -q '"success":false' "$f" 2>/dev/null && SUCCESS="false" + while IFS= read -r c; do + [[ -n "$c" ]] && FAILED="${FAILED}${FAILED:+,}\"${c//\"/\\\"}\"" + done < <(grep -oP '"failed_cases":\[\K[^\]]*' "$f" 2>/dev/null | tr ',' '\n' | sed 's/^"\|"$//g') + done < <(find slice_results -name 'seekdb_result.json' 2>/dev/null) + echo "{\"success\":$SUCCESS,\"run_id\":\"$RUN_ID\",\"failed_cases\":[${FAILED:-}]}" > seekdb_result.json + echo "=== SeekDB native result ===" + cat seekdb_result.json + DEST_DIR="${{ env.SEEKDB_RESULT_NFS_ROOT }}/${{ github.run_id }}/attempt-${{ github.run_attempt }}/collect" + mkdir -p "$DEST_DIR" + cp -f seekdb_result.json "$DEST_DIR/" + echo "seekdb_result.json saved to $DEST_DIR/seekdb_result.json" \ No newline at end of file From c48ccf4e20832bfcb8353b0586da7792b8d9b31e Mon Sep 17 00:00:00 2001 From: guoyan1996 <1028109394@qq.com> Date: Fri, 3 Apr 2026 17:47:18 +0800 Subject: [PATCH 03/11] delete --- .github/workflows/seekdb-farm2.yml | 40 ------------------------------ test/test_ci | 1 - 2 files changed, 41 deletions(-) delete mode 100644 .github/workflows/seekdb-farm2.yml delete mode 100644 test/test_ci diff --git a/.github/workflows/seekdb-farm2.yml b/.github/workflows/seekdb-farm2.yml deleted file mode 100644 index 28b0206b1..000000000 --- a/.github/workflows/seekdb-farm2.yml +++ /dev/null @@ -1,40 +0,0 @@ -# 示例：在 GitHub 上触发 SeekDB 测试并由 Farm2 执行，结果回写到 OSS 后本 workflow 轮询并回调 -# -# 使用方式： -# 1. 将本文件复制到 oceanbase/seekdb 的 .github/workflows/，并把 farm2-client 的 bin/seekdb_wait_result.py 复制到该仓库 .github/scripts/seekdb_wait_result.py -# 2. 确保 seekdb_robot_for_ob_for_retry.py 在服务器上常驻运行（轮询 GitHub 发现本 job 的 step in_progress 后发 Farm2 任务，结束写 OSS） -# 3. job 名必须为 seekdb3，且有一个 step 名以 "action by seekdb3" 开头 -# -name: SeekDB Farm2 - -on: - push: - branches: [ develop, master ] - pull_request: - branches: [ develop, master ] - workflow_dispatch: - -jobs: - seekdb3: - name: seekdb3 - runs-on: ubuntu-latest - steps: - - name: Checkout - uses: actions/checkout@v4 - - - name: action by seekdb3 - id: seekdb_farm2 - env: - GITHUB_RUN_ID: ${{ github.run_id }} - GITHUB_RUN_ATTEMPT: ${{ github.run_attempt }} - run: | - pip install -q requests - python3 .github/script/seekdb_wait_result.py --timeout 14400 --poll-interval 15 - continue-on-error: true - - - name: Farm2 result - if: always() - run: | - echo "task_id: ${{ steps.seekdb_farm2.outputs.task_id }}" - echo "success: ${{ steps.seekdb_farm2.outputs.success }}" - if [ "${{ steps.seekdb_farm2.outputs.success }}" != "true" ]; then exit 1; fi diff --git a/test/test_ci b/test/test_ci deleted file mode 100644 index c79d0df0e..000000000 --- a/test/test_ci +++ /dev/null @@ -1 +0,0 @@ -1111111111111testtttttt \ No newline at end of file From d17cea96fadcc555ec62c40ee970a7c2682626e0 Mon Sep 17 00:00:00 2001 From: guoyan1996 <1028109394@qq.com> Date: Fri, 3 Apr 2026 18:03:26 +0800 Subject: [PATCH 04/11] alter --- .github/workflows/{buildbase => }/seekdb.yml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{buildbase => }/seekdb.yml (100%) diff --git a/.github/workflows/buildbase/seekdb.yml b/.github/workflows/seekdb.yml similarity index 100% rename from .github/workflows/buildbase/seekdb.yml rename to .github/workflows/seekdb.yml From d40264187ff64ecba1ddb09e965cbb9956414cf0 Mon Sep 17 00:00:00 2001 From: guoyan1996 <1028109394@qq.com> Date: Fri, 3 Apr 2026 21:11:31 +0800 Subject: [PATCH 05/11] IS_CE=1 --- .../script/seekdb/scripts/mysqltest_for_seekdb.sh | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh b/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh index be0694524..82015dc4e 100644 --- a/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh +++ b/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh @@ -196,15 +196,9 @@ function obd_prepare_global { export SLOT_ID="${SLOT_ID:-0}" PORT_NUM=$((5000 + SLOT_ID * 100)) - # 根据传入的observer binary判断是否开源版 - if [[ `$HOME/observer -V 2>&1 | grep -E '(OceanBase CE|OceanBase_CE)'` ]] - then - COMPONENT="seekdb" - export IS_CE=1 - else - COMPONENT="seekdb" - export IS_CE=0 - fi + # SeekDB CI：固定按 CE 路径走 OBD/init（原先用 observer -V 判 CE，SeekDB 输出不含 CE 字样会被判成 0） + COMPONENT="seekdb" + export IS_CE=1 # 根据build.sh中的内容判断依赖安装路径 if grep 'dep_create.sh' $HOME/seekdb/build.sh then From 10b8700a849c31fe9566a828f7dddce7524ba67d Mon Sep 17 00:00:00 2001 From: guoyan1996 <1028109394@qq.com> Date: Fri, 3 Apr 2026 21:57:01 +0800 Subject: [PATCH 06/11] alter 0.0.3 to 0.0.2 --- .github/workflows/seekdb.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/seekdb.yml b/.github/workflows/seekdb.yml index 6aa3e447e..5b85f348f 100644 --- a/.github/workflows/seekdb.yml +++ b/.github/workflows/seekdb.yml @@ -29,7 +29,7 @@ jobs: if: ${{ github.repository_owner == 'oceanbase' && (github.event_name != 'pull_request' || !github.event.pull_request.draft) }} runs-on: arc-runner-k8s-ci-org container: - image: harbor.oceanbase-dev.com/obrde-test/ob-actions-runner-worker:0.0.3 + image: harbor.oceanbase-dev.com/obrde-test/ob-actions-runner-worker:0.0.2 steps: - name: Checkout uses: actions/checkout@v4 @@ -107,7 +107,7 @@ jobs: needs: seekdb-setup runs-on: arc-runner-k8s-ci-org container: - image: harbor.oceanbase-dev.com/obrde-test/ob-actions-runner-worker:0.0.3 + image: harbor.oceanbase-dev.com/obrde-test/ob-actions-runner-worker:0.0.2 strategy: fail-fast: false matrix: @@ -192,7 +192,7 @@ jobs: needs: seekdb-mysqltest runs-on: arc-runner-k8s-ci-org container: - image: harbor.oceanbase-dev.com/obrde-test/ob-actions-runner-worker:0.0.3 + image: harbor.oceanbase-dev.com/obrde-test/ob-actions-runner-worker:0.0.2 steps: - name: Checkout uses: actions/checkout@v4 From e916faddd657ba34772331a614b27f338f6bef8e Mon Sep 17 00:00:00 2001 From: guoyan1996 <1028109394@qq.com> Date: Sat, 4 Apr 2026 15:02:19 +0800 Subject: [PATCH 07/11] alter seekdb --- .../seekdb/scripts/mysqltest_for_seekdb.sh | 710 +++++++++++++----- 1 file changed, 510 insertions(+), 200 deletions(-) diff --git a/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh b/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh index 82015dc4e..731841ae8 100644 --- a/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh +++ b/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh @@ -1,49 +1,25 @@ -#!/bin/bash +#!/bin/sh set -x -# 脚本所在目录，末尾 source frame.sh 用 -SEEKDB_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" - -# seekdb native：GITHUB_WORKSPACE=仓库根，SLICE_IDX=slice 序号（0,1,2,3） -# 脚本在 .github/script/seekdb/scripts/，仓库根为 4 级上级 -REPO_ROOT_BY_SCRIPT="$(cd "$SEEKDB_SCRIPT_DIR/../../../.." && pwd)" -export GITHUB_WORKSPACE="${GITHUB_WORKSPACE:-$REPO_ROOT_BY_SCRIPT}" -# 若传入的 GITHUB_WORKSPACE 在容器内无 build.sh（如 runner 传主机路径），改用实际存在的仓库根 -if [[ ! -f "$GITHUB_WORKSPACE/build.sh" ]]; then - if [[ -f "$REPO_ROOT_BY_SCRIPT/build.sh" ]]; then - export GITHUB_WORKSPACE="$REPO_ROOT_BY_SCRIPT" - elif [[ -f "$(pwd)/build.sh" ]]; then - export GITHUB_WORKSPACE="$(pwd)" - fi -fi -export USER="${USER:-$(whoami)}" -SEEKDB_HOME="${GITHUB_WORKSPACE}/seekdb_home_${SLICE_IDX:-0}" -mkdir -p "$SEEKDB_HOME" -ln -sf "$GITHUB_WORKSPACE" "$SEEKDB_HOME/seekdb" -export HOME="$SEEKDB_HOME" -export SLOT_ID="${SLICE_IDX:-0}" -# 保证 HOME/observer 存在：优先用 WORKSPACE 下 observer，否则解压 observer.zst -if [[ -x "$GITHUB_WORKSPACE/observer" ]]; then - cp -f "$GITHUB_WORKSPACE/observer" "$HOME/observer" && chmod +x "$HOME/observer" -elif [[ -f "$GITHUB_WORKSPACE/observer.zst" ]]; then - (command -v zstd >/dev/null 2>&1 && zstd -d -f -o "$HOME/observer" "$GITHUB_WORKSPACE/observer.zst" && chmod +x "$HOME/observer") || true -fi - +export HOME=$_CONDOR_JOB_IWD export PATH=/bin:/usr/bin -HOST=$(hostname -i) -DOWNLOAD_DIR=$HOME/seekdb/tools/deploy +export USER=$(whoami) + +HOST=`hostname -i` +DOWNLOAD_DIR=$HOME/downloads +SLOT_ID=`echo $_CONDOR_SLOT | cut -c5-` function prepare_config { - if [[ "$MINI" == "1" ]] || ([[ "$MINI" == "-1" ]] && [[ -f $HOME/seekdb/tools/deploy/enable_mini_mode ]]) + if [[ "$MINI" == "1" ]] || ([[ "$MINI" == "-1" ]] && [[ -f $HOME/oceanbase/tools/deploy/enable_mini_mode ]]) then - [[ -f $HOME/seekdb/tools/deploy/enable_mini_mode ]] && MINI_SIZE=$(cat $HOME/seekdb/tools/deploy/enable_mini_mode) + [[ -f $HOME/oceanbase/tools/deploy/enable_mini_mode ]] && MINI_SIZE=$(cat $HOME/oceanbase/tools/deploy/enable_mini_mode) [[ $MINI_SIZE =~ ^[0-9]+G ]] || MINI_SIZE="8G" MINI_CONFIG_ITEM="ObCfg.init_config['memory_limit']='$MINI_SIZE'" fi - cd $HOME/seekdb/tools/deploy + cd $HOME/oceanbase/tools/deploy if [ "$WITH_PROXY" ] && [ "$WITH_PROXY" != "0" ] then if [ "$CLUSTER_SPEC" == '2x1' ] @@ -60,8 +36,6 @@ function prepare_config { SPEC="[$HOST]@zone1" fi fi - # farm中由于机器变小了，需要设置下 - CPU_COUNT_CONFIG="ObCfg.init_config['cpu_count']='24'" cat >config7.py < test.fifo 2>&1 elif [[ "$SLAVE" == "1" ]] then - cat test.fifo | tee result.out & ./hap.py $USER.obs1.mysqltest collect_all slices=$SLICES slice_idx=$SLICE_IDX $slb `echo $ARGV` 1> test.fifo 2>&1 + cat test.fifo | tee result.out & ./hap.py $USER.obs1.mysqltest collect_all slices=$SLICES slice_idx=$SLICE_IDX $slb `echo $ARGV` 1> test.fifo 2>&1 else cat test.fifo | tee result.out & ./hap.py $USER.mysqltest collect_all slices=$SLICES slice_idx=$SLICE_IDX $slb `echo $ARGV` 1> test.fifo 2>&1 fi @@ -141,14 +134,14 @@ function run_mysqltest { # check if there is error if [ "$?" == "0" ] then - grep -E 'PASSED' $HOME/seekdb/tools/deploy/result.out > /dev/null && ! grep -E "FAIL LST" $HOME/seekdb/tools/deploy/result.out > /dev/null || return 1 + grep -E 'PASSED' $HOME/oceanbase/tools/deploy/result.out > /dev/null && ! grep -E "FAIL LST" $HOME/oceanbase/tools/deploy/result.out > /dev/null || return 1 else return 1 fi } function collect_log { - cd $HOME/seekdb/tools/deploy || return 1 + cd $HOME/oceanbase/tools/deploy || return 1 if [ -d "collected_log" ] then mv mysql_test/var/log/* collected_log @@ -161,25 +154,25 @@ function collect_log { export -f run_mysqltest function obd_prepare_obd { - if [[ -f $HOME/seekdb/deps/init/dep_create.sh ]] + if [[ -f $HOME/oceanbase/deps/init/dep_create.sh ]] then if [[ "$IS_CE" == "1" ]] then - cd $HOME/seekdb && ./build.sh init --ce || return 3 + cd $HOME/oceanbase && ./build.sh init --ce || return 3 else - cd $HOME/seekdb && ./build.sh init || return 3 + cd $HOME/oceanbase && ./build.sh init || return 3 fi else - if grep 'dep_create.sh' $HOME/seekdb/build.sh + if grep 'dep_create.sh' $HOME/oceanbase/build.sh then - cd $HOME/seekdb/deps/3rd && bash dep_create.sh all || return 4 + cd $HOME/oceanbase/deps/3rd && bash dep_create.sh all || return 4 else - cd $HOME/seekdb && ./build.sh init || return 3 + cd $HOME/oceanbase && ./build.sh init || return 3 fi fi $obd devmode enable - $obd env set OBD_DEPLOY_BASE_DIR $HOME/seekdb/tools/deploy + $obd env set OBD_DEPLOY_BASE_DIR $HOME/oceanbase/tools/deploy $obd --version if [[ "$IS_CE" == "1" && ! -f $OBD_HOME/.obd/mirror/remote/taobao.repo ]] then @@ -191,23 +184,29 @@ export -f obd_prepare_obd function obd_prepare_global { export LANG=en_US.UTF-8 - HOST=$(hostname -i) + HOST=`hostname -i` DOWNLOAD_DIR=$HOME/downloads - export SLOT_ID="${SLOT_ID:-0}" - PORT_NUM=$((5000 + SLOT_ID * 100)) + SLOT_ID=`echo $_CONDOR_SLOT | cut -c5-` + PORT_NUM=`expr 5000 + $SLOT_ID \* 100` - # SeekDB CI：固定按 CE 路径走 OBD/init（原先用 observer -V 判 CE，SeekDB 输出不含 CE 字样会被判成 0） - COMPONENT="seekdb" - export IS_CE=1 + # 根据传入的observer binary判断是否开源版 + if [[ `$HOME/observer -V 2>&1 | grep -E '(OceanBase CE|OceanBase_CE)'` ]] + then + COMPONENT="oceanbase-ce" + export IS_CE=1 + else + COMPONENT="oceanbase" + export IS_CE=0 + fi # 根据build.sh中的内容判断依赖安装路径 - if grep 'dep_create.sh' $HOME/seekdb/build.sh + if grep 'dep_create.sh' $HOME/oceanbase/build.sh then - DEP_PATH=$HOME/seekdb/deps/3rd + DEP_PATH=$HOME/oceanbase/deps/3rd else - DEP_PATH=$HOME/seekdb/rpm/.dep_create/var + DEP_PATH=$HOME/oceanbase/rpm/.dep_create/var fi - if [[ -f "$HOME/seekdb/tools/deploy/mysqltest_config.yaml" ]] + if [[ -f "$HOME/oceanbase/tools/deploy/mysqltest_config.yaml" ]] then export WITH_MYSQLTEST_CONFIG_YAML=1 else @@ -217,22 +216,22 @@ function obd_prepare_global { ob_name=obcluster$SLOT_ID app_name=$ob_name.$USER.$HOST ocp_config_server='http://ocp-cfg.alibaba.net:8080/services?User_ID=alibaba&UID=test' + proxy_cfg_url=${ocp_config_server}\&Action=GetObProxyConfig\&ObRegionGroup=$app_name cfg_url=${ocp_config_server} export obd=$DEP_PATH/usr/bin/obd export OBD_HOME=$HOME export OBD_INSTALL_PRE=$DEP_PATH export DATA_PATH=$HOME/data - [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh } function obd_prepare_config { MINI_SIZE="10G" - if [[ "$MINI" == "1" ]] || ([[ "$MINI" == "-1" ]] && [[ -f $HOME/seekdb/tools/deploy/enable_mini_mode ]]) + if [[ "$MINI" == "1" ]] || ([[ "$MINI" == "-1" ]] && [[ -f $HOME/oceanbase/tools/deploy/enable_mini_mode ]]) then - [[ -f $HOME/seekdb/tools/deploy/enable_mini_mode ]] && MINI_SIZE=$(cat $HOME/seekdb/tools/deploy/enable_mini_mode) + [[ -f $HOME/oceanbase/tools/deploy/enable_mini_mode ]] && MINI_SIZE=$(cat $HOME/oceanbase/tools/deploy/enable_mini_mode) [[ $MINI_SIZE =~ ^[0-9]+G ]] || MINI_SIZE="8G" fi if [[ "$CLUSTER_SPEC" == '2x1' ]] || [[ "$SLAVE" == "1" ]] @@ -275,23 +274,74 @@ EOF ) fi export MYSQL_PORT=$mysql_port + proxy_conf="" + if [ "$WITH_PROXY" ] && [ "$WITH_PROXY" != "0" ] + then + listen_port=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1` + prometheus_listen_port=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1` + + proxy_conf=$(cat $HOME/oceanbase/tools/deploy/obd/obproxy.yaml.template) + if [[ -f $HOME/oceanbase/test/obproxy_test_config ]] + then + obproxy_switch=`sed '/^obproxy_switch=/!d;s/.*=//' $HOME/oceanbase/test/obproxy_test_config` - if [[ "$MINI" == "1" && -f $HOME/seekdb/tools/deploy/obd/core-test.yaml.template ]] + # 当且仅当 obproxy 指定模式为wget时，影响mysqltest obproxy配置，否则mysqltest依旧沿用原有obproxy获取逻辑 + mkdir $HOME/bin + if [[ -f $HOME/obproxy ]] + then + cp $HOME/obproxy $HOME/bin + else + wget_url=`sed '/^wget_url=/!d;s/.*=//' $HOME/oceanbase/test/obproxy_test_config` + wget $wget_url -O $HOME/bin/obproxy + if [[ -f $HOME/bin/obproxy ]] + then + echo "obproxy 文件下载成功" + chmod +x $HOME/bin/obproxy + + else + # 下载失败，沿用原有逻辑 + echo "obproxy 文件下载失败" + return 1 + fi + fi + + echo "obrpxoy版本:" + echo `$HOME/bin/obproxy -V 2>&1` + + obproxy_version=$("$HOME"/bin/obproxy --version 2>&1 | grep -E 'obproxy \(OceanBase [\.0-9]+ \S+\)' | grep -Eo '\s[.0-9]+\s') + + $obd mirror create -n obproxy -p $HOME -t "latest" -V $obproxy_version + echo "执行结果: $?" + + proxy_conf=$(echo "$proxy_conf" | sed '/package_hash: [0-9a-z]*/d') + proxy_conf="""$proxy_conf + tag: latest +""" + fi + proxy_conf=${proxy_conf//'{{%% COMPONENT %%}}'/$COMPONENT} + proxy_conf=${proxy_conf//'{{%% LISTEN_PORT %%}}'/$listen_port} + proxy_conf=${proxy_conf//'{{%% PROMETHEUS_LISTEN_PORT %%}}'/$prometheus_listen_port} + proxy_conf=${proxy_conf//'{{%% OBPORXY_HOME_PATH %%}}'/$DATA_PATH\/obproxy} + proxy_conf=${proxy_conf//'{{%% OBPROXY_CONFIG_SERVER_URL %%}}'/$proxy_cfg_url} + fi + + if [[ "$MINI" == "1" && -f $HOME/oceanbase/tools/deploy/obd/core-test.yaml.template ]] then - conf=$(cat $HOME/seekdb/tools/deploy/obd/core-test.yaml.template) + conf=$(cat $HOME/oceanbase/tools/deploy/obd/core-test.yaml.template) else - conf=$(cat $HOME/seekdb/tools/deploy/obd/config.yaml.template) + conf=$(cat $HOME/oceanbase/tools/deploy/obd/config.yaml.template) fi # cgroup support - if [[ -f $HOME/seekdb/tools/deploy/obd/.use_cgroup || $(echo " $ARGV " | grep " use-cgroup ") ]] + if [[ -f $HOME/oceanbase/tools/deploy/obd/.use_cgroup || $(echo " $ARGV " | grep " use-cgroup ") ]] then CGROUP_CONFIG="cgroup_dir: /sys/fs/cgroup/cpu/$USER" fi - extra_conf="""$MYSQLRTEST_ARGS + # MYSQLRTEST_ARGS contains changes for special run + proxy_conf="""$MYSQLRTEST_ARGS $CGROUP_CONFIG -""" - conf=${conf//'{{%% PROXY_CONF %%}}'/"$extra_conf"} - conf=${conf//'{{%% DEPLOY_PATH %%}}'/"$HOME/seekdb/tools/deploy"} +$proxy_conf""" + conf=${conf//'{{%% PROXY_CONF %%}}'/"$proxy_conf"} + conf=${conf//'{{%% DEPLOY_PATH %%}}'/"$HOME/oceanbase/tools/deploy"} conf=${conf//'{{%% COMPONENT %%}}'/$COMPONENT} conf=${conf//'{{%% SERVERS %%}}'/$SERVERS} conf=${conf//'{{%% TAG %%}}'/'latest'} @@ -303,86 +353,233 @@ EOF then conf=$(echo "$conf" | sed 's/oceanbase:/oceanbase\-ce:/g' | sed 's/\- oceanbase$/- oceanbase-ce/g') fi - echo "$conf" > $HOME/seekdb/tools/deploy/config.yaml - - if [[ $WITH_SHARED_STORAGE == "1" ]] - then - # shared_storage 需要在global修改相关的mode - share_storage_workdir="$(date +%s)-$(uuidgen)" - first_global_line_count=`sed -n '/global:/=' $HOME/seekdb/tools/deploy/config.yaml | head -n 1` - set +x - sed -i "${first_global_line_count}a\ mode: 'shared_storage'\n\ - shared_storage: \"oss://oss-436751-0701-all-test/$FARM2_RUN_USER/$share_storage_workdir?host=oss-cn-hangzhou.aliyuncs.com&access_id=${SENSITIVE_TEST_OSS_ID_FOR_OBJECT_STORAGE_FOR_TOTAL}&access_key=${SENSITIVE_TEST_OSS_KEY_FOR_OBJECT_STORAGE_FOR_TOTAL}&max_iops=0&max_bandwidth=0B&scope=region\"" $HOME/seekdb/tools/deploy/config.yaml - sed -i "s/datafile_size: '20G'/datafile_size: '200G'/g" $HOME/seekdb/tools/deploy/config.yaml - set -x - fi - + echo "$conf" > $HOME/oceanbase/tools/deploy/config.yaml if [[ $SHARED_STORAGE_MODE == "1" ]] then # shared_storage 需要在global修改相关的mode - MINIO_IP="100.88.99.213" - if [[ $TEST_NAME == "farm2" ]] + sh $HOME/common/start_farm_minio.sh $HOME/minio_home /data/farm/tools/minio /data/farm/tools/mc + local_mino_res=$? + MINIO_IP="127.0.0.1" + if [[ $local_mino_res -ne 0 ]] then MINIO_IP="100.88.99.213" + if [[ $TEST_NAME == "farm2" ]] + then + minio_ips=("100.88.99.213" "100.88.99.214") + random_index=$(( $RANDOM % ${#minio_ips[@]} )) + MINIO_IP=${minio_ips[$random_index]} + echo "CURRENT MINIO IP $MINIO_IP" + else + minio_ips=("100.88.99.213" "100.88.99.214") + random_index=$(( $RANDOM % ${#minio_ips[@]} )) + MINIO_IP=${minio_ips[$random_index]} + echo "CURRENT MINIO IP $MINIO_IP" + fi fi share_storage_workdir="$(date +%s)-$(uuidgen)" - first_global_line_count=`sed -n '/global:/=' $HOME/seekdb/tools/deploy/config.yaml | head -n 1` + first_global_line_count=`sed -n '/global:/=' $HOME/oceanbase/tools/deploy/config.yaml | head -n 1` sed -i "${first_global_line_count}a\ mode: 'shared_storage'\n\ - shared_storage: \"s3://farm-test/mysqltest/$share_storage_workdir/clog_and_data?host=http://$MINIO_IP:9000&access_id=minioadmin&access_key=minioadmin&s3_region=us-east-1&max_iops=10000&max_bandwidth=1GB&scope=region\"" $HOME/seekdb/tools/deploy/config.yaml + shared_storage: \"s3://farm-test/mysqltest/$share_storage_workdir/clog_and_data?host=http://$MINIO_IP:9000&access_id=minioadmin&access_key=minioadmin&s3_region=us-east-1&max_iops=10000&max_bandwidth=1GB&scope=region\"\n\ + random_shared_storage_bucket: \"1\"" $HOME/oceanbase/tools/deploy/config.yaml + sed -i "s/# datafile_size: '20G'/datafile_size: '200G'/g" $HOME/oceanbase/tools/deploy/config.yaml + sed -i "s/datafile_size: '20G'/datafile_size: '200G'/g" $HOME/oceanbase/tools/deploy/config.yaml + need_oblogservice=`cat $HOME/oceanbase/test/farm.yaml | grep -Pz "mysqltest:\n\s+need_oblogservice:\s+true"` + if [[ $need_oblogservice ]] + then + first_global_line_count=`sed -n '/global:/=' $HOME/oceanbase/tools/deploy/config.yaml | head -n 1` + sed -i "${first_global_line_count}a\ logservice_cluster_id: 1234\n\ + object_store_url: s3://farm-test/?host=http://$MINIO_IP:9000&access_id=minioadmin&access_key=minioadmin&s3_region=us-east-1&max_iops=10000&max_bandwidth=1GB&scope=region\n\ + enable_logservice: true" $HOME/oceanbase/tools/deploy/config.yaml + for i in 1 2 3; + do + export oblogservice_port${i}=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1`; + export oblogservice_rpc_port${i}=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1`; + done + sed -i "1i\\ +oblogservice:\\ + servers:\\ + - name: node1\\ + ip: 127.0.0.1 # 启动IP地址\\ + - name: node2\\ + ip: 127.0.0.1 # 启动IP地址\\ + - name: node3\\ + ip: 127.0.0.1 # 启动IP地址\\ + node1:\\ + rpc_port: $oblogservice_rpc_port1 # RPC协议端口号\\ + http_port: $oblogservice_port1 # http端口\\ + region: beijing\\ + home_path: $DATA_PATH/logservice/node1 # 部署路径，默认oceanbase, home_path\\ + local_storage_dir: $DATA_PATH/logservice/node1/store # 默认store\\ + node2:\\ + rpc_port: $oblogservice_rpc_port2 # RPC协议端口号\\ + http_port: $oblogservice_port2 # http端口\\ + region: beijing\\ + home_path: $DATA_PATH/logservice/node2 # 部署路径，默认oceanbase, home_path\\ + local_storage_dir: $DATA_PATH/logservice/node2/store # 默认store\\ + node3:\\ + rpc_port: $oblogservice_rpc_port3 # RPC协议端口号\\ + http_port: $oblogservice_port3 # http端口\\ + region: beijing\\ + home_path: $DATA_PATH/logservice/node3 # 部署路径，默认oceanbase, home_path\\ + local_storage_dir: $DATA_PATH/logservice/node3/store # 默认store\\ + global:\\ + cluster_id: 1234 # cluster_id\\ + object_store_url: s3://farm-test/?host=http://$MINIO_IP:9000&access_id=minioadmin&access_key=minioadmin&s3_region=us-east-1&max_iops=10000&max_bandwidth=1GB&scope=region" $HOME/oceanbase/tools/deploy/config.yaml + # 修改内存增加1G + current_value=$(grep -Eo "memory_limit: ['\"]?[0-9]+G['\"]?" $HOME/oceanbase/tools/deploy/config.yaml | grep -Eo '[0-9]+') + # 计算新值 + new_value=$((current_value + 5)) + # 替换文件中的值（兼容单/双引号） + sed -i "s/memory_limit: ['\"]${current_value}G['\"]/memory_limit: '${new_value}G'/" $HOME/oceanbase/tools/deploy/config.yaml + # 修改内存增加1G + current_system_memory_value=$(grep -Eo "system_memory: ['\"]?[0-9]+G['\"]?" $HOME/oceanbase/tools/deploy/config.yaml | grep -Eo '[0-9]+') + # 计算新值 + new_system_memory_value=$((current_system_memory_value + 2)) + # 替换文件中的值（兼容单/双引号） + sed -i "s/system_memory: ['\"]${current_system_memory_value}G['\"]/system_memory: '${new_system_memory_value}G'/" $HOME/oceanbase/tools/deploy/config.yaml + + first_oceanbase_line_count=`sed -n '/oceanbase:/=' $HOME/oceanbase/tools/deploy/config.yaml | head -n 1` + sed -i "${first_oceanbase_line_count}a\ depends:\n\ + - oblogservice" $HOME/oceanbase/tools/deploy/config.yaml + fi fi + cat $HOME/oceanbase/tools/deploy/config.yaml +} +function get_baseurl { + repo_file=$OBD_HOME/.obd/mirror/remote/taobao.repo + start=0 + end=$(cat $repo_file | wc -l ) + for line in $(grep -nE "^\[.*\]$" $repo_file) + do + num=$(echo "$line" | awk -F ":" '{print $1}') + [ "$find_section" == "1" ] && end=$num + is_target_section=$(echo $(echo "$line" | grep -E "\[\s*$repository\s*\]")) + [ "$is_target_section" != "" ] && start=$num && find_section='1' + done + repo_section=$(awk "NR>=$start && NR<=$end" $repo_file) + baseurl=$(echo "$repo_section" | grep baseurl | awk -F '=' '{print $2}') +} - if [[ ! $WITH_SHARED_STORAGE == "1" ]] +function get_version_and_release { + start=0 + end=$(cat $config_yaml | wc -l) + for line in $(grep -nE '^\S+:' $config_yaml) + do + num=$(echo "$line" | awk -F ":" '{print $1}') + [ "$find_obproxy" == "1" ] && end=$num + [ "$(echo "$line" | grep obproxy)" != "" ] && start=$num && find_obproxy='1' + done + odp_conf=$(awk "NR>=$start && NR<=$end" $config_yaml) + version=$(echo "$odp_conf" | grep version | awk '{print $2}') + release=$(echo "$odp_conf" | grep release | awk '{print $2}') + include_file=$DEPLOY_PATH/$(echo "$odp_conf"| grep 'include:'| awk '{print $2}') + if [[ -f $include_file ]] then - cat $HOME/seekdb/tools/deploy/config.yaml + _repo=$(echo $(grep '__mirror_repository_section_name:' $include_file | awk '{print $2}')) + [ "$_repo" != "" ] && repository=$_repo + version=$(echo $(grep 'version:' $include_file | awk '{print $2}')) + release=$(echo $(grep 'release:' $include_file | awk '{print $2}')) fi } +function get_obproxy { + repository="taobao" + config_yaml=$HOME/oceanbase/tools/deploy/config.yaml + DEPLOY_PATH=$HOME/oceanbase/tools/deploy + obproxy_mirror_repository=$(echo $(grep '__mirror_repository_section_name' $config_yaml | awk -F':' '{print $2}')) + [ "$obproxy_mirror_repository" != "" ] && repository=$obproxy_mirror_repository + + get_version_and_release + get_baseurl + $obd mirror disable remote + + if [[ "$baseurl" != "" && "$version" != "" && "$release" != "" ]] + then + OS_ARCH="$(uname -m)" + if [[ $OS_ARCH == "aarch64" ]] + then + pkg_name="obproxy-$version-$release.aarch64.rpm" + else + pkg_name="obproxy-$version-$release.x86_64.rpm" + fi + if [ "$(find $OBD_HOME/.obd/mirror/local -name $pkg_name)" == "" ] + then + download_dir=$OBD_HOME/.obd_download + mkdir -p $download_dir + wget $baseurl/obproxy/$pkg_name -O "$download_dir/$pkg_name" -o $download_dir/obproxy.down + $obd mirror clone "$download_dir/$pkg_name" && rm -rf $download_dir && return 0 + else + return 0 + fi + fi + echo "Download pkg failed. Try to get obproxy from repository $repository" + count=0 + interval=5 + retry_limit=100 + while (( $count < $retry_limit )) + do + $obd mirror disable remote + $obd mirror enable $repository + $obd mirror update && $obd mirror list $repository > /dev/null && return 0 + count=`expr $count + 1` + (( $count <= $retry_limit )) && sleep $interval + done + return 1 +} + function obd_prepare_bin { cd $HOME mkdir -p $DOWNLOAD_DIR/{bin,etc,admin} || return 1 + if [ "$WITH_PROXY" ] && [ "$WITH_PROXY" != "0" ] + then + get_obproxy || (echo "failed to get obdproxy" || return 1) + fi + if [ ! -x observer ] then ( cd $DOWNLOAD_DIR/bin && - wget http://6.12.233.4:5000/observer -O observer && + wget http://11.166.86.153:8877/observer -O observer && chmod +x observer ) || return 1 else cp observer $DOWNLOAD_DIR/bin/ || return 2 fi - - if [[ "$COMPONENT" == "seekdb" ]] && [[ -x $DOWNLOAD_DIR/bin/observer ]] - then - cp $DOWNLOAD_DIR/bin/observer $DOWNLOAD_DIR/bin/seekdb && chmod +x $DOWNLOAD_DIR/bin/seekdb - fi - - - cd $HOME/seekdb/tools/deploy - [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh - [[ -x $HOME/seekdb/tools/deploy/obd.sh ]] && ./obd.sh prepare + + cd $HOME/oceanbase/tools/deploy cp $DEP_PATH/u01/obclient/bin/obclient ./obclient && chmod 777 obclient cp $DEP_PATH/u01/obclient/bin/mysqltest ./mysqltest && chmod 777 mysqltest - + if [ "$WITH_DEPS" ] && [ "$WITH_DEPS" != "0" ] then - cd $HOME/seekdb/tools/deploy && [[ -f copy.sh ]] && sh copy.sh + cd $HOME/oceanbase/tools/deploy && [[ -f copy.sh ]] && sh copy.sh fi - if [[ -f "$HOME/seekdb/tools/deploy/obd/.observer_obd_plugin_version" ]] + if [[ -f "$HOME/oceanbase/tools/deploy/obd/.observer_obd_plugin_version" ]] then - obs_version=$(cat $HOME/seekdb/tools/deploy/obd/.observer_obd_plugin_version) + obs_version=$(cat $HOME/oceanbase/tools/deploy/obd/.observer_obd_plugin_version) else - $DOWNLOAD_DIR/bin/observer -V - obs_version=`$DOWNLOAD_DIR/bin/observer -V 2>&1 | grep -E "(observer|seekdb) \(OceanBase([ \_]CE|[ \_]Lite|[ \_]SeekDB|[ \_]seekdb)? [0-9]" | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?' | head -1` + obs_version=`$DOWNLOAD_DIR/bin/observer -V 2>&1 | grep -E "observer \(OceanBase([ \_]CE)? ([.0-9]+)\)" | grep -Eo '([.0-9]+)'` fi - mkdir $HOME/seekdb/tools/deploy/admin - if [[ -d $HOME/seekdb/src/share/inner_table/sys_package ]] + mkdir $HOME/oceanbase/tools/deploy/admin + if [[ -d $HOME/oceanbase/src/share/inner_table/sys_package ]] then - cp $HOME/seekdb/src/share/inner_table/sys_package/*.sql $HOME/seekdb/tools/deploy/admin/ + cp $HOME/oceanbase/src/share/inner_table/sys_package/*.sql $HOME/oceanbase/tools/deploy/admin/ fi $obd mirror create -n $COMPONENT -p $DOWNLOAD_DIR -t latest -V $obs_version || return 2 + + need_oblogservice=`cat $HOME/oceanbase/test/farm.yaml | grep -Pz "mysqltest:\n\s+need_oblogservice:\s+true"` + if [[ -x $HOME/oblogservice && -f $HOME/oceanbase/test/oblogservice_test_config && $need_oblogservice ]] + then + echo "存在指定oblogservice, 指定版本为" + oblogservice_version=`$HOME/oblogservice -V 2>&1 | grep -E "oblogservice \(ObLogService: ([.0-9]+)\)" | grep -Eo '([.0-9]+)'` + + cp $HOME/oblogservice $DOWNLOAD_DIR/bin/ || return 5 + $obd mirror create -n oblogservice -p $DOWNLOAD_DIR -t latest -V $oblogservice_version || return 2 + + fi } @@ -392,10 +589,8 @@ function obd_init_cluster { do if [[ "$retries" == "$reboot_retries" ]] then - [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh - $obd cluster deploy $ob_name -c $HOME/seekdb/tools/deploy/config.yaml -f && $obd cluster start $ob_name -f && $obd cluster display $ob_name + $obd cluster deploy $ob_name -c $HOME/oceanbase/tools/deploy/config.yaml -f && $obd cluster start $ob_name -f && $obd cluster display $ob_name else - [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh $obd cluster redeploy $ob_name -f fi retries=`expr $retries - 1` @@ -406,16 +601,16 @@ function obd_init_cluster { init_files=('init.sql' 'init_mini.sql' 'init_for_ce.sql') for init_file in ${init_files[*]} do - if [[ -f $HOME/seekdb/tools/deploy/$init_file ]] + if [[ -f $HOME/oceanbase/tools/deploy/$init_file ]] then - if ! grep "alter system set_tp tp_no = 509, error_code = 4016, frequency = 1;" $HOME/seekdb/tools/deploy/$init_file + if ! grep "alter system set_tp tp_no = 509, error_code = 4016, frequency = 1;" $HOME/oceanbase/tools/deploy/$init_file then - echo -e "\nalter system set_tp tp_no = 509, error_code = 4016, frequency = 1;" >> $HOME/seekdb/tools/deploy/$init_file + echo -e "\nalter system set_tp tp_no = 509, error_code = 4016, frequency = 1;" >> $HOME/oceanbase/tools/deploy/$init_file fi fi done fi - $obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient --init-only --init-sql-dir=$HOME/seekdb/tools/deploy $INIT_FLIES --log-dir=$HOME/init_case $EXTRA_ARGS -v > ./init_sql_log && init_seccess=1 && break + $obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient --init-only --init-sql-dir=$HOME/oceanbase/tools/deploy $INIT_FLIES --log-dir=$HOME/init_case $COLLECT_ARG $EXTRA_ARGS -v > ./init_sql_log && init_seccess=1 && break done } @@ -435,8 +630,7 @@ function obd_run_mysqltest { set -x obd_prepare_global - cd $HOME/seekdb/tools/deploy - [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh + cd $HOME/oceanbase/tools/deploy if [[ "$SLB" != "" ]] && [[ "$EXECID" != "" ]] then SCHE_ARGS="--slb-host=$SLB --exec-id=$EXECID " @@ -450,7 +644,7 @@ function obd_run_mysqltest { INIT_FLIES="--init-sql-files=core-test.init_mini.sql,init_user.sql|root@mysql|test" [ -f init_user_oracle.sql ] && INIT_FLIES="${INIT_FLIES},init_user_oracle.sql|SYS@oracle|SYS" else - if [[ "$IS_CE" == "1" && -f $HOME/seekdb/.ce ]] + if [[ "$IS_CE" == "1" && -f $HOME/oceanbase/.ce ]] then INIT_FLIES="--init-sql-files=init_mini.sql,init_user.sql|root@mysql|test " fi @@ -474,11 +668,19 @@ function obd_run_mysqltest { else EXTRA_ARGS="--cluster-mode=$CLUSTER_MODE " fi - if [[ -f $HOME/seekdb/tools/deploy/obd/.fast-reboot ]] + if [[ -f $HOME/oceanbase/tools/deploy/obd/.fast-reboot ]] then EXTRA_ARGS="${EXTRA_ARGS}--fast-reboot " fi + if [[ $SHARED_STORAGE_MODE == "1" ]] + then + EXTRA_ARGS="${EXTRA_ARGS}--fast-reboot=false " + elif [[ -f $HOME/oceanbase/tools/deploy/obd/.fast-reboot ]] + then + EXTRA_ARGS="${EXTRA_ARGS}--fast-reboot " + fi + if [[ "$IS_CE" == "1" ]] then EXTRA_ARGS="${EXTRA_ARGS}--mode=mysql " @@ -510,10 +712,17 @@ function obd_run_mysqltest { if [[ "$disable_collect" != "1" ]] then COLLECT_ARG="--collect-all" - if [[ "$WITH_MYSQLTEST_CONFIG_YAML" == "0" ]] && [ "$WITH_PROXY" ] && [ "$WITH_PROXY" != "0" ] && [[ -f "$HOME/seekdb/tools/deploy/obd/.collect_proxy_log" ]] + if [[ "$WITH_MYSQLTEST_CONFIG_YAML" == "0" ]] && [ "$WITH_PROXY" ] && [ "$WITH_PROXY" != "0" ] && [[ -f "$HOME/oceanbase/tools/deploy/obd/.collect_proxy_log" ]] then COLLECT_ARG="$COLLECT_ARG --collect-components=$COMPONENT,obproxy" fi + # 由于config.yaml 会给这部分默认设置一个proxy模式下的配置，并且没办法被覆盖，因此先单独在这里处理 + need_oblogservice=`cat $HOME/oceanbase/test/farm.yaml | grep -Pz "mysqltest:\n\s+need_oblogservice:\s+true"` + if [[ $need_oblogservice && $SHARED_STORAGE_MODE == "1" ]] + then + COLLECT_ARG="$COLLECT_ARG --collect-components=$COMPONENT,obproxy,oblogservice" + fi + fi obd_init_cluster if [[ "$init_seccess" != "1" ]] @@ -521,9 +730,13 @@ function obd_run_mysqltest { cat ./init_sql_log echo "Failed to init sql, see more log at ./collected_log/_test_init" mkdir -p $HOME/collected_log/_test_init/ - [[ -d "$DATA_PATH/observer1/log" ]] && mkdir -p $HOME/collected_log/_test_init/observer1 && mv $DATA_PATH/observer1/log/* $HOME/collected_log/_test_init/observer1 - [[ -d "$DATA_PATH/observer2/log" ]] && mkdir -p $HOME/collected_log/_test_init/observer2 && mv $DATA_PATH/observer2/log/* $HOME/collected_log/_test_init/observer2 - [[ -d "$DATA_PATH/obproxy/log" ]] && mkdir -p $HOME/collected_log/_test_init/obproxy && mv $DATA_PATH/obproxy/log/* $HOME/collected_log/_test_init/obproxy + [[ -d "$HOME/init_case" ]] && mv $HOME/init_case $HOME/collected_log/ + [[ -d "$DATA_PATH/observer1/log" ]] && mkdir -p $HOME/collected_log/_test_init/observer1 && cp -r $DATA_PATH/observer1/log/* $HOME/collected_log/_test_init/observer1 + [[ -d "$DATA_PATH/observer2/log" ]] && mkdir -p $HOME/collected_log/_test_init/observer2 && cp -r $DATA_PATH/observer2/log/* $HOME/collected_log/_test_init/observer2 + [[ -d "$DATA_PATH/obproxy/log" ]] && mkdir -p $HOME/collected_log/_test_init/obproxy && cp -r $DATA_PATH/obproxy/log/* $HOME/collected_log/_test_init/obproxy + [[ -d "$DATA_PATH/logservice/node1/log" ]] && mkdir -p $HOME/collected_log/_test_init/logservice1 && cp -r $DATA_PATH/logservice/node1/log/* $HOME/collected_log/_test_init/logservice1 + [[ -d "$DATA_PATH/logservice/node2/log" ]] && mkdir -p $HOME/collected_log/_test_init/logservice2 && cp -r $DATA_PATH/logservice/node2/log/* $HOME/collected_log/_test_init/logservice2 + [[ -d "$DATA_PATH/logservice/node3/log" ]] && mkdir -p $HOME/collected_log/_test_init/logservice3 && cp -r $DATA_PATH/logservice/node3/log/* $HOME/collected_log/_test_init/logservice3 exit 1 fi if [[ "$SPECIAL_RUN" == "1" ]] @@ -533,7 +746,7 @@ function obd_run_mysqltest { mkdir -p $result_dir RECORD_ARGS="--record-dir=$result_dir --record --log-dir=./var/rec_log" RESULT_ARGS="--result-dir=$result_dir --result-file-suffix=.record --log-dir=./var/log" - mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient --init-sql-dir=$HOME/seekdb/tools/deploy --special-run --sort-result $COLLECT_ARG $REBOOT_TIMEOUT $VERBOSE_ARG" + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient --init-sql-dir=$HOME/oceanbase/tools/deploy --special-run --sort-result $COLLECT_ARG $REBOOT_TIMEOUT $VERBOSE_ARG" # $mysqltest_cmd --all $INIT_FLIES $SCHE_ARGS $RECORD_ARGS 2>&1 > record.out $mysqltest_cmd $INIT_FLIES $SCHE_ARGS $RECORD_ARGS $EXTRA_ARGS 2>&1 | tee record.out && ( exit ${PIPESTATUS[0]}) rec_ret=$? @@ -551,8 +764,8 @@ function obd_run_mysqltest { done obd_prepare_config echo "config after change:" - cat $HOME/seekdb/tools/deploy/config.yaml - $obd cluster destroy $ob_name && $obd cluster deploy $ob_name -c $HOME/seekdb/tools/deploy/config.yaml && $obd cluster start $ob_name -f && $obd cluster display $ob_name + cat $HOME/oceanbase/tools/deploy/config.yaml + $obd cluster destroy $ob_name && $obd cluster deploy $ob_name -c $HOME/oceanbase/tools/deploy/config.yaml && $obd cluster start $ob_name -f && $obd cluster display $ob_name else $obd cluster redeploy $ob_name fi @@ -565,6 +778,9 @@ function obd_run_mysqltest { if [[ $JOBNAME == 'mysqltest_opensource' ]] then submarker="_opensource" + elif [[ $JOBNAME == 'mysqltest_sanity' ]] + then + submarker="_sanity" else submarker="" fi @@ -581,43 +797,43 @@ function obd_run_mysqltest { # todo: 修改为把tools deploy下的init sql都带入过去 mkdir $HOME/tmp_init_sql mkdir $HOME/obd_tmp - find $HOME/seekdb/tools/deploy -name 'init_*.sql' | xargs -i cp {} $HOME/tmp_init_sql - cp $HOME/seekdb/tools/deploy/obd/* $HOME/obd_tmp + find $HOME/oceanbase/tools/deploy -name 'init_*.sql' | xargs -i cp {} $HOME/tmp_init_sql + cp $HOME/oceanbase/tools/deploy/obd/* $HOME/obd_tmp cd $HOME && tar -cvf tmp_init_sql.tar.gz $HOME/tmp_init_sql cd $HOME && tar -cvf origin_obd_file.tar.gz $HOME/obd_tmp - cd $HOME/seekdb + cd $HOME/oceanbase # 针对多个分支进行处理 ret=0 for commit in $CROSS_VALIDATION_COMMIT do git checkout -f $commit || ret=2 # 然后重新针对obd 进行重新构建路径 - cd $HOME && tar -xvf tmp_init_sql.tar.gz && cp tmp_init_sql/* $HOME/seekdb/tools/deploy - cd $HOME && tar -xvf origin_obd_file.tar.gz && cp obd_tmp/* $HOME/seekdb/tools/deploy/obd + cd $HOME && tar -xvf tmp_init_sql.tar.gz && cp tmp_init_sql/* $HOME/oceanbase/tools/deploy + cd $HOME && tar -xvf origin_obd_file.tar.gz && cp obd_tmp/* $HOME/oceanbase/tools/deploy/obd - if [[ -f $HOME/seekdb/deps/init/dep_create.sh ]] + if [[ -f $HOME/oceanbase/deps/init/dep_create.sh ]] then if [[ "$IS_CE" == "1" ]] then - cd $HOME/seekdb && ./build.sh init --ce || ret=3 + cd $HOME/oceanbase && ./build.sh init --ce || ret=3 else - cd $HOME/seekdb && ./build.sh init || ret=3 + cd $HOME/oceanbase && ./build.sh init || ret=3 fi else - if grep 'dep_create.sh' $HOME/seekdb/build.sh + if grep 'dep_create.sh' $HOME/oceanbase/build.sh then - cd $HOME/seekdb/deps/3rd && bash dep_create.sh all || ret=4 + cd $HOME/oceanbase/deps/3rd && bash dep_create.sh all || ret=4 else - cd $HOME/seekdb && ./build.sh init || ret=4 + cd $HOME/oceanbase && ./build.sh init || ret=4 fi fi # 根据build.sh中的内容判断依赖安装路径 - if grep 'dep_create.sh' $HOME/seekdb/build.sh + if grep 'dep_create.sh' $HOME/oceanbase/build.sh then - DEP_PATH=$HOME/seekdb/deps/3rd + DEP_PATH=$HOME/oceanbase/deps/3rd else - DEP_PATH=$HOME/seekdb/rpm/.dep_create/var + DEP_PATH=$HOME/oceanbase/rpm/.dep_create/var fi extra_ignore_cmd="" if [[ `$DEP_PATH/u01/obclient/bin/mysqltest --help | grep "cmp-ignore-explain"` ]] @@ -630,17 +846,17 @@ function obd_run_mysqltest { extra_ignore_cmd=" $extra_ignore_cmd --special-run" fi - cd $HOME/seekdb/tools/deploy - mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=$DEP_PATH/u01/obclient/bin/mysqltest --obclient-bin=$DEP_PATH/u01/obclient/bin/obclient $COLLECT_ARG --init-sql-dir=$HOME/seekdb/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS_WITHOUT_CASE --suite=cross_validation $extra_ignore_cmd" + cd $HOME/oceanbase/tools/deploy + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=$DEP_PATH/u01/obclient/bin/mysqltest --obclient-bin=$DEP_PATH/u01/obclient/bin/obclient $COLLECT_ARG --init-sql-dir=$HOME/oceanbase/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS_WITHOUT_CASE --suite=cross_validation $extra_ignore_cmd" $mysqltest_cmd $INIT_FLIES $SCHE_ARGS 2>&1 | tee compare.out && ( exit ${PIPESTATUS[0]}) test_suite_ret=$? - FILTER=`ls -ltr $HOME/seekdb/tools/deploy/mysql_test/test_suite/cross_validation/t | grep -v total | awk '{gsub(".test","",$9);print "cross_validation."$9}' | tr '\n' ',' | head -c -1` + FILTER=`ls -ltr $HOME/oceanbase/tools/deploy/mysql_test/test_suite/cross_validation/t | grep -v total | awk '{gsub(".test","",$9);print "cross_validation."$9}' | tr '\n' ',' | head -c -1` if [[ $FILTER == "" ]] then echo "without testsuite cases" fi - mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=$DEP_PATH/u01/obclient/bin/mysqltest --obclient-bin=$DEP_PATH/u01/obclient/bin/obclient $COLLECT_ARG --init-sql-dir=$HOME/seekdb/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS_WITHOUT_CASE --exclude=$FILTER --tag=cross_validation $extra_ignore_cmd" + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=$DEP_PATH/u01/obclient/bin/mysqltest --obclient-bin=$DEP_PATH/u01/obclient/bin/obclient $COLLECT_ARG --init-sql-dir=$HOME/oceanbase/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS_WITHOUT_CASE --exclude=$FILTER --tag=cross_validation $extra_ignore_cmd" $mysqltest_cmd $INIT_FLIES $SCHE_ARGS 2>&1 | tee compare.out && ( exit ${PIPESTATUS[0]}) tag_ret=$? [[ $test_suite_ret = 0 && $tag_ret = 0 ]] && current_ret=0 || current=1 @@ -650,51 +866,130 @@ function obd_run_mysqltest { if [[ $JOBNAME == 'mysqltest_opensource' ]] then submarker="_opensource" + elif [[ $JOBNAME == 'mysqltest_sanity' ]] + then + submarker="_sanity" else submarker="" fi - mv compare.out $HOME/mysqltest${submarker}_compare_output.$SLICE_IDX + mv compare.out $HOME/mysqltest${submarker}_compare_output${JOB_SUFFIX}.$SLICE_IDX echo "finish!" - return $ret - elif [[ $WITH_SHARED_STORAGE == "1" ]] - then - # 把sensitive_test目录下的测试集合移动到tools/deploy/mysql_test - # 并且创建一个新的目录用来表示单独的测试 - if [[ ! -d $HOME/seekdb/sensitive_test/mysql_test ]] + return $ret + else + if [[ -f $HOME/oceanbase/tools/deploy/error_log_filter.json && ( $BRANCH == 'master' || $BRANCH == "4_2_x_release" ) && "$FROM_FARM" == '1' ]] then - return 0 + echo "python $HOME/common/analyse_the_observer_log.py collect -p ${DATA_PATH}/observer1/log -p ${DATA_PATH}/observer2/log -i $GID -j ${JOBNAME}.${SLICE_IDX} -o $HOME -F $HOME/oceanbase/tools/deploy/error_log_filter.json" > $HOME/oceanbase/tools/deploy/analyse_the_observer_log.sh + chmod a+x $HOME/oceanbase/tools/deploy/analyse_the_observer_log.sh + RUN_EXTRA_CHECK_CMD='--extra-script-after-case="$HOME/oceanbase/tools/deploy/./analyse_the_observer_log.sh"' fi - need_mv_dirs=`ls -1 $HOME/seekdb/sensitive_test/mysql_test/test_suite|xargs` - run_suites="" - for need_mv_dir in $need_mv_dirs - do - new_dir=$HOME/seekdb/tools/deploy/mysql_test/test_suite/shared_storage__${need_mv_dir} - run_suites="${run_suites}shared_storage__${need_mv_dir}," - mkdir -p $new_dir - cp -r $HOME/seekdb/sensitive_test/mysql_test/test_suite/$need_mv_dir/* $new_dir - done - run_suites=`echo $run_suites | awk '{print substr($0, 1, length($0)-1)}'` - mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient $COLLECT_ARG --init-sql-dir=$HOME/seekdb/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS_WITHOUT_CASE --suite=$run_suites" - $mysqltest_cmd $INIT_FLIES $SCHE_ARGS 2>&1 | tee compare.out && ( exit ${PIPESTATUS[0]}) - else - if [[ -f $HOME/seekdb/tools/deploy/error_log_filter.json && ( $BRANCH == 'master' || $BRANCH == "4_2_x_release" ) && "$FROM_FARM" == '1' ]] + if [[ $SHARED_STORAGE_MODE == "1" ]] then - echo "python $HOME/common/analyse_the_observer_log.py collect -p ${DATA_PATH}/observer1/log -p ${DATA_PATH}/observer2/log -i $GID -j ${JOBNAME}.${SLICE_IDX} -o $HOME -F $HOME/seekdb/tools/deploy/error_log_filter.json" > $HOME/seekdb/tools/deploy/analyse_the_observer_log.sh - chmod a+x $HOME/seekdb/tools/deploy/analyse_the_observer_log.sh - RUN_EXTRA_CHECK_CMD='--extra-script-after-case="$HOME/seekdb/tools/deploy/./analyse_the_observer_log.sh"' + # 由于--filter也是识别参数，因为会被带入到模式里面，需要放在ARGV识别之后处理 + EXTRA_ARGS="$EXTRA_ARGS--filter=shared_storage " + fi + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient $COLLECT_ARG --init-sql-dir=$HOME/oceanbase/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS" + if [[ $SHARED_STORAGE_MODE == "1" && $TEST_NAME == "farm" ]] + then + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient $COLLECT_ARG --init-sql-dir=$HOME/oceanbase/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG --shared_storage_pstest $EXTRA_ARGS_WITHOUT_CASE" + fi + if [[ $TEST_NAME != "farm" ]] + then + # 大farm处理下odpscmd + mkdir $HOME/odps + cd $HOME/odps && timeout 200 bash -c 'wget https://antsys-enc.cn-heyuan-alipay-office.oss-alipay.aliyuncs.com/public/core-test/tools/odps/odpscmd_public.zip' + unzip odpscmd_public.zip + export PATH=$PATH:$HOME/odps/bin + + mkdir $HOME/s3 + yum install https://antsys-enc.cn-heyuan-alipay-office.oss-alipay.aliyuncs.com/public/core-test/tools/s3/python-dateutil-1.5-7.1.alios7.noarch.rpm -y + cd $HOME/s3 && timeout 200 bash -c 'wget https://antsys-enc.cn-heyuan-alipay-office.oss-alipay.aliyuncs.com/public/core-test/tools/s3/s3cmd.tar.gz' + tar -xvf s3cmd.tar.gz -C $HOME/s3 + export PATH=$PATH:$HOME/s3/s3cmd-2.4.0 + + cd $HOME/oceanbase/tools/deploy + set +x + sed -i 's#project_name=#project_name=mysqltest_regression#g' $HOME/odps/conf/odps_config.ini + sed -i 's@end_point=@end_point=http://service.cn-hangzhou.maxcompute.aliyun.com/api@g' $HOME/odps/conf/odps_config.ini + sed -i s/''/${MYSQLTEST_ODPS_ACCESSID}/g $HOME/odps/conf/odps_config.ini + sed -i s/''/${MYSQLTEST_ODPS_ACCESSKEY}/g $HOME/odps/conf/odps_config.ini + set -x + + if [[ -f $HOME/oceanbase/tools/deploy/export_vars_for_mysqltest.sh ]] + then + export VAULT_ADDR='https://vault.oceanbase-dev.com/' + source $HOME/oceanbase/tools/deploy/export_vars_for_mysqltest.sh + fi + set +x + sed -i s/'#OSS_HOST#'/cn-heyuan-alipay-b.oss-alipay.aliyuncs.com/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + sed -i s/'#OSS_ACCESS_KEY#'/${MYSQLTEST_OSS_KEY}/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + sed -i s/'#OSS_ACCESS_ID#'/${MYSQLTEST_OSS_ID}/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + sed -i 's@#ODPS_ENDPOINT#@http://service.cn-hangzhou.maxcompute.aliyun.com/api@g' $HOME/oceanbase/tools/deploy/mysql_test/include/odps_config_for_cloud.inc + sed -i s/'#ODPS_ACCESSID#'/${MYSQLTEST_ODPS_ACCESSID}/g $HOME/oceanbase/tools/deploy/mysql_test/include/odps_config_for_cloud.inc + sed -i s/'#ODPS_ACCESSKEY#'/${MYSQLTEST_ODPS_ACCESSKEY}/g $HOME/oceanbase/tools/deploy/mysql_test/include/odps_config_for_cloud.inc + + export VAULT_ADDR='https://vault.oceanbase-dev.com/' + export S3_BUCKET=`vault kv get -mount="secret/data/mysqltest/dev" "s3_access_info" | grep bucket | awk '{print \$2}'` + export S3_AK=`vault kv get -mount="secret/data/mysqltest/dev" "s3_access_info" | grep access_key | awk '{print \$2}'` + export S3_SK=`vault kv get -mount="secret/data/mysqltest/dev" "s3_access_info" | grep secret_key | awk '{print \$2}'` + + sed -i "s#\#S3_BUCKET\##s3://${S3_BUCKET}#g" $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + sed -i 's/#S3_REGION#/cn-north-1/g' $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + sed -i 's/#S3_ENDPOINT#/s3.cn-north-1.amazonaws.com.cn/g' $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + sed -i s/'#S3_AK#'/${S3_AK}/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + sed -i "s#\#S3_SK\##${S3_SK}#g" $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + + sed -i 's/#OBS_BUCKET#/s3:\/\/obs-436751-long-data-all-test/g' $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + sed -i 's/#OBS_REGION#/cn-north-4/g' $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + sed -i 's/#OBS_ENDPOINT#/obs.cn-north-4.myhuaweicloud.com/g' $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + sed -i s/'#OBS_AK#'/${SENSITIVE_TEST_OBS_ID_FOR_OBJECT_STORAGE_FOR_TOTAL}/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + sed -i s/'#OBS_SK#'/${SENSITIVE_TEST_OBS_KEY_FOR_OBJECT_STORAGE_FOR_TOTAL}/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + + sed -i 's/#OSS_BUCKET#/oss:\/\/oss-436751-self-all-test/g' $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + sed -i 's/#OSS_ENDPOINT#/oss-cn-hangzhou.aliyuncs.com/g' $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + sed -i s/'#OSS_AK#'/${SENSITIVE_TEST_OSS_ID_FOR_OBJECT_STORAGE}/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + sed -i s/'#OSS_SK#'/${SENSITIVE_TEST_OSS_KEY_FOR_OBJECT_STORAGE}/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + + export COS_BUCKET=`vault kv get -mount="secret/data/mysqltest/dev" "cos_access_info" | grep bucket | awk '{print \$2}'` + export COS_AK=`vault kv get -mount="secret/data/mysqltest/dev" "cos_access_info" | grep access_key | awk '{print \$2}'` + export COS_SK=`vault kv get -mount="secret/data/mysqltest/dev" "cos_access_info" | grep secret_key | awk '{print \$2}'` + + sed -i "s#\#COS_BUCKET\##cos://${COS_BUCKET}#g" $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + sed -i 's/#COS_ENDPOINT#/cos.ap-shanghai.myqcloud.com/g' $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + sed -i 's/#COS_APPID#/1317978819/g' $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + sed -i s/'#COS_AK#'/${COS_AK}/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + sed -i s/'#COS_SK#'/${COS_SK}/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc + set -x fi - mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient $COLLECT_ARG --init-sql-dir=$HOME/seekdb/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS" $mysqltest_cmd $RUN_EXTRA_CHECK_CMD $INIT_FLIES $SCHE_ARGS 2>&1 | tee compare.out && ( exit ${PIPESTATUS[0]}) ret=$? if [[ $JOBNAME == 'mysqltest_opensource' ]] then submarker="_opensource" + elif [[ $JOBNAME == 'mysqltest_sanity' ]] + then + submarker="_sanity" else submarker="" fi - mv compare.out $HOME/mysqltest${submarker}_compare_output.$SLICE_IDX + mv compare.out $HOME/mysqltest${submarker}_compare_output${JOB_SUFFIX}.$SLICE_IDX echo "finish!" + if [[ $ret -ne 0 ]] + then + fail_cases=`python3 $HOME/common/parse_the_case_output_for_case.py mysqltest $HOME/mysqltest${submarker}_compare_output${JOB_SUFFIX}.$SLICE_IDX |xargs` + if [[ $fail_cases ]] + then + python3 $HOME/common/get_cases_run_result.py "$fail_cases" $BRANCH $FORK_COMMIT $HOME/case_whitelist.txt + filter_by_whitelist_res=$? + if [[ $filter_by_whitelist_res -eq 0 ]] + then + echo "CASE 运行失败，但是由于case都在白名单，认为运行成功" + ret=0 + # 还是收集下日志 + obd_collect_log + fi + fi + fi return $ret fi } @@ -708,7 +1003,7 @@ function obd_prepare_env { function obd_collect_log { - echo "collect log" + echo "`date "+%F %T"` [INFO] 开始collect log" cd $HOME/ mkdir -p collected_log mkdir -p collected_log/obd_log @@ -716,18 +1011,22 @@ function obd_collect_log { mkdir -p collected_log/mysqltest_rec_log find $DATA_PATH -name 'core[.-]*' | xargs -i cp {} collected_log [[ "$OBD_HOME" != "" ]] && mv $OBD_HOME/.obd/log/* collected_log/obd_log/ - set +x - if [[ $WITH_SHARED_STORAGE == "1" ]] + mv oceanbase/tools/deploy/var/log/* collected_log/mysqltest_log/ + mv oceanbase/tools/deploy/var/rec_log/* collected_log/mysqltest_rec_log/ + mv collected_log collected_log_$SLICE_IDX + echo "`date "+%F %T"` [INFO] 开始压缩 log" + if [[ -f /data/farm/tools/pigz ]] then - sed -i "s/${SENSITIVE_TEST_OSS_KEY_FOR_OBJECT_STORAGE_FOR_TOTAL}//" collected_log/obd_log/obd + cp /data/farm/tools/pigz $HOME/ && chmod a+x $HOME/pigz + tar -cvf - collected_log_$SLICE_IDX | $HOME/./pigz -1 > collected_log_$SLICE_IDX${JOB_SUFFIX}.tar.gz + else + tar -cvf - collected_log_$SLICE_IDX |gzip -1 > collected_log_$SLICE_IDX${JOB_SUFFIX}.tar.gz fi - set -x - mv seekdb/tools/deploy/var/log/* collected_log/mysqltest_log/ - mv seekdb/tools/deploy/var/rec_log/* collected_log/mysqltest_rec_log/ - mv collected_log collected_log_$SLICE_IDX - tar zcvf collected_log_${SLICE_IDX}${JOB_SUFFIX}.tar.gz collected_log_$SLICE_IDX + echo "`date "+%F %T"` [INFO] 日志收集结束" } +export -f obd_collect_log + function collect_obd_case_log { echo "collect obd case log" @@ -735,6 +1034,9 @@ function collect_obd_case_log { if [[ $JOBNAME == 'mysqltest_opensource' ]] then submarker="_opensource" + elif [[ $JOBNAME == 'mysqltest_sanity' ]] + then + submarker="_sanity" else submarker="" fi @@ -754,25 +1056,17 @@ function run { then mv /root/tmp/.ssh/id_rsa /root/.ssh/ fi - if ([[ -f $HOME/seekdb/rpm/oceanbase.deps ]] && [[ "$(grep 'ob-deploy' $HOME/seekdb/rpm/oceanbase.deps )" != "" ]]) || - ([[ -f $HOME/seekdb/deps/3rd/oceanbase.el7.x86_64.deps ]] && [[ "$(grep 'ob-deploy' $HOME/seekdb/deps/3rd/oceanbase.el7.x86_64.deps )" != "" ]]) || - ([[ -f $HOME/seekdb/deps/init/oceanbase.el7.x86_64.deps ]] && [[ "$(grep 'ob-deploy' $HOME/seekdb/deps/init/oceanbase.el7.x86_64.deps )" != "" ]]) + if ([[ -f $HOME/oceanbase/rpm/oceanbase.deps ]] && [[ "$(grep 'ob-deploy' $HOME/oceanbase/rpm/oceanbase.deps )" != "" ]]) || + ([[ -f $HOME/oceanbase/deps/3rd/oceanbase.el7.x86_64.deps ]] && [[ "$(grep 'ob-deploy' $HOME/oceanbase/deps/3rd/oceanbase.el7.x86_64.deps )" != "" ]]) || + ([[ -f $HOME/oceanbase/deps/init/oceanbase.el7.x86_64.deps ]] && [[ "$(grep 'ob-deploy' $HOME/oceanbase/deps/init/oceanbase.el7.x86_64.deps )" != "" ]]) then - if [[ $WITH_SHARED_STORAGE == "1" ]] - then - # 单独判断下是否有对应的case集合 - if [[ ! -d $HOME/seekdb/sensitive_test/mysql_test ]] - then - return 0 - fi - fi - timeout=18000 + timeout=34800 [[ "$SPECIAL_RUN" == "1" ]] && timeout=72000 obd_prepare_env && timeout $timeout bash -c "obd_run_mysqltest" test_ret=$? error_log_ret=0 - if [[ -f $HOME/seekdb/tools/deploy/error_log_filter.json && $BRANCH == 'master' ]] + if [[ -f $HOME/oceanbase/tools/deploy/error_log_filter.json && $BRANCH == 'master' ]] then # notice the core python $HOME/common/analyse_the_observer_core.py collect -p ${DATA_PATH} -j ${JOBNAME}.${SLICE_IDX} -o $HOME @@ -783,15 +1077,31 @@ function run { then obd_collect_log fi - return `[[ $test_ret = 0 && $error_log_ret = 0 ]]` + return `[[ $test_ret = 0 && $error_log_ret = 0 ]]` + # if [[ $test_ret -ne 0 ]] + # then + # fail_cases=`python3 $HOME/common/parse_the_case_output_for_case.py mysqltest $HOME/mysqltest${submarker}_compare_output${JOB_SUFFIX}.$SLICE_IDX |xargs` + # if [[ $fail_cases ]] + # then + # python3 $HOME/common/get_cases_run_result.py "$fail_cases" $BRANCH $FORK_COMMIT + # filter_by_whitelist_res=$? + # if [[ $filter_by_whitelist_res -eq 0 ]] + # then + # echo "CASE 运行失败，但是由于case都在白名单，认为运行成功" + # test_ret=0 + # fi + # fi + # fi + + # return $test_ret else prepare_env && timeout 18000 bash -c "run_mysqltest" test_ret=$? error_log_ret=0 - if [[ -f $HOME/seekdb/tools/deploy/error_log_filter.json && ( $BRANCH == 'master' || $BRANCH == "4_2_x_release" ) ]] + if [[ -f $HOME/oceanbase/tools/deploy/error_log_filter.json && ( $BRANCH == 'master' || $BRANCH == "4_2_x_release" ) ]] then - python $HOME/common/analyse_the_observer_log.py collect -p ${DATA_PATH}/observer1/log -p ${DATA_PATH}/observer2/log -i $GID -j ${JOBNAME}.${SLICE_IDX} -o $HOME -F $HOME/seekdb/tools/deploy/error_log_filter.json + python $HOME/common/analyse_the_observer_log.py collect -p ${DATA_PATH}/observer1/log -p ${DATA_PATH}/observer2/log -i $GID -j ${JOBNAME}.${SLICE_IDX} -o $HOME -F $HOME/oceanbase/tools/deploy/error_log_filter.json error_log_ret=0 fi if [[ $test_ret -ne 0 || $error_log_ret -ne 0 ]] From 2150c70237b4ff3b7b40ed226bc3007f082691ad Mon Sep 17 00:00:00 2001 From: guoyan1996 <1028109394@qq.com> Date: Sun, 5 Apr 2026 05:40:12 +0800 Subject: [PATCH 08/11] test --- .../seekdb/scripts/mysqltest_for_seekdb.sh | 45 +++++++++++++++---- 1 file changed, 37 insertions(+), 8 deletions(-) diff --git a/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh b/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh index 731841ae8..ad49a73df 100644 --- a/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh +++ b/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh @@ -1,14 +1,39 @@ #!/bin/sh +# Normalise HOME. Some CI/k8s wrappers export HOME=, which yields: +# DOWNLOAD_DIR=$HOME/downloads -> /downloads (often not writable) +# $HOME/oceanbase/... -> /oceanbase/... (wrong path). +if [ -n "${GITHUB_WORKSPACE:-}" ]; then + export HOME="${GITHUB_WORKSPACE}" +elif [ -n "${_CONDOR_JOB_IWD:-}" ]; then + export HOME="${_CONDOR_JOB_IWD}" +elif [ -z "${HOME:-}" ] || [ "${HOME}" = "/" ]; then + _MF_ROOT="$(cd "$(dirname "$0")/../../../.." >/dev/null 2>&1 && pwd)" + if [ -n "$_MF_ROOT" ] && [ -d "$_MF_ROOT/tools/deploy" ]; then + export HOME="$_MF_ROOT" + fi +fi +if [ -z "${HOME:-}" ] && [ -d "$(pwd)/tools/deploy" ]; then + export HOME="$(pwd)" +fi + +# seekdb checkout: tools/deploy lives at repo root; this script uses $HOME/oceanbase/tools/deploy. +if [ -n "${HOME:-}" ] && [ -d "$HOME/tools/deploy" ] && [ ! -e "$HOME/oceanbase" ]; then + (cd "$HOME" && ln -s . oceanbase) +fi + set -x -export HOME=$_CONDOR_JOB_IWD export PATH=/bin:/usr/bin export USER=$(whoami) HOST=`hostname -i` DOWNLOAD_DIR=$HOME/downloads -SLOT_ID=`echo $_CONDOR_SLOT | cut -c5-` +if [ -n "${_CONDOR_SLOT:-}" ]; then + SLOT_ID=`echo "$_CONDOR_SLOT" | cut -c5-` +else + SLOT_ID="${SLICE_IDX:-0}" +fi function prepare_config { if [[ "$MINI" == "1" ]] || ([[ "$MINI" == "-1" ]] && [[ -f $HOME/oceanbase/tools/deploy/enable_mini_mode ]]) @@ -186,7 +211,11 @@ function obd_prepare_global { export LANG=en_US.UTF-8 HOST=`hostname -i` DOWNLOAD_DIR=$HOME/downloads - SLOT_ID=`echo $_CONDOR_SLOT | cut -c5-` + if [[ -n "${_CONDOR_SLOT:-}" ]]; then + SLOT_ID=`echo "$_CONDOR_SLOT" | cut -c5-` + else + SLOT_ID="${SLICE_IDX:-0}" + fi PORT_NUM=`expr 5000 + $SLOT_ID \* 100` # 根据传入的observer binary判断是否开源版 @@ -562,7 +591,7 @@ function obd_prepare_bin { obs_version=`$DOWNLOAD_DIR/bin/observer -V 2>&1 | grep -E "observer \(OceanBase([ \_]CE)? ([.0-9]+)\)" | grep -Eo '([.0-9]+)'` fi - mkdir $HOME/oceanbase/tools/deploy/admin + mkdir -p $HOME/oceanbase/tools/deploy/admin if [[ -d $HOME/oceanbase/src/share/inner_table/sys_package ]] then cp $HOME/oceanbase/src/share/inner_table/sys_package/*.sql $HOME/oceanbase/tools/deploy/admin/ @@ -1009,10 +1038,10 @@ function obd_collect_log { mkdir -p collected_log/obd_log mkdir -p collected_log/mysqltest_log mkdir -p collected_log/mysqltest_rec_log - find $DATA_PATH -name 'core[.-]*' | xargs -i cp {} collected_log - [[ "$OBD_HOME" != "" ]] && mv $OBD_HOME/.obd/log/* collected_log/obd_log/ - mv oceanbase/tools/deploy/var/log/* collected_log/mysqltest_log/ - mv oceanbase/tools/deploy/var/rec_log/* collected_log/mysqltest_rec_log/ + find "$DATA_PATH" -name 'core[.-]*' 2>/dev/null -exec cp -f {} collected_log/ \; || true + [[ "$OBD_HOME" != "" && -d "$OBD_HOME/.obd/log" ]] && mv "$OBD_HOME"/.obd/log/* collected_log/obd_log/ 2>/dev/null || true + [[ -d oceanbase/tools/deploy/var/log ]] && mv oceanbase/tools/deploy/var/log/* collected_log/mysqltest_log/ 2>/dev/null || true + [[ -d oceanbase/tools/deploy/var/rec_log ]] && mv oceanbase/tools/deploy/var/rec_log/* collected_log/mysqltest_rec_log/ 2>/dev/null || true mv collected_log collected_log_$SLICE_IDX echo "`date "+%F %T"` [INFO] 开始压缩 log" if [[ -f /data/farm/tools/pigz ]] From b3eb415d0dc649d63c33bce3a4f8fd2461ce793e Mon Sep 17 00:00:00 2001 From: guoyan1996 <1028109394@qq.com> Date: Sun, 5 Apr 2026 10:42:03 +0800 Subject: [PATCH 09/11] reset --- .../seekdb/scripts/mysqltest_for_seekdb.sh | 741 +++++------------- 1 file changed, 201 insertions(+), 540 deletions(-) diff --git a/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh b/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh index ad49a73df..9b173bce4 100644 --- a/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh +++ b/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh @@ -1,50 +1,49 @@ -#!/bin/sh +#!/bin/bash -# Normalise HOME. Some CI/k8s wrappers export HOME=, which yields: -# DOWNLOAD_DIR=$HOME/downloads -> /downloads (often not writable) -# $HOME/oceanbase/... -> /oceanbase/... (wrong path). -if [ -n "${GITHUB_WORKSPACE:-}" ]; then - export HOME="${GITHUB_WORKSPACE}" -elif [ -n "${_CONDOR_JOB_IWD:-}" ]; then - export HOME="${_CONDOR_JOB_IWD}" -elif [ -z "${HOME:-}" ] || [ "${HOME}" = "/" ]; then - _MF_ROOT="$(cd "$(dirname "$0")/../../../.." >/dev/null 2>&1 && pwd)" - if [ -n "$_MF_ROOT" ] && [ -d "$_MF_ROOT/tools/deploy" ]; then - export HOME="$_MF_ROOT" - fi -fi -if [ -z "${HOME:-}" ] && [ -d "$(pwd)/tools/deploy" ]; then - export HOME="$(pwd)" -fi +set -x -# seekdb checkout: tools/deploy lives at repo root; this script uses $HOME/oceanbase/tools/deploy. -if [ -n "${HOME:-}" ] && [ -d "$HOME/tools/deploy" ] && [ ! -e "$HOME/oceanbase" ]; then - (cd "$HOME" && ln -s . oceanbase) +# 脚本所在目录，末尾 source frame.sh 用 +SEEKDB_SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +# seekdb native：GITHUB_WORKSPACE=仓库根，SLICE_IDX=slice 序号（0,1,2,3） +# 脚本在 .github/script/seekdb/scripts/，仓库根为 4 级上级 +REPO_ROOT_BY_SCRIPT="$(cd "$SEEKDB_SCRIPT_DIR/../../../.." && pwd)" +export GITHUB_WORKSPACE="${GITHUB_WORKSPACE:-$REPO_ROOT_BY_SCRIPT}" +# 若传入的 GITHUB_WORKSPACE 在容器内无 build.sh（如 runner 传主机路径），改用实际存在的仓库根 +if [[ ! -f "$GITHUB_WORKSPACE/build.sh" ]]; then + if [[ -f "$REPO_ROOT_BY_SCRIPT/build.sh" ]]; then + export GITHUB_WORKSPACE="$REPO_ROOT_BY_SCRIPT" + elif [[ -f "$(pwd)/build.sh" ]]; then + export GITHUB_WORKSPACE="$(pwd)" + fi +fi +export USER="${USER:-$(whoami)}" +SEEKDB_HOME="${GITHUB_WORKSPACE}/seekdb_home_${SLICE_IDX:-0}" +mkdir -p "$SEEKDB_HOME" +ln -sf "$GITHUB_WORKSPACE" "$SEEKDB_HOME/seekdb" +export HOME="$SEEKDB_HOME" +export SLOT_ID="${SLICE_IDX:-0}" +# 保证 HOME/observer 存在：优先用 WORKSPACE 下 observer，否则解压 observer.zst +if [[ -x "$GITHUB_WORKSPACE/observer" ]]; then + cp -f "$GITHUB_WORKSPACE/observer" "$HOME/observer" && chmod +x "$HOME/observer" +elif [[ -f "$GITHUB_WORKSPACE/observer.zst" ]]; then + (command -v zstd >/dev/null 2>&1 && zstd -d -f -o "$HOME/observer" "$GITHUB_WORKSPACE/observer.zst" && chmod +x "$HOME/observer") || true fi - -set -x export PATH=/bin:/usr/bin -export USER=$(whoami) - -HOST=`hostname -i` -DOWNLOAD_DIR=$HOME/downloads -if [ -n "${_CONDOR_SLOT:-}" ]; then - SLOT_ID=`echo "$_CONDOR_SLOT" | cut -c5-` -else - SLOT_ID="${SLICE_IDX:-0}" -fi +HOST=$(hostname -i) +DOWNLOAD_DIR=$HOME/seekdb/tools/deploy function prepare_config { - if [[ "$MINI" == "1" ]] || ([[ "$MINI" == "-1" ]] && [[ -f $HOME/oceanbase/tools/deploy/enable_mini_mode ]]) + if [[ "$MINI" == "1" ]] || ([[ "$MINI" == "-1" ]] && [[ -f $HOME/seekdb/tools/deploy/enable_mini_mode ]]) then - [[ -f $HOME/oceanbase/tools/deploy/enable_mini_mode ]] && MINI_SIZE=$(cat $HOME/oceanbase/tools/deploy/enable_mini_mode) + [[ -f $HOME/seekdb/tools/deploy/enable_mini_mode ]] && MINI_SIZE=$(cat $HOME/seekdb/tools/deploy/enable_mini_mode) [[ $MINI_SIZE =~ ^[0-9]+G ]] || MINI_SIZE="8G" MINI_CONFIG_ITEM="ObCfg.init_config['memory_limit']='$MINI_SIZE'" fi - cd $HOME/oceanbase/tools/deploy + cd $HOME/seekdb/tools/deploy if [ "$WITH_PROXY" ] && [ "$WITH_PROXY" != "0" ] then if [ "$CLUSTER_SPEC" == '2x1' ] @@ -61,6 +60,8 @@ function prepare_config { SPEC="[$HOST]@zone1" fi fi + # farm中由于机器变小了，需要设置下 + CPU_COUNT_CONFIG="ObCfg.init_config['cpu_count']='24'" cat >config7.py < test.fifo 2>&1 elif [[ "$SLAVE" == "1" ]] then - cat test.fifo | tee result.out & ./hap.py $USER.obs1.mysqltest collect_all slices=$SLICES slice_idx=$SLICE_IDX $slb `echo $ARGV` 1> test.fifo 2>&1 + cat test.fifo | tee result.out & ./hap.py $USER.obs1.mysqltest collect_all slices=$SLICES slice_idx=$SLICE_IDX $slb `echo $ARGV` 1> test.fifo 2>&1 else cat test.fifo | tee result.out & ./hap.py $USER.mysqltest collect_all slices=$SLICES slice_idx=$SLICE_IDX $slb `echo $ARGV` 1> test.fifo 2>&1 fi @@ -159,14 +141,14 @@ function run_mysqltest { # check if there is error if [ "$?" == "0" ] then - grep -E 'PASSED' $HOME/oceanbase/tools/deploy/result.out > /dev/null && ! grep -E "FAIL LST" $HOME/oceanbase/tools/deploy/result.out > /dev/null || return 1 + grep -E 'PASSED' $HOME/seekdb/tools/deploy/result.out > /dev/null && ! grep -E "FAIL LST" $HOME/seekdb/tools/deploy/result.out > /dev/null || return 1 else return 1 fi } function collect_log { - cd $HOME/oceanbase/tools/deploy || return 1 + cd $HOME/seekdb/tools/deploy || return 1 if [ -d "collected_log" ] then mv mysql_test/var/log/* collected_log @@ -179,25 +161,25 @@ function collect_log { export -f run_mysqltest function obd_prepare_obd { - if [[ -f $HOME/oceanbase/deps/init/dep_create.sh ]] + if [[ -f $HOME/seekdb/deps/init/dep_create.sh ]] then if [[ "$IS_CE" == "1" ]] then - cd $HOME/oceanbase && ./build.sh init --ce || return 3 + cd $HOME/seekdb && ./build.sh init --ce || return 3 else - cd $HOME/oceanbase && ./build.sh init || return 3 + cd $HOME/seekdb && ./build.sh init || return 3 fi else - if grep 'dep_create.sh' $HOME/oceanbase/build.sh + if grep 'dep_create.sh' $HOME/seekdb/build.sh then - cd $HOME/oceanbase/deps/3rd && bash dep_create.sh all || return 4 + cd $HOME/seekdb/deps/3rd && bash dep_create.sh all || return 4 else - cd $HOME/oceanbase && ./build.sh init || return 3 + cd $HOME/seekdb && ./build.sh init || return 3 fi fi $obd devmode enable - $obd env set OBD_DEPLOY_BASE_DIR $HOME/oceanbase/tools/deploy + $obd env set OBD_DEPLOY_BASE_DIR $HOME/seekdb/tools/deploy $obd --version if [[ "$IS_CE" == "1" && ! -f $OBD_HOME/.obd/mirror/remote/taobao.repo ]] then @@ -209,33 +191,23 @@ export -f obd_prepare_obd function obd_prepare_global { export LANG=en_US.UTF-8 - HOST=`hostname -i` + HOST=$(hostname -i) DOWNLOAD_DIR=$HOME/downloads - if [[ -n "${_CONDOR_SLOT:-}" ]]; then - SLOT_ID=`echo "$_CONDOR_SLOT" | cut -c5-` - else - SLOT_ID="${SLICE_IDX:-0}" - fi - PORT_NUM=`expr 5000 + $SLOT_ID \* 100` + export SLOT_ID="${SLOT_ID:-0}" + PORT_NUM=$((5000 + SLOT_ID * 100)) - # 根据传入的observer binary判断是否开源版 - if [[ `$HOME/observer -V 2>&1 | grep -E '(OceanBase CE|OceanBase_CE)'` ]] - then - COMPONENT="oceanbase-ce" - export IS_CE=1 - else - COMPONENT="oceanbase" - export IS_CE=0 - fi + # SeekDB CI：固定按 CE 路径走 OBD/init（原先用 observer -V 判 CE，SeekDB 输出不含 CE 字样会被判成 0） + COMPONENT="seekdb" + export IS_CE=1 # 根据build.sh中的内容判断依赖安装路径 - if grep 'dep_create.sh' $HOME/oceanbase/build.sh + if grep 'dep_create.sh' $HOME/seekdb/build.sh then - DEP_PATH=$HOME/oceanbase/deps/3rd + DEP_PATH=$HOME/seekdb/deps/3rd else - DEP_PATH=$HOME/oceanbase/rpm/.dep_create/var + DEP_PATH=$HOME/seekdb/rpm/.dep_create/var fi - if [[ -f "$HOME/oceanbase/tools/deploy/mysqltest_config.yaml" ]] + if [[ -f "$HOME/seekdb/tools/deploy/mysqltest_config.yaml" ]] then export WITH_MYSQLTEST_CONFIG_YAML=1 else @@ -245,22 +217,22 @@ function obd_prepare_global { ob_name=obcluster$SLOT_ID app_name=$ob_name.$USER.$HOST ocp_config_server='http://ocp-cfg.alibaba.net:8080/services?User_ID=alibaba&UID=test' - proxy_cfg_url=${ocp_config_server}\&Action=GetObProxyConfig\&ObRegionGroup=$app_name cfg_url=${ocp_config_server} export obd=$DEP_PATH/usr/bin/obd export OBD_HOME=$HOME export OBD_INSTALL_PRE=$DEP_PATH export DATA_PATH=$HOME/data + [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh } function obd_prepare_config { MINI_SIZE="10G" - if [[ "$MINI" == "1" ]] || ([[ "$MINI" == "-1" ]] && [[ -f $HOME/oceanbase/tools/deploy/enable_mini_mode ]]) + if [[ "$MINI" == "1" ]] || ([[ "$MINI" == "-1" ]] && [[ -f $HOME/seekdb/tools/deploy/enable_mini_mode ]]) then - [[ -f $HOME/oceanbase/tools/deploy/enable_mini_mode ]] && MINI_SIZE=$(cat $HOME/oceanbase/tools/deploy/enable_mini_mode) + [[ -f $HOME/seekdb/tools/deploy/enable_mini_mode ]] && MINI_SIZE=$(cat $HOME/seekdb/tools/deploy/enable_mini_mode) [[ $MINI_SIZE =~ ^[0-9]+G ]] || MINI_SIZE="8G" fi if [[ "$CLUSTER_SPEC" == '2x1' ]] || [[ "$SLAVE" == "1" ]] @@ -303,74 +275,23 @@ EOF ) fi export MYSQL_PORT=$mysql_port - proxy_conf="" - if [ "$WITH_PROXY" ] && [ "$WITH_PROXY" != "0" ] - then - listen_port=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1` - prometheus_listen_port=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1` - - proxy_conf=$(cat $HOME/oceanbase/tools/deploy/obd/obproxy.yaml.template) - if [[ -f $HOME/oceanbase/test/obproxy_test_config ]] - then - obproxy_switch=`sed '/^obproxy_switch=/!d;s/.*=//' $HOME/oceanbase/test/obproxy_test_config` - - # 当且仅当 obproxy 指定模式为wget时，影响mysqltest obproxy配置，否则mysqltest依旧沿用原有obproxy获取逻辑 - mkdir $HOME/bin - if [[ -f $HOME/obproxy ]] - then - cp $HOME/obproxy $HOME/bin - else - wget_url=`sed '/^wget_url=/!d;s/.*=//' $HOME/oceanbase/test/obproxy_test_config` - wget $wget_url -O $HOME/bin/obproxy - if [[ -f $HOME/bin/obproxy ]] - then - echo "obproxy 文件下载成功" - chmod +x $HOME/bin/obproxy - - else - # 下载失败，沿用原有逻辑 - echo "obproxy 文件下载失败" - return 1 - fi - fi - - echo "obrpxoy版本:" - echo `$HOME/bin/obproxy -V 2>&1` - - obproxy_version=$("$HOME"/bin/obproxy --version 2>&1 | grep -E 'obproxy \(OceanBase [\.0-9]+ \S+\)' | grep -Eo '\s[.0-9]+\s') - - $obd mirror create -n obproxy -p $HOME -t "latest" -V $obproxy_version - echo "执行结果: $?" - - proxy_conf=$(echo "$proxy_conf" | sed '/package_hash: [0-9a-z]*/d') - proxy_conf="""$proxy_conf - tag: latest -""" - fi - proxy_conf=${proxy_conf//'{{%% COMPONENT %%}}'/$COMPONENT} - proxy_conf=${proxy_conf//'{{%% LISTEN_PORT %%}}'/$listen_port} - proxy_conf=${proxy_conf//'{{%% PROMETHEUS_LISTEN_PORT %%}}'/$prometheus_listen_port} - proxy_conf=${proxy_conf//'{{%% OBPORXY_HOME_PATH %%}}'/$DATA_PATH\/obproxy} - proxy_conf=${proxy_conf//'{{%% OBPROXY_CONFIG_SERVER_URL %%}}'/$proxy_cfg_url} - fi - if [[ "$MINI" == "1" && -f $HOME/oceanbase/tools/deploy/obd/core-test.yaml.template ]] + if [[ "$MINI" == "1" && -f $HOME/seekdb/tools/deploy/obd/core-test.yaml.template ]] then - conf=$(cat $HOME/oceanbase/tools/deploy/obd/core-test.yaml.template) + conf=$(cat $HOME/seekdb/tools/deploy/obd/core-test.yaml.template) else - conf=$(cat $HOME/oceanbase/tools/deploy/obd/config.yaml.template) + conf=$(cat $HOME/seekdb/tools/deploy/obd/config.yaml.template) fi # cgroup support - if [[ -f $HOME/oceanbase/tools/deploy/obd/.use_cgroup || $(echo " $ARGV " | grep " use-cgroup ") ]] + if [[ -f $HOME/seekdb/tools/deploy/obd/.use_cgroup || $(echo " $ARGV " | grep " use-cgroup ") ]] then CGROUP_CONFIG="cgroup_dir: /sys/fs/cgroup/cpu/$USER" fi - # MYSQLRTEST_ARGS contains changes for special run - proxy_conf="""$MYSQLRTEST_ARGS + extra_conf="""$MYSQLRTEST_ARGS $CGROUP_CONFIG -$proxy_conf""" - conf=${conf//'{{%% PROXY_CONF %%}}'/"$proxy_conf"} - conf=${conf//'{{%% DEPLOY_PATH %%}}'/"$HOME/oceanbase/tools/deploy"} +""" + conf=${conf//'{{%% PROXY_CONF %%}}'/"$extra_conf"} + conf=${conf//'{{%% DEPLOY_PATH %%}}'/"$HOME/seekdb/tools/deploy"} conf=${conf//'{{%% COMPONENT %%}}'/$COMPONENT} conf=${conf//'{{%% SERVERS %%}}'/$SERVERS} conf=${conf//'{{%% TAG %%}}'/'latest'} @@ -382,233 +303,86 @@ $proxy_conf""" then conf=$(echo "$conf" | sed 's/oceanbase:/oceanbase\-ce:/g' | sed 's/\- oceanbase$/- oceanbase-ce/g') fi - echo "$conf" > $HOME/oceanbase/tools/deploy/config.yaml + echo "$conf" > $HOME/seekdb/tools/deploy/config.yaml + + if [[ $WITH_SHARED_STORAGE == "1" ]] + then + # shared_storage 需要在global修改相关的mode + share_storage_workdir="$(date +%s)-$(uuidgen)" + first_global_line_count=`sed -n '/global:/=' $HOME/seekdb/tools/deploy/config.yaml | head -n 1` + set +x + sed -i "${first_global_line_count}a\ mode: 'shared_storage'\n\ + shared_storage: \"oss://oss-436751-0701-all-test/$FARM2_RUN_USER/$share_storage_workdir?host=oss-cn-hangzhou.aliyuncs.com&access_id=${SENSITIVE_TEST_OSS_ID_FOR_OBJECT_STORAGE_FOR_TOTAL}&access_key=${SENSITIVE_TEST_OSS_KEY_FOR_OBJECT_STORAGE_FOR_TOTAL}&max_iops=0&max_bandwidth=0B&scope=region\"" $HOME/seekdb/tools/deploy/config.yaml + sed -i "s/datafile_size: '20G'/datafile_size: '200G'/g" $HOME/seekdb/tools/deploy/config.yaml + set -x + fi + if [[ $SHARED_STORAGE_MODE == "1" ]] then # shared_storage 需要在global修改相关的mode - sh $HOME/common/start_farm_minio.sh $HOME/minio_home /data/farm/tools/minio /data/farm/tools/mc - local_mino_res=$? - MINIO_IP="127.0.0.1" - if [[ $local_mino_res -ne 0 ]] + MINIO_IP="100.88.99.213" + if [[ $TEST_NAME == "farm2" ]] then MINIO_IP="100.88.99.213" - if [[ $TEST_NAME == "farm2" ]] - then - minio_ips=("100.88.99.213" "100.88.99.214") - random_index=$(( $RANDOM % ${#minio_ips[@]} )) - MINIO_IP=${minio_ips[$random_index]} - echo "CURRENT MINIO IP $MINIO_IP" - else - minio_ips=("100.88.99.213" "100.88.99.214") - random_index=$(( $RANDOM % ${#minio_ips[@]} )) - MINIO_IP=${minio_ips[$random_index]} - echo "CURRENT MINIO IP $MINIO_IP" - fi fi share_storage_workdir="$(date +%s)-$(uuidgen)" - first_global_line_count=`sed -n '/global:/=' $HOME/oceanbase/tools/deploy/config.yaml | head -n 1` + first_global_line_count=`sed -n '/global:/=' $HOME/seekdb/tools/deploy/config.yaml | head -n 1` sed -i "${first_global_line_count}a\ mode: 'shared_storage'\n\ - shared_storage: \"s3://farm-test/mysqltest/$share_storage_workdir/clog_and_data?host=http://$MINIO_IP:9000&access_id=minioadmin&access_key=minioadmin&s3_region=us-east-1&max_iops=10000&max_bandwidth=1GB&scope=region\"\n\ - random_shared_storage_bucket: \"1\"" $HOME/oceanbase/tools/deploy/config.yaml - sed -i "s/# datafile_size: '20G'/datafile_size: '200G'/g" $HOME/oceanbase/tools/deploy/config.yaml - sed -i "s/datafile_size: '20G'/datafile_size: '200G'/g" $HOME/oceanbase/tools/deploy/config.yaml - need_oblogservice=`cat $HOME/oceanbase/test/farm.yaml | grep -Pz "mysqltest:\n\s+need_oblogservice:\s+true"` - if [[ $need_oblogservice ]] - then - first_global_line_count=`sed -n '/global:/=' $HOME/oceanbase/tools/deploy/config.yaml | head -n 1` - sed -i "${first_global_line_count}a\ logservice_cluster_id: 1234\n\ - object_store_url: s3://farm-test/?host=http://$MINIO_IP:9000&access_id=minioadmin&access_key=minioadmin&s3_region=us-east-1&max_iops=10000&max_bandwidth=1GB&scope=region\n\ - enable_logservice: true" $HOME/oceanbase/tools/deploy/config.yaml - for i in 1 2 3; - do - export oblogservice_port${i}=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1`; - export oblogservice_rpc_port${i}=$PORT_NUM && PORT_NUM=`expr $PORT_NUM + 1`; - done - sed -i "1i\\ -oblogservice:\\ - servers:\\ - - name: node1\\ - ip: 127.0.0.1 # 启动IP地址\\ - - name: node2\\ - ip: 127.0.0.1 # 启动IP地址\\ - - name: node3\\ - ip: 127.0.0.1 # 启动IP地址\\ - node1:\\ - rpc_port: $oblogservice_rpc_port1 # RPC协议端口号\\ - http_port: $oblogservice_port1 # http端口\\ - region: beijing\\ - home_path: $DATA_PATH/logservice/node1 # 部署路径，默认oceanbase, home_path\\ - local_storage_dir: $DATA_PATH/logservice/node1/store # 默认store\\ - node2:\\ - rpc_port: $oblogservice_rpc_port2 # RPC协议端口号\\ - http_port: $oblogservice_port2 # http端口\\ - region: beijing\\ - home_path: $DATA_PATH/logservice/node2 # 部署路径，默认oceanbase, home_path\\ - local_storage_dir: $DATA_PATH/logservice/node2/store # 默认store\\ - node3:\\ - rpc_port: $oblogservice_rpc_port3 # RPC协议端口号\\ - http_port: $oblogservice_port3 # http端口\\ - region: beijing\\ - home_path: $DATA_PATH/logservice/node3 # 部署路径，默认oceanbase, home_path\\ - local_storage_dir: $DATA_PATH/logservice/node3/store # 默认store\\ - global:\\ - cluster_id: 1234 # cluster_id\\ - object_store_url: s3://farm-test/?host=http://$MINIO_IP:9000&access_id=minioadmin&access_key=minioadmin&s3_region=us-east-1&max_iops=10000&max_bandwidth=1GB&scope=region" $HOME/oceanbase/tools/deploy/config.yaml - # 修改内存增加1G - current_value=$(grep -Eo "memory_limit: ['\"]?[0-9]+G['\"]?" $HOME/oceanbase/tools/deploy/config.yaml | grep -Eo '[0-9]+') - # 计算新值 - new_value=$((current_value + 5)) - # 替换文件中的值（兼容单/双引号） - sed -i "s/memory_limit: ['\"]${current_value}G['\"]/memory_limit: '${new_value}G'/" $HOME/oceanbase/tools/deploy/config.yaml - # 修改内存增加1G - current_system_memory_value=$(grep -Eo "system_memory: ['\"]?[0-9]+G['\"]?" $HOME/oceanbase/tools/deploy/config.yaml | grep -Eo '[0-9]+') - # 计算新值 - new_system_memory_value=$((current_system_memory_value + 2)) - # 替换文件中的值（兼容单/双引号） - sed -i "s/system_memory: ['\"]${current_system_memory_value}G['\"]/system_memory: '${new_system_memory_value}G'/" $HOME/oceanbase/tools/deploy/config.yaml - - first_oceanbase_line_count=`sed -n '/oceanbase:/=' $HOME/oceanbase/tools/deploy/config.yaml | head -n 1` - sed -i "${first_oceanbase_line_count}a\ depends:\n\ - - oblogservice" $HOME/oceanbase/tools/deploy/config.yaml - fi + shared_storage: \"s3://farm-test/mysqltest/$share_storage_workdir/clog_and_data?host=http://$MINIO_IP:9000&access_id=minioadmin&access_key=minioadmin&s3_region=us-east-1&max_iops=10000&max_bandwidth=1GB&scope=region\"" $HOME/seekdb/tools/deploy/config.yaml fi - cat $HOME/oceanbase/tools/deploy/config.yaml -} -function get_baseurl { - repo_file=$OBD_HOME/.obd/mirror/remote/taobao.repo - start=0 - end=$(cat $repo_file | wc -l ) - for line in $(grep -nE "^\[.*\]$" $repo_file) - do - num=$(echo "$line" | awk -F ":" '{print $1}') - [ "$find_section" == "1" ] && end=$num - is_target_section=$(echo $(echo "$line" | grep -E "\[\s*$repository\s*\]")) - [ "$is_target_section" != "" ] && start=$num && find_section='1' - done - repo_section=$(awk "NR>=$start && NR<=$end" $repo_file) - baseurl=$(echo "$repo_section" | grep baseurl | awk -F '=' '{print $2}') -} -function get_version_and_release { - start=0 - end=$(cat $config_yaml | wc -l) - for line in $(grep -nE '^\S+:' $config_yaml) - do - num=$(echo "$line" | awk -F ":" '{print $1}') - [ "$find_obproxy" == "1" ] && end=$num - [ "$(echo "$line" | grep obproxy)" != "" ] && start=$num && find_obproxy='1' - done - odp_conf=$(awk "NR>=$start && NR<=$end" $config_yaml) - version=$(echo "$odp_conf" | grep version | awk '{print $2}') - release=$(echo "$odp_conf" | grep release | awk '{print $2}') - include_file=$DEPLOY_PATH/$(echo "$odp_conf"| grep 'include:'| awk '{print $2}') - if [[ -f $include_file ]] + if [[ ! $WITH_SHARED_STORAGE == "1" ]] then - _repo=$(echo $(grep '__mirror_repository_section_name:' $include_file | awk '{print $2}')) - [ "$_repo" != "" ] && repository=$_repo - version=$(echo $(grep 'version:' $include_file | awk '{print $2}')) - release=$(echo $(grep 'release:' $include_file | awk '{print $2}')) + cat $HOME/seekdb/tools/deploy/config.yaml fi } -function get_obproxy { - repository="taobao" - config_yaml=$HOME/oceanbase/tools/deploy/config.yaml - DEPLOY_PATH=$HOME/oceanbase/tools/deploy - obproxy_mirror_repository=$(echo $(grep '__mirror_repository_section_name' $config_yaml | awk -F':' '{print $2}')) - [ "$obproxy_mirror_repository" != "" ] && repository=$obproxy_mirror_repository - - get_version_and_release - get_baseurl - $obd mirror disable remote - - if [[ "$baseurl" != "" && "$version" != "" && "$release" != "" ]] - then - OS_ARCH="$(uname -m)" - if [[ $OS_ARCH == "aarch64" ]] - then - pkg_name="obproxy-$version-$release.aarch64.rpm" - else - pkg_name="obproxy-$version-$release.x86_64.rpm" - fi - if [ "$(find $OBD_HOME/.obd/mirror/local -name $pkg_name)" == "" ] - then - download_dir=$OBD_HOME/.obd_download - mkdir -p $download_dir - wget $baseurl/obproxy/$pkg_name -O "$download_dir/$pkg_name" -o $download_dir/obproxy.down - $obd mirror clone "$download_dir/$pkg_name" && rm -rf $download_dir && return 0 - else - return 0 - fi - fi - echo "Download pkg failed. Try to get obproxy from repository $repository" - count=0 - interval=5 - retry_limit=100 - while (( $count < $retry_limit )) - do - $obd mirror disable remote - $obd mirror enable $repository - $obd mirror update && $obd mirror list $repository > /dev/null && return 0 - count=`expr $count + 1` - (( $count <= $retry_limit )) && sleep $interval - done - return 1 -} - function obd_prepare_bin { cd $HOME mkdir -p $DOWNLOAD_DIR/{bin,etc,admin} || return 1 - if [ "$WITH_PROXY" ] && [ "$WITH_PROXY" != "0" ] - then - get_obproxy || (echo "failed to get obdproxy" || return 1) - fi - if [ ! -x observer ] then ( cd $DOWNLOAD_DIR/bin && - wget http://11.166.86.153:8877/observer -O observer && + wget http://6.12.233.4:5000/observer -O observer && chmod +x observer ) || return 1 else cp observer $DOWNLOAD_DIR/bin/ || return 2 fi - - cd $HOME/oceanbase/tools/deploy + + if [[ "$COMPONENT" == "seekdb" ]] && [[ -x $DOWNLOAD_DIR/bin/observer ]] + then + cp $DOWNLOAD_DIR/bin/observer $DOWNLOAD_DIR/bin/seekdb && chmod +x $DOWNLOAD_DIR/bin/seekdb + fi + + + cd $HOME/seekdb/tools/deploy + [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh + [[ -x $HOME/seekdb/tools/deploy/obd.sh ]] && ./obd.sh prepare cp $DEP_PATH/u01/obclient/bin/obclient ./obclient && chmod 777 obclient cp $DEP_PATH/u01/obclient/bin/mysqltest ./mysqltest && chmod 777 mysqltest - + if [ "$WITH_DEPS" ] && [ "$WITH_DEPS" != "0" ] then - cd $HOME/oceanbase/tools/deploy && [[ -f copy.sh ]] && sh copy.sh + cd $HOME/seekdb/tools/deploy && [[ -f copy.sh ]] && sh copy.sh fi - if [[ -f "$HOME/oceanbase/tools/deploy/obd/.observer_obd_plugin_version" ]] + if [[ -f "$HOME/seekdb/tools/deploy/obd/.observer_obd_plugin_version" ]] then - obs_version=$(cat $HOME/oceanbase/tools/deploy/obd/.observer_obd_plugin_version) + obs_version=$(cat $HOME/seekdb/tools/deploy/obd/.observer_obd_plugin_version) else - obs_version=`$DOWNLOAD_DIR/bin/observer -V 2>&1 | grep -E "observer \(OceanBase([ \_]CE)? ([.0-9]+)\)" | grep -Eo '([.0-9]+)'` + $DOWNLOAD_DIR/bin/observer -V + obs_version=`$DOWNLOAD_DIR/bin/observer -V 2>&1 | grep -E "(observer|seekdb) \(OceanBase([ \_]CE|[ \_]Lite|[ \_]SeekDB|[ \_]seekdb)? [0-9]" | grep -Eo '[0-9]+\.[0-9]+\.[0-9]+(\.[0-9]+)?' | head -1` fi - mkdir -p $HOME/oceanbase/tools/deploy/admin - if [[ -d $HOME/oceanbase/src/share/inner_table/sys_package ]] + mkdir -p $HOME/seekdb/tools/deploy/admin + if [[ -d $HOME/seekdb/src/share/inner_table/sys_package ]] then - cp $HOME/oceanbase/src/share/inner_table/sys_package/*.sql $HOME/oceanbase/tools/deploy/admin/ + cp $HOME/seekdb/src/share/inner_table/sys_package/*.sql $HOME/seekdb/tools/deploy/admin/ fi $obd mirror create -n $COMPONENT -p $DOWNLOAD_DIR -t latest -V $obs_version || return 2 - - need_oblogservice=`cat $HOME/oceanbase/test/farm.yaml | grep -Pz "mysqltest:\n\s+need_oblogservice:\s+true"` - if [[ -x $HOME/oblogservice && -f $HOME/oceanbase/test/oblogservice_test_config && $need_oblogservice ]] - then - echo "存在指定oblogservice, 指定版本为" - oblogservice_version=`$HOME/oblogservice -V 2>&1 | grep -E "oblogservice \(ObLogService: ([.0-9]+)\)" | grep -Eo '([.0-9]+)'` - - cp $HOME/oblogservice $DOWNLOAD_DIR/bin/ || return 5 - $obd mirror create -n oblogservice -p $DOWNLOAD_DIR -t latest -V $oblogservice_version || return 2 - - fi } @@ -618,8 +392,10 @@ function obd_init_cluster { do if [[ "$retries" == "$reboot_retries" ]] then - $obd cluster deploy $ob_name -c $HOME/oceanbase/tools/deploy/config.yaml -f && $obd cluster start $ob_name -f && $obd cluster display $ob_name + [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh + $obd cluster deploy $ob_name -c $HOME/seekdb/tools/deploy/config.yaml -f && $obd cluster start $ob_name -f && $obd cluster display $ob_name else + [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh $obd cluster redeploy $ob_name -f fi retries=`expr $retries - 1` @@ -630,16 +406,16 @@ function obd_init_cluster { init_files=('init.sql' 'init_mini.sql' 'init_for_ce.sql') for init_file in ${init_files[*]} do - if [[ -f $HOME/oceanbase/tools/deploy/$init_file ]] + if [[ -f $HOME/seekdb/tools/deploy/$init_file ]] then - if ! grep "alter system set_tp tp_no = 509, error_code = 4016, frequency = 1;" $HOME/oceanbase/tools/deploy/$init_file + if ! grep "alter system set_tp tp_no = 509, error_code = 4016, frequency = 1;" $HOME/seekdb/tools/deploy/$init_file then - echo -e "\nalter system set_tp tp_no = 509, error_code = 4016, frequency = 1;" >> $HOME/oceanbase/tools/deploy/$init_file + echo -e "\nalter system set_tp tp_no = 509, error_code = 4016, frequency = 1;" >> $HOME/seekdb/tools/deploy/$init_file fi fi done fi - $obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient --init-only --init-sql-dir=$HOME/oceanbase/tools/deploy $INIT_FLIES --log-dir=$HOME/init_case $COLLECT_ARG $EXTRA_ARGS -v > ./init_sql_log && init_seccess=1 && break + $obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient --init-only --init-sql-dir=$HOME/seekdb/tools/deploy $INIT_FLIES --log-dir=$HOME/init_case $EXTRA_ARGS -v > ./init_sql_log && init_seccess=1 && break done } @@ -659,7 +435,8 @@ function obd_run_mysqltest { set -x obd_prepare_global - cd $HOME/oceanbase/tools/deploy + cd $HOME/seekdb/tools/deploy + [[ -f $HOME/seekdb/tools/deploy/activate_obd.sh ]] && source $HOME/seekdb/tools/deploy/activate_obd.sh if [[ "$SLB" != "" ]] && [[ "$EXECID" != "" ]] then SCHE_ARGS="--slb-host=$SLB --exec-id=$EXECID " @@ -673,7 +450,7 @@ function obd_run_mysqltest { INIT_FLIES="--init-sql-files=core-test.init_mini.sql,init_user.sql|root@mysql|test" [ -f init_user_oracle.sql ] && INIT_FLIES="${INIT_FLIES},init_user_oracle.sql|SYS@oracle|SYS" else - if [[ "$IS_CE" == "1" && -f $HOME/oceanbase/.ce ]] + if [[ "$IS_CE" == "1" && -f $HOME/seekdb/.ce ]] then INIT_FLIES="--init-sql-files=init_mini.sql,init_user.sql|root@mysql|test " fi @@ -697,19 +474,11 @@ function obd_run_mysqltest { else EXTRA_ARGS="--cluster-mode=$CLUSTER_MODE " fi - if [[ -f $HOME/oceanbase/tools/deploy/obd/.fast-reboot ]] + if [[ -f $HOME/seekdb/tools/deploy/obd/.fast-reboot ]] then EXTRA_ARGS="${EXTRA_ARGS}--fast-reboot " fi - if [[ $SHARED_STORAGE_MODE == "1" ]] - then - EXTRA_ARGS="${EXTRA_ARGS}--fast-reboot=false " - elif [[ -f $HOME/oceanbase/tools/deploy/obd/.fast-reboot ]] - then - EXTRA_ARGS="${EXTRA_ARGS}--fast-reboot " - fi - if [[ "$IS_CE" == "1" ]] then EXTRA_ARGS="${EXTRA_ARGS}--mode=mysql " @@ -741,17 +510,10 @@ function obd_run_mysqltest { if [[ "$disable_collect" != "1" ]] then COLLECT_ARG="--collect-all" - if [[ "$WITH_MYSQLTEST_CONFIG_YAML" == "0" ]] && [ "$WITH_PROXY" ] && [ "$WITH_PROXY" != "0" ] && [[ -f "$HOME/oceanbase/tools/deploy/obd/.collect_proxy_log" ]] + if [[ "$WITH_MYSQLTEST_CONFIG_YAML" == "0" ]] && [ "$WITH_PROXY" ] && [ "$WITH_PROXY" != "0" ] && [[ -f "$HOME/seekdb/tools/deploy/obd/.collect_proxy_log" ]] then COLLECT_ARG="$COLLECT_ARG --collect-components=$COMPONENT,obproxy" fi - # 由于config.yaml 会给这部分默认设置一个proxy模式下的配置，并且没办法被覆盖，因此先单独在这里处理 - need_oblogservice=`cat $HOME/oceanbase/test/farm.yaml | grep -Pz "mysqltest:\n\s+need_oblogservice:\s+true"` - if [[ $need_oblogservice && $SHARED_STORAGE_MODE == "1" ]] - then - COLLECT_ARG="$COLLECT_ARG --collect-components=$COMPONENT,obproxy,oblogservice" - fi - fi obd_init_cluster if [[ "$init_seccess" != "1" ]] @@ -759,13 +521,9 @@ function obd_run_mysqltest { cat ./init_sql_log echo "Failed to init sql, see more log at ./collected_log/_test_init" mkdir -p $HOME/collected_log/_test_init/ - [[ -d "$HOME/init_case" ]] && mv $HOME/init_case $HOME/collected_log/ - [[ -d "$DATA_PATH/observer1/log" ]] && mkdir -p $HOME/collected_log/_test_init/observer1 && cp -r $DATA_PATH/observer1/log/* $HOME/collected_log/_test_init/observer1 - [[ -d "$DATA_PATH/observer2/log" ]] && mkdir -p $HOME/collected_log/_test_init/observer2 && cp -r $DATA_PATH/observer2/log/* $HOME/collected_log/_test_init/observer2 - [[ -d "$DATA_PATH/obproxy/log" ]] && mkdir -p $HOME/collected_log/_test_init/obproxy && cp -r $DATA_PATH/obproxy/log/* $HOME/collected_log/_test_init/obproxy - [[ -d "$DATA_PATH/logservice/node1/log" ]] && mkdir -p $HOME/collected_log/_test_init/logservice1 && cp -r $DATA_PATH/logservice/node1/log/* $HOME/collected_log/_test_init/logservice1 - [[ -d "$DATA_PATH/logservice/node2/log" ]] && mkdir -p $HOME/collected_log/_test_init/logservice2 && cp -r $DATA_PATH/logservice/node2/log/* $HOME/collected_log/_test_init/logservice2 - [[ -d "$DATA_PATH/logservice/node3/log" ]] && mkdir -p $HOME/collected_log/_test_init/logservice3 && cp -r $DATA_PATH/logservice/node3/log/* $HOME/collected_log/_test_init/logservice3 + [[ -d "$DATA_PATH/observer1/log" ]] && mkdir -p $HOME/collected_log/_test_init/observer1 && mv $DATA_PATH/observer1/log/* $HOME/collected_log/_test_init/observer1 + [[ -d "$DATA_PATH/observer2/log" ]] && mkdir -p $HOME/collected_log/_test_init/observer2 && mv $DATA_PATH/observer2/log/* $HOME/collected_log/_test_init/observer2 + [[ -d "$DATA_PATH/obproxy/log" ]] && mkdir -p $HOME/collected_log/_test_init/obproxy && mv $DATA_PATH/obproxy/log/* $HOME/collected_log/_test_init/obproxy exit 1 fi if [[ "$SPECIAL_RUN" == "1" ]] @@ -775,7 +533,7 @@ function obd_run_mysqltest { mkdir -p $result_dir RECORD_ARGS="--record-dir=$result_dir --record --log-dir=./var/rec_log" RESULT_ARGS="--result-dir=$result_dir --result-file-suffix=.record --log-dir=./var/log" - mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient --init-sql-dir=$HOME/oceanbase/tools/deploy --special-run --sort-result $COLLECT_ARG $REBOOT_TIMEOUT $VERBOSE_ARG" + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient --init-sql-dir=$HOME/seekdb/tools/deploy --special-run --sort-result $COLLECT_ARG $REBOOT_TIMEOUT $VERBOSE_ARG" # $mysqltest_cmd --all $INIT_FLIES $SCHE_ARGS $RECORD_ARGS 2>&1 > record.out $mysqltest_cmd $INIT_FLIES $SCHE_ARGS $RECORD_ARGS $EXTRA_ARGS 2>&1 | tee record.out && ( exit ${PIPESTATUS[0]}) rec_ret=$? @@ -793,8 +551,8 @@ function obd_run_mysqltest { done obd_prepare_config echo "config after change:" - cat $HOME/oceanbase/tools/deploy/config.yaml - $obd cluster destroy $ob_name && $obd cluster deploy $ob_name -c $HOME/oceanbase/tools/deploy/config.yaml && $obd cluster start $ob_name -f && $obd cluster display $ob_name + cat $HOME/seekdb/tools/deploy/config.yaml + $obd cluster destroy $ob_name && $obd cluster deploy $ob_name -c $HOME/seekdb/tools/deploy/config.yaml && $obd cluster start $ob_name -f && $obd cluster display $ob_name else $obd cluster redeploy $ob_name fi @@ -807,9 +565,6 @@ function obd_run_mysqltest { if [[ $JOBNAME == 'mysqltest_opensource' ]] then submarker="_opensource" - elif [[ $JOBNAME == 'mysqltest_sanity' ]] - then - submarker="_sanity" else submarker="" fi @@ -826,43 +581,43 @@ function obd_run_mysqltest { # todo: 修改为把tools deploy下的init sql都带入过去 mkdir $HOME/tmp_init_sql mkdir $HOME/obd_tmp - find $HOME/oceanbase/tools/deploy -name 'init_*.sql' | xargs -i cp {} $HOME/tmp_init_sql - cp $HOME/oceanbase/tools/deploy/obd/* $HOME/obd_tmp + find $HOME/seekdb/tools/deploy -name 'init_*.sql' | xargs -i cp {} $HOME/tmp_init_sql + cp $HOME/seekdb/tools/deploy/obd/* $HOME/obd_tmp cd $HOME && tar -cvf tmp_init_sql.tar.gz $HOME/tmp_init_sql cd $HOME && tar -cvf origin_obd_file.tar.gz $HOME/obd_tmp - cd $HOME/oceanbase + cd $HOME/seekdb # 针对多个分支进行处理 ret=0 for commit in $CROSS_VALIDATION_COMMIT do git checkout -f $commit || ret=2 # 然后重新针对obd 进行重新构建路径 - cd $HOME && tar -xvf tmp_init_sql.tar.gz && cp tmp_init_sql/* $HOME/oceanbase/tools/deploy - cd $HOME && tar -xvf origin_obd_file.tar.gz && cp obd_tmp/* $HOME/oceanbase/tools/deploy/obd + cd $HOME && tar -xvf tmp_init_sql.tar.gz && cp tmp_init_sql/* $HOME/seekdb/tools/deploy + cd $HOME && tar -xvf origin_obd_file.tar.gz && cp obd_tmp/* $HOME/seekdb/tools/deploy/obd - if [[ -f $HOME/oceanbase/deps/init/dep_create.sh ]] + if [[ -f $HOME/seekdb/deps/init/dep_create.sh ]] then if [[ "$IS_CE" == "1" ]] then - cd $HOME/oceanbase && ./build.sh init --ce || ret=3 + cd $HOME/seekdb && ./build.sh init --ce || ret=3 else - cd $HOME/oceanbase && ./build.sh init || ret=3 + cd $HOME/seekdb && ./build.sh init || ret=3 fi else - if grep 'dep_create.sh' $HOME/oceanbase/build.sh + if grep 'dep_create.sh' $HOME/seekdb/build.sh then - cd $HOME/oceanbase/deps/3rd && bash dep_create.sh all || ret=4 + cd $HOME/seekdb/deps/3rd && bash dep_create.sh all || ret=4 else - cd $HOME/oceanbase && ./build.sh init || ret=4 + cd $HOME/seekdb && ./build.sh init || ret=4 fi fi # 根据build.sh中的内容判断依赖安装路径 - if grep 'dep_create.sh' $HOME/oceanbase/build.sh + if grep 'dep_create.sh' $HOME/seekdb/build.sh then - DEP_PATH=$HOME/oceanbase/deps/3rd + DEP_PATH=$HOME/seekdb/deps/3rd else - DEP_PATH=$HOME/oceanbase/rpm/.dep_create/var + DEP_PATH=$HOME/seekdb/rpm/.dep_create/var fi extra_ignore_cmd="" if [[ `$DEP_PATH/u01/obclient/bin/mysqltest --help | grep "cmp-ignore-explain"` ]] @@ -875,17 +630,17 @@ function obd_run_mysqltest { extra_ignore_cmd=" $extra_ignore_cmd --special-run" fi - cd $HOME/oceanbase/tools/deploy - mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=$DEP_PATH/u01/obclient/bin/mysqltest --obclient-bin=$DEP_PATH/u01/obclient/bin/obclient $COLLECT_ARG --init-sql-dir=$HOME/oceanbase/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS_WITHOUT_CASE --suite=cross_validation $extra_ignore_cmd" + cd $HOME/seekdb/tools/deploy + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=$DEP_PATH/u01/obclient/bin/mysqltest --obclient-bin=$DEP_PATH/u01/obclient/bin/obclient $COLLECT_ARG --init-sql-dir=$HOME/seekdb/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS_WITHOUT_CASE --suite=cross_validation $extra_ignore_cmd" $mysqltest_cmd $INIT_FLIES $SCHE_ARGS 2>&1 | tee compare.out && ( exit ${PIPESTATUS[0]}) test_suite_ret=$? - FILTER=`ls -ltr $HOME/oceanbase/tools/deploy/mysql_test/test_suite/cross_validation/t | grep -v total | awk '{gsub(".test","",$9);print "cross_validation."$9}' | tr '\n' ',' | head -c -1` + FILTER=`ls -ltr $HOME/seekdb/tools/deploy/mysql_test/test_suite/cross_validation/t | grep -v total | awk '{gsub(".test","",$9);print "cross_validation."$9}' | tr '\n' ',' | head -c -1` if [[ $FILTER == "" ]] then echo "without testsuite cases" fi - mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=$DEP_PATH/u01/obclient/bin/mysqltest --obclient-bin=$DEP_PATH/u01/obclient/bin/obclient $COLLECT_ARG --init-sql-dir=$HOME/oceanbase/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS_WITHOUT_CASE --exclude=$FILTER --tag=cross_validation $extra_ignore_cmd" + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=$DEP_PATH/u01/obclient/bin/mysqltest --obclient-bin=$DEP_PATH/u01/obclient/bin/obclient $COLLECT_ARG --init-sql-dir=$HOME/seekdb/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS_WITHOUT_CASE --exclude=$FILTER --tag=cross_validation $extra_ignore_cmd" $mysqltest_cmd $INIT_FLIES $SCHE_ARGS 2>&1 | tee compare.out && ( exit ${PIPESTATUS[0]}) tag_ret=$? [[ $test_suite_ret = 0 && $tag_ret = 0 ]] && current_ret=0 || current=1 @@ -895,130 +650,51 @@ function obd_run_mysqltest { if [[ $JOBNAME == 'mysqltest_opensource' ]] then submarker="_opensource" - elif [[ $JOBNAME == 'mysqltest_sanity' ]] - then - submarker="_sanity" else submarker="" fi - mv compare.out $HOME/mysqltest${submarker}_compare_output${JOB_SUFFIX}.$SLICE_IDX + mv compare.out $HOME/mysqltest${submarker}_compare_output.$SLICE_IDX echo "finish!" - return $ret - else - if [[ -f $HOME/oceanbase/tools/deploy/error_log_filter.json && ( $BRANCH == 'master' || $BRANCH == "4_2_x_release" ) && "$FROM_FARM" == '1' ]] - then - echo "python $HOME/common/analyse_the_observer_log.py collect -p ${DATA_PATH}/observer1/log -p ${DATA_PATH}/observer2/log -i $GID -j ${JOBNAME}.${SLICE_IDX} -o $HOME -F $HOME/oceanbase/tools/deploy/error_log_filter.json" > $HOME/oceanbase/tools/deploy/analyse_the_observer_log.sh - chmod a+x $HOME/oceanbase/tools/deploy/analyse_the_observer_log.sh - RUN_EXTRA_CHECK_CMD='--extra-script-after-case="$HOME/oceanbase/tools/deploy/./analyse_the_observer_log.sh"' - fi - if [[ $SHARED_STORAGE_MODE == "1" ]] + return $ret + elif [[ $WITH_SHARED_STORAGE == "1" ]] + then + # 把sensitive_test目录下的测试集合移动到tools/deploy/mysql_test + # 并且创建一个新的目录用来表示单独的测试 + if [[ ! -d $HOME/seekdb/sensitive_test/mysql_test ]] then - # 由于--filter也是识别参数，因为会被带入到模式里面，需要放在ARGV识别之后处理 - EXTRA_ARGS="$EXTRA_ARGS--filter=shared_storage " + return 0 fi - mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient $COLLECT_ARG --init-sql-dir=$HOME/oceanbase/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS" - if [[ $SHARED_STORAGE_MODE == "1" && $TEST_NAME == "farm" ]] - then - mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient $COLLECT_ARG --init-sql-dir=$HOME/oceanbase/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG --shared_storage_pstest $EXTRA_ARGS_WITHOUT_CASE" - fi - if [[ $TEST_NAME != "farm" ]] + need_mv_dirs=`ls -1 $HOME/seekdb/sensitive_test/mysql_test/test_suite|xargs` + run_suites="" + for need_mv_dir in $need_mv_dirs + do + new_dir=$HOME/seekdb/tools/deploy/mysql_test/test_suite/shared_storage__${need_mv_dir} + run_suites="${run_suites}shared_storage__${need_mv_dir}," + mkdir -p $new_dir + cp -r $HOME/seekdb/sensitive_test/mysql_test/test_suite/$need_mv_dir/* $new_dir + done + run_suites=`echo $run_suites | awk '{print substr($0, 1, length($0)-1)}'` + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient $COLLECT_ARG --init-sql-dir=$HOME/seekdb/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS_WITHOUT_CASE --suite=$run_suites" + $mysqltest_cmd $INIT_FLIES $SCHE_ARGS 2>&1 | tee compare.out && ( exit ${PIPESTATUS[0]}) + else + if [[ -f $HOME/seekdb/tools/deploy/error_log_filter.json && ( $BRANCH == 'master' || $BRANCH == "4_2_x_release" ) && "$FROM_FARM" == '1' ]] then - # 大farm处理下odpscmd - mkdir $HOME/odps - cd $HOME/odps && timeout 200 bash -c 'wget https://antsys-enc.cn-heyuan-alipay-office.oss-alipay.aliyuncs.com/public/core-test/tools/odps/odpscmd_public.zip' - unzip odpscmd_public.zip - export PATH=$PATH:$HOME/odps/bin - - mkdir $HOME/s3 - yum install https://antsys-enc.cn-heyuan-alipay-office.oss-alipay.aliyuncs.com/public/core-test/tools/s3/python-dateutil-1.5-7.1.alios7.noarch.rpm -y - cd $HOME/s3 && timeout 200 bash -c 'wget https://antsys-enc.cn-heyuan-alipay-office.oss-alipay.aliyuncs.com/public/core-test/tools/s3/s3cmd.tar.gz' - tar -xvf s3cmd.tar.gz -C $HOME/s3 - export PATH=$PATH:$HOME/s3/s3cmd-2.4.0 - - cd $HOME/oceanbase/tools/deploy - set +x - sed -i 's#project_name=#project_name=mysqltest_regression#g' $HOME/odps/conf/odps_config.ini - sed -i 's@end_point=@end_point=http://service.cn-hangzhou.maxcompute.aliyun.com/api@g' $HOME/odps/conf/odps_config.ini - sed -i s/''/${MYSQLTEST_ODPS_ACCESSID}/g $HOME/odps/conf/odps_config.ini - sed -i s/''/${MYSQLTEST_ODPS_ACCESSKEY}/g $HOME/odps/conf/odps_config.ini - set -x - - if [[ -f $HOME/oceanbase/tools/deploy/export_vars_for_mysqltest.sh ]] - then - export VAULT_ADDR='https://vault.oceanbase-dev.com/' - source $HOME/oceanbase/tools/deploy/export_vars_for_mysqltest.sh - fi - set +x - sed -i s/'#OSS_HOST#'/cn-heyuan-alipay-b.oss-alipay.aliyuncs.com/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - sed -i s/'#OSS_ACCESS_KEY#'/${MYSQLTEST_OSS_KEY}/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - sed -i s/'#OSS_ACCESS_ID#'/${MYSQLTEST_OSS_ID}/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - sed -i 's@#ODPS_ENDPOINT#@http://service.cn-hangzhou.maxcompute.aliyun.com/api@g' $HOME/oceanbase/tools/deploy/mysql_test/include/odps_config_for_cloud.inc - sed -i s/'#ODPS_ACCESSID#'/${MYSQLTEST_ODPS_ACCESSID}/g $HOME/oceanbase/tools/deploy/mysql_test/include/odps_config_for_cloud.inc - sed -i s/'#ODPS_ACCESSKEY#'/${MYSQLTEST_ODPS_ACCESSKEY}/g $HOME/oceanbase/tools/deploy/mysql_test/include/odps_config_for_cloud.inc - - export VAULT_ADDR='https://vault.oceanbase-dev.com/' - export S3_BUCKET=`vault kv get -mount="secret/data/mysqltest/dev" "s3_access_info" | grep bucket | awk '{print \$2}'` - export S3_AK=`vault kv get -mount="secret/data/mysqltest/dev" "s3_access_info" | grep access_key | awk '{print \$2}'` - export S3_SK=`vault kv get -mount="secret/data/mysqltest/dev" "s3_access_info" | grep secret_key | awk '{print \$2}'` - - sed -i "s#\#S3_BUCKET\##s3://${S3_BUCKET}#g" $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - sed -i 's/#S3_REGION#/cn-north-1/g' $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - sed -i 's/#S3_ENDPOINT#/s3.cn-north-1.amazonaws.com.cn/g' $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - sed -i s/'#S3_AK#'/${S3_AK}/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - sed -i "s#\#S3_SK\##${S3_SK}#g" $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - - sed -i 's/#OBS_BUCKET#/s3:\/\/obs-436751-long-data-all-test/g' $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - sed -i 's/#OBS_REGION#/cn-north-4/g' $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - sed -i 's/#OBS_ENDPOINT#/obs.cn-north-4.myhuaweicloud.com/g' $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - sed -i s/'#OBS_AK#'/${SENSITIVE_TEST_OBS_ID_FOR_OBJECT_STORAGE_FOR_TOTAL}/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - sed -i s/'#OBS_SK#'/${SENSITIVE_TEST_OBS_KEY_FOR_OBJECT_STORAGE_FOR_TOTAL}/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - - sed -i 's/#OSS_BUCKET#/oss:\/\/oss-436751-self-all-test/g' $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - sed -i 's/#OSS_ENDPOINT#/oss-cn-hangzhou.aliyuncs.com/g' $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - sed -i s/'#OSS_AK#'/${SENSITIVE_TEST_OSS_ID_FOR_OBJECT_STORAGE}/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - sed -i s/'#OSS_SK#'/${SENSITIVE_TEST_OSS_KEY_FOR_OBJECT_STORAGE}/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - - export COS_BUCKET=`vault kv get -mount="secret/data/mysqltest/dev" "cos_access_info" | grep bucket | awk '{print \$2}'` - export COS_AK=`vault kv get -mount="secret/data/mysqltest/dev" "cos_access_info" | grep access_key | awk '{print \$2}'` - export COS_SK=`vault kv get -mount="secret/data/mysqltest/dev" "cos_access_info" | grep secret_key | awk '{print \$2}'` - - sed -i "s#\#COS_BUCKET\##cos://${COS_BUCKET}#g" $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - sed -i 's/#COS_ENDPOINT#/cos.ap-shanghai.myqcloud.com/g' $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - sed -i 's/#COS_APPID#/1317978819/g' $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - sed -i s/'#COS_AK#'/${COS_AK}/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - sed -i s/'#COS_SK#'/${COS_SK}/g $HOME/oceanbase/tools/deploy/mysql_test/include/oss_config_for_import.inc - set -x + echo "python $HOME/common/analyse_the_observer_log.py collect -p ${DATA_PATH}/observer1/log -p ${DATA_PATH}/observer2/log -i $GID -j ${JOBNAME}.${SLICE_IDX} -o $HOME -F $HOME/seekdb/tools/deploy/error_log_filter.json" > $HOME/seekdb/tools/deploy/analyse_the_observer_log.sh + chmod a+x $HOME/seekdb/tools/deploy/analyse_the_observer_log.sh + RUN_EXTRA_CHECK_CMD='--extra-script-after-case="$HOME/seekdb/tools/deploy/./analyse_the_observer_log.sh"' fi + mysqltest_cmd="$obd test mysqltest $ob_name $SERVER_ARGS --mysqltest-bin=./mysqltest --obclient-bin=./obclient $COLLECT_ARG --init-sql-dir=$HOME/seekdb/tools/deploy --log-dir=./var/log $REBOOT_TIMEOUT $VERBOSE_ARG $EXTRA_ARGS" $mysqltest_cmd $RUN_EXTRA_CHECK_CMD $INIT_FLIES $SCHE_ARGS 2>&1 | tee compare.out && ( exit ${PIPESTATUS[0]}) ret=$? if [[ $JOBNAME == 'mysqltest_opensource' ]] then submarker="_opensource" - elif [[ $JOBNAME == 'mysqltest_sanity' ]] - then - submarker="_sanity" else submarker="" fi - mv compare.out $HOME/mysqltest${submarker}_compare_output${JOB_SUFFIX}.$SLICE_IDX + mv compare.out $HOME/mysqltest${submarker}_compare_output.$SLICE_IDX echo "finish!" - if [[ $ret -ne 0 ]] - then - fail_cases=`python3 $HOME/common/parse_the_case_output_for_case.py mysqltest $HOME/mysqltest${submarker}_compare_output${JOB_SUFFIX}.$SLICE_IDX |xargs` - if [[ $fail_cases ]] - then - python3 $HOME/common/get_cases_run_result.py "$fail_cases" $BRANCH $FORK_COMMIT $HOME/case_whitelist.txt - filter_by_whitelist_res=$? - if [[ $filter_by_whitelist_res -eq 0 ]] - then - echo "CASE 运行失败，但是由于case都在白名单，认为运行成功" - ret=0 - # 还是收集下日志 - obd_collect_log - fi - fi - fi return $ret fi } @@ -1032,30 +708,26 @@ function obd_prepare_env { function obd_collect_log { - echo "`date "+%F %T"` [INFO] 开始collect log" + echo "collect log" cd $HOME/ mkdir -p collected_log mkdir -p collected_log/obd_log mkdir -p collected_log/mysqltest_log mkdir -p collected_log/mysqltest_rec_log - find "$DATA_PATH" -name 'core[.-]*' 2>/dev/null -exec cp -f {} collected_log/ \; || true - [[ "$OBD_HOME" != "" && -d "$OBD_HOME/.obd/log" ]] && mv "$OBD_HOME"/.obd/log/* collected_log/obd_log/ 2>/dev/null || true - [[ -d oceanbase/tools/deploy/var/log ]] && mv oceanbase/tools/deploy/var/log/* collected_log/mysqltest_log/ 2>/dev/null || true - [[ -d oceanbase/tools/deploy/var/rec_log ]] && mv oceanbase/tools/deploy/var/rec_log/* collected_log/mysqltest_rec_log/ 2>/dev/null || true - mv collected_log collected_log_$SLICE_IDX - echo "`date "+%F %T"` [INFO] 开始压缩 log" - if [[ -f /data/farm/tools/pigz ]] + find $DATA_PATH -name 'core[.-]*' | xargs -i cp {} collected_log + [[ "$OBD_HOME" != "" ]] && mv $OBD_HOME/.obd/log/* collected_log/obd_log/ + set +x + if [[ $WITH_SHARED_STORAGE == "1" ]] then - cp /data/farm/tools/pigz $HOME/ && chmod a+x $HOME/pigz - tar -cvf - collected_log_$SLICE_IDX | $HOME/./pigz -1 > collected_log_$SLICE_IDX${JOB_SUFFIX}.tar.gz - else - tar -cvf - collected_log_$SLICE_IDX |gzip -1 > collected_log_$SLICE_IDX${JOB_SUFFIX}.tar.gz + sed -i "s/${SENSITIVE_TEST_OSS_KEY_FOR_OBJECT_STORAGE_FOR_TOTAL}//" collected_log/obd_log/obd fi - echo "`date "+%F %T"` [INFO] 日志收集结束" + set -x + mv seekdb/tools/deploy/var/log/* collected_log/mysqltest_log/ + mv seekdb/tools/deploy/var/rec_log/* collected_log/mysqltest_rec_log/ + mv collected_log collected_log_$SLICE_IDX + tar zcvf collected_log_${SLICE_IDX}${JOB_SUFFIX}.tar.gz collected_log_$SLICE_IDX } -export -f obd_collect_log - function collect_obd_case_log { echo "collect obd case log" @@ -1063,9 +735,6 @@ function collect_obd_case_log { if [[ $JOBNAME == 'mysqltest_opensource' ]] then submarker="_opensource" - elif [[ $JOBNAME == 'mysqltest_sanity' ]] - then - submarker="_sanity" else submarker="" fi @@ -1085,17 +754,25 @@ function run { then mv /root/tmp/.ssh/id_rsa /root/.ssh/ fi - if ([[ -f $HOME/oceanbase/rpm/oceanbase.deps ]] && [[ "$(grep 'ob-deploy' $HOME/oceanbase/rpm/oceanbase.deps )" != "" ]]) || - ([[ -f $HOME/oceanbase/deps/3rd/oceanbase.el7.x86_64.deps ]] && [[ "$(grep 'ob-deploy' $HOME/oceanbase/deps/3rd/oceanbase.el7.x86_64.deps )" != "" ]]) || - ([[ -f $HOME/oceanbase/deps/init/oceanbase.el7.x86_64.deps ]] && [[ "$(grep 'ob-deploy' $HOME/oceanbase/deps/init/oceanbase.el7.x86_64.deps )" != "" ]]) + if ([[ -f $HOME/seekdb/rpm/oceanbase.deps ]] && [[ "$(grep 'ob-deploy' $HOME/seekdb/rpm/oceanbase.deps )" != "" ]]) || + ([[ -f $HOME/seekdb/deps/3rd/oceanbase.el7.x86_64.deps ]] && [[ "$(grep 'ob-deploy' $HOME/seekdb/deps/3rd/oceanbase.el7.x86_64.deps )" != "" ]]) || + ([[ -f $HOME/seekdb/deps/init/oceanbase.el7.x86_64.deps ]] && [[ "$(grep 'ob-deploy' $HOME/seekdb/deps/init/oceanbase.el7.x86_64.deps )" != "" ]]) then - timeout=34800 + if [[ $WITH_SHARED_STORAGE == "1" ]] + then + # 单独判断下是否有对应的case集合 + if [[ ! -d $HOME/seekdb/sensitive_test/mysql_test ]] + then + return 0 + fi + fi + timeout=18000 [[ "$SPECIAL_RUN" == "1" ]] && timeout=72000 obd_prepare_env && timeout $timeout bash -c "obd_run_mysqltest" test_ret=$? error_log_ret=0 - if [[ -f $HOME/oceanbase/tools/deploy/error_log_filter.json && $BRANCH == 'master' ]] + if [[ -f $HOME/seekdb/tools/deploy/error_log_filter.json && $BRANCH == 'master' ]] then # notice the core python $HOME/common/analyse_the_observer_core.py collect -p ${DATA_PATH} -j ${JOBNAME}.${SLICE_IDX} -o $HOME @@ -1106,31 +783,15 @@ function run { then obd_collect_log fi - return `[[ $test_ret = 0 && $error_log_ret = 0 ]]` - # if [[ $test_ret -ne 0 ]] - # then - # fail_cases=`python3 $HOME/common/parse_the_case_output_for_case.py mysqltest $HOME/mysqltest${submarker}_compare_output${JOB_SUFFIX}.$SLICE_IDX |xargs` - # if [[ $fail_cases ]] - # then - # python3 $HOME/common/get_cases_run_result.py "$fail_cases" $BRANCH $FORK_COMMIT - # filter_by_whitelist_res=$? - # if [[ $filter_by_whitelist_res -eq 0 ]] - # then - # echo "CASE 运行失败，但是由于case都在白名单，认为运行成功" - # test_ret=0 - # fi - # fi - # fi - - # return $test_ret + return `[[ $test_ret = 0 && $error_log_ret = 0 ]]` else prepare_env && timeout 18000 bash -c "run_mysqltest" test_ret=$? error_log_ret=0 - if [[ -f $HOME/oceanbase/tools/deploy/error_log_filter.json && ( $BRANCH == 'master' || $BRANCH == "4_2_x_release" ) ]] + if [[ -f $HOME/seekdb/tools/deploy/error_log_filter.json && ( $BRANCH == 'master' || $BRANCH == "4_2_x_release" ) ]] then - python $HOME/common/analyse_the_observer_log.py collect -p ${DATA_PATH}/observer1/log -p ${DATA_PATH}/observer2/log -i $GID -j ${JOBNAME}.${SLICE_IDX} -o $HOME -F $HOME/oceanbase/tools/deploy/error_log_filter.json + python $HOME/common/analyse_the_observer_log.py collect -p ${DATA_PATH}/observer1/log -p ${DATA_PATH}/observer2/log -i $GID -j ${JOBNAME}.${SLICE_IDX} -o $HOME -F $HOME/seekdb/tools/deploy/error_log_filter.json error_log_ret=0 fi if [[ $test_ret -ne 0 || $error_log_ret -ne 0 ]] From 737214efe5aa41ff71967b15ed0b279450607c76 Mon Sep 17 00:00:00 2001 From: guoyan1996 <1028109394@qq.com> Date: Sun, 5 Apr 2026 19:12:35 +0800 Subject: [PATCH 10/11] alter retry --- .github/workflows/seekdb.yml | 28 ++++++++++++++++++++++++---- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/.github/workflows/seekdb.yml b/.github/workflows/seekdb.yml index 5b85f348f..c4879db1e 100644 --- a/.github/workflows/seekdb.yml +++ b/.github/workflows/seekdb.yml @@ -95,10 +95,13 @@ jobs: run: | zstd -f -q -T0 -o observer.zst observer echo "observer.zst size: $(stat -c%s observer.zst 2>/dev/null || stat -f%z observer.zst 2>/dev/null) bytes" + # observer 与 run_attempt 无关：仅重跑失败 job 时 setup 可能跳过，若路径含 attempt-N + # 会去找 attempt-2/setup 而产物仍在 attempt-1/setup，导致「observer not found on NFS」。 - name: Persist observer to NFS shell: bash run: | - DEST_DIR="${{ env.SEEKDB_RESULT_NFS_ROOT }}/${{ github.run_id }}/attempt-${{ github.run_attempt }}/setup" + RUN_ROOT="${{ env.SEEKDB_RESULT_NFS_ROOT }}/${{ github.run_id }}" + DEST_DIR="$RUN_ROOT/setup" mkdir -p "$DEST_DIR" cp -f observer.zst "$DEST_DIR/" echo "observer.zst saved to $DEST_DIR/observer.zst" @@ -119,11 +122,28 @@ jobs: - name: Place observer (from NFS) shell: bash run: | - OBSERVER_ZST="${{ env.SEEKDB_RESULT_NFS_ROOT }}/${{ github.run_id }}/attempt-${{ github.run_attempt }}/setup/observer.zst" - if [[ -f "$OBSERVER_ZST" ]]; then + set -eo pipefail + RUN_ROOT="${{ env.SEEKDB_RESULT_NFS_ROOT }}/${{ github.run_id }}" + ATT="${{ github.run_attempt }}" + OBSERVER_ZST="" + if [[ -f "$RUN_ROOT/setup/observer.zst" ]]; then + OBSERVER_ZST="$RUN_ROOT/setup/observer.zst" + elif [[ -f "$RUN_ROOT/attempt-${ATT}/setup/observer.zst" ]]; then + OBSERVER_ZST="$RUN_ROOT/attempt-${ATT}/setup/observer.zst" + else + shopt -s nullglob + _fb=( "$RUN_ROOT"/attempt-*/setup/observer.zst ) + shopt -u nullglob + if (( ${#_fb[@]} > 0 )); then + OBSERVER_ZST=$(printf '%s\n' "${_fb[@]}" | xargs ls -t | head -1) + fi + fi + if [[ -n "$OBSERVER_ZST" && -f "$OBSERVER_ZST" ]]; then + echo "Using observer.zst: $OBSERVER_ZST" zstd -d -f -q -o observer "$OBSERVER_ZST" && chmod +x observer else - echo "observer not found on NFS" + echo "observer not found on NFS under $RUN_ROOT (tried setup/ and attempt-*/setup/)" + exit 1 fi ls -la observer - name: Run mysqltest slice From 9721a5f55269e99f220a09b9caf8cabf929d2c5b Mon Sep 17 00:00:00 2001 From: guoyan1996 <1028109394@qq.com> Date: Sun, 5 Apr 2026 21:14:07 +0800 Subject: [PATCH 11/11] delete IS_CE=1 --- .github/script/seekdb/scripts/mysqltest_for_seekdb.sh | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh b/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh index 9b173bce4..fc58b6b8f 100644 --- a/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh +++ b/.github/script/seekdb/scripts/mysqltest_for_seekdb.sh @@ -198,7 +198,6 @@ function obd_prepare_global { # SeekDB CI：固定按 CE 路径走 OBD/init（原先用 observer -V 判 CE，SeekDB 输出不含 CE 字样会被判成 0） COMPONENT="seekdb" - export IS_CE=1 # 根据build.sh中的内容判断依赖安装路径 if grep 'dep_create.sh' $HOME/seekdb/build.sh then