From 49a2cfaecc2cd9fdc36a0a44cfb504be3f077564 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Sat, 8 Feb 2025 14:45:44 +0800 Subject: [PATCH 1/8] update --- .../conf/changefeed.toml | 2 + .../conf/diff_config.toml | 29 ++++++++++++ .../overwrite_resume_with_syncpoint/run.sh | 46 +++++++++++++++++++ 3 files changed, 77 insertions(+) create mode 100644 tests/integration_tests/overwrite_resume_with_syncpoint/conf/changefeed.toml create mode 100644 tests/integration_tests/overwrite_resume_with_syncpoint/conf/diff_config.toml create mode 100644 tests/integration_tests/overwrite_resume_with_syncpoint/run.sh diff --git a/tests/integration_tests/overwrite_resume_with_syncpoint/conf/changefeed.toml b/tests/integration_tests/overwrite_resume_with_syncpoint/conf/changefeed.toml new file mode 100644 index 00000000000..63153e1dc53 --- /dev/null +++ b/tests/integration_tests/overwrite_resume_with_syncpoint/conf/changefeed.toml @@ -0,0 +1,2 @@ +enable-sync-point = true +sync-point-interval = "30s" \ No newline at end of file diff --git a/tests/integration_tests/overwrite_resume_with_syncpoint/conf/diff_config.toml b/tests/integration_tests/overwrite_resume_with_syncpoint/conf/diff_config.toml new file mode 100644 index 00000000000..a1f7520d9cb --- /dev/null +++ b/tests/integration_tests/overwrite_resume_with_syncpoint/conf/diff_config.toml @@ -0,0 +1,29 @@ +# diff Configuration. + +check-thread-count = 4 + +export-fix-sql = true + +check-struct-only = false + +[task] + output-dir = "/tmp/tidb_cdc_test/overwrite_resume_with_syncpoint/sync_diff/output" + + source-instances = ["mysql1"] + + target-instance = "tidb0" + + target-check-tables = ["test.t?*"] + +[data-sources] +[data-sources.mysql1] + host = "127.0.0.1" + port = 4000 + user = "root" + password = "" + +[data-sources.tidb0] + host = "127.0.0.1" + port = 3306 + user = "root" + password = "" diff --git a/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh b/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh new file mode 100644 index 00000000000..47b81d2ecff --- /dev/null +++ b/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh @@ -0,0 +1,46 @@ +#!/bin/bash +# the script test when we enable syncpoint, and pause the changefeed, +# then resume with a forward checkpoint, to ensure the changefeed can be sync correctly. + +set -eu + +CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) +source $CUR/../_utils/test_prepare +WORK_DIR=$OUT_DIR/$TEST_NAME +CDC_BINARY=cdc.test +SINK_TYPE=$1 + +function run() { + # No need to test kafka and storage sink. + if [ "$SINK_TYPE" != "mysql" ]; then + return + fi + + rm -rf $WORK_DIR && mkdir -p $WORK_DIR + + start_tidb_cluster --workdir $WORK_DIR --tidb-config $CUR/conf/tidb_config.toml + + cd $WORK_DIR + + run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY + + SINK_URI="mysql://root@127.0.0.1:3306/" + run_cdc_cli changefeed create --sink-uri="$SINK_URI" --config=$CUR/conf/changefeed.toml --changefeed-id="test4" + + check_ts_forward "test4" + + run_cdc_cli changefeed pause --changefeed-id="test4" + + # resume a forward checkpointTs + run_cdc_cli changefeed resume --changefeed-id="test4" --no-confirm --overwrite-checkpoint-ts=999999999999999999 + + check_ts_forward "test4" + + cleanup_process $CDC_BINARY +} + + +trap stop_tidb_cluster EXIT +run $* +check_logs $WORK_DIR +echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>" From 24e5513631f2cd14a7e299d1d659b3d1ee1aaba1 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Sat, 8 Feb 2025 14:59:44 +0800 Subject: [PATCH 2/8] update --- .../integration_tests/overwrite_resume_with_syncpoint/run.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh b/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh index 47b81d2ecff..fe42eae147f 100644 --- a/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh +++ b/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh @@ -10,7 +10,7 @@ WORK_DIR=$OUT_DIR/$TEST_NAME CDC_BINARY=cdc.test SINK_TYPE=$1 -function run() { +function run() # No need to test kafka and storage sink. if [ "$SINK_TYPE" != "mysql" ]; then return @@ -18,7 +18,7 @@ function run() { rm -rf $WORK_DIR && mkdir -p $WORK_DIR - start_tidb_cluster --workdir $WORK_DIR --tidb-config $CUR/conf/tidb_config.toml + start_tidb_cluster --workdir $WORK_DIR cd $WORK_DIR From 00ddca9a7b48c00f819910e25e8016cec9dbee3a Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Sat, 8 Feb 2025 15:41:01 +0800 Subject: [PATCH 3/8] update --- .../overwrite_resume_with_syncpoint/run.sh | 27 ++++++++++++++++--- 1 file changed, 24 insertions(+), 3 deletions(-) diff --git a/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh b/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh index fe42eae147f..4d0a4d91045 100644 --- a/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh +++ b/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh @@ -2,7 +2,7 @@ # the script test when we enable syncpoint, and pause the changefeed, # then resume with a forward checkpoint, to ensure the changefeed can be sync correctly. -set -eu +set -eux CUR=$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd) source $CUR/../_utils/test_prepare @@ -10,7 +10,23 @@ WORK_DIR=$OUT_DIR/$TEST_NAME CDC_BINARY=cdc.test SINK_TYPE=$1 -function run() +function check_ts_forward() { + changefeedid=$1 + rts1=$(cdc cli changefeed query --changefeed-id=${changefeedid} 2>&1 | jq '.resolved_ts') + checkpoint1=$(cdc cli changefeed query --changefeed-id=${changefeedid} 2>&1 | jq '.checkpoint_tso') + sleep 1 + rts2=$(cdc cli changefeed query --changefeed-id=${changefeedid} 2>&1 | jq '.resolved_ts') + checkpoint2=$(cdc cli changefeed query --changefeed-id=${changefeedid} 2>&1 | jq '.checkpoint_tso') + if [[ "$rts1" != "null" ]] && [[ "$rts1" != "0" ]]; then + if [[ "$rts1" -ne "$rts2" ]] || [[ "$checkpoint1" -ne "$checkpoint2" ]]; then + echo "changefeed is working normally rts: ${rts1}->${rts2} checkpoint: ${checkpoint1}->${checkpoint2}" + return + fi + fi + exit 1 +} + +function run() { # No need to test kafka and storage sink. if [ "$SINK_TYPE" != "mysql" ]; then return @@ -31,8 +47,13 @@ function run() run_cdc_cli changefeed pause --changefeed-id="test4" + sleep 20 + + checkpoint1=$(cdc cli changefeed query --changefeed-id="test4" 2>&1 | jq '.checkpoint_tso') + checkpoint1=$((checkpoint1 + 1)) + # resume a forward checkpointTs - run_cdc_cli changefeed resume --changefeed-id="test4" --no-confirm --overwrite-checkpoint-ts=999999999999999999 + run_cdc_cli changefeed resume --changefeed-id="test4" --no-confirm --overwrite-checkpoint-ts=$checkpoint1 check_ts_forward "test4" From 87869d93fe8b12a672464fbdaf1b7781e76bba0d Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Sat, 8 Feb 2025 19:00:15 +0800 Subject: [PATCH 4/8] update --- .../conf/diff_config.toml | 29 ------------------- .../overwrite_resume_with_syncpoint/run.sh | 7 +++-- 2 files changed, 4 insertions(+), 32 deletions(-) delete mode 100644 tests/integration_tests/overwrite_resume_with_syncpoint/conf/diff_config.toml diff --git a/tests/integration_tests/overwrite_resume_with_syncpoint/conf/diff_config.toml b/tests/integration_tests/overwrite_resume_with_syncpoint/conf/diff_config.toml deleted file mode 100644 index a1f7520d9cb..00000000000 --- a/tests/integration_tests/overwrite_resume_with_syncpoint/conf/diff_config.toml +++ /dev/null @@ -1,29 +0,0 @@ -# diff Configuration. - -check-thread-count = 4 - -export-fix-sql = true - -check-struct-only = false - -[task] - output-dir = "/tmp/tidb_cdc_test/overwrite_resume_with_syncpoint/sync_diff/output" - - source-instances = ["mysql1"] - - target-instance = "tidb0" - - target-check-tables = ["test.t?*"] - -[data-sources] -[data-sources.mysql1] - host = "127.0.0.1" - port = 4000 - user = "root" - password = "" - -[data-sources.tidb0] - host = "127.0.0.1" - port = 3306 - user = "root" - password = "" diff --git a/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh b/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh index 4d0a4d91045..d7ec8a150a0 100644 --- a/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh +++ b/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh @@ -47,10 +47,11 @@ function run() { run_cdc_cli changefeed pause --changefeed-id="test4" - sleep 20 + sleep 15 checkpoint1=$(cdc cli changefeed query --changefeed-id="test4" 2>&1 | jq '.checkpoint_tso') - checkpoint1=$((checkpoint1 + 1)) + # add a large number to avoid the problem of losing precision when jq processing large integers + checkpoint1=$((checkpoint1 + 1000000)) # resume a forward checkpointTs run_cdc_cli changefeed resume --changefeed-id="test4" --no-confirm --overwrite-checkpoint-ts=$checkpoint1 @@ -64,4 +65,4 @@ function run() { trap stop_tidb_cluster EXIT run $* check_logs $WORK_DIR -echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>" +echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>" \ No newline at end of file From 9de897446e359656dc654f958822c9a281fa53f4 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Sat, 8 Feb 2025 19:24:04 +0800 Subject: [PATCH 5/8] update --- .../overwrite_resume_with_syncpoint/run.sh | 27 +++++++++---------- 1 file changed, 13 insertions(+), 14 deletions(-) diff --git a/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh b/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh index d7ec8a150a0..37704a38e2b 100644 --- a/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh +++ b/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh @@ -1,5 +1,5 @@ #!/bin/bash -# the script test when we enable syncpoint, and pause the changefeed, +# the script test when we enable syncpoint, and pause the changefeed, # then resume with a forward checkpoint, to ensure the changefeed can be sync correctly. set -eux @@ -34,34 +34,33 @@ function run() { rm -rf $WORK_DIR && mkdir -p $WORK_DIR - start_tidb_cluster --workdir $WORK_DIR + start_tidb_cluster --workdir $WORK_DIR cd $WORK_DIR - run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY + run_cdc_server --workdir $WORK_DIR --binary $CDC_BINARY - SINK_URI="mysql://root@127.0.0.1:3306/" - run_cdc_cli changefeed create --sink-uri="$SINK_URI" --config=$CUR/conf/changefeed.toml --changefeed-id="test4" + SINK_URI="mysql://root@127.0.0.1:3306/" + run_cdc_cli changefeed create --sink-uri="$SINK_URI" --config=$CUR/conf/changefeed.toml --changefeed-id="test4" check_ts_forward "test4" - run_cdc_cli changefeed pause --changefeed-id="test4" + run_cdc_cli changefeed pause --changefeed-id="test4" - sleep 15 + sleep 15 - checkpoint1=$(cdc cli changefeed query --changefeed-id="test4" 2>&1 | jq '.checkpoint_tso') - # add a large number to avoid the problem of losing precision when jq processing large integers - checkpoint1=$((checkpoint1 + 1000000)) + checkpoint1=$(cdc cli changefeed query --changefeed-id="test4" 2>&1 | jq '.checkpoint_tso') + # add a large number to avoid the problem of losing precision when jq processing large integers + checkpoint1=$((checkpoint1 + 1000000)) - # resume a forward checkpointTs - run_cdc_cli changefeed resume --changefeed-id="test4" --no-confirm --overwrite-checkpoint-ts=$checkpoint1 + # resume a forward checkpointTs + run_cdc_cli changefeed resume --changefeed-id="test4" --no-confirm --overwrite-checkpoint-ts=$checkpoint1 - check_ts_forward "test4" + check_ts_forward "test4" cleanup_process $CDC_BINARY } - trap stop_tidb_cluster EXIT run $* check_logs $WORK_DIR From 6ea2884cfecc8947a424de3fc36e3814de030c11 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Sat, 8 Feb 2025 19:24:57 +0800 Subject: [PATCH 6/8] update --- tests/integration_tests/run_group.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_tests/run_group.sh b/tests/integration_tests/run_group.sh index 78f86f40935..5297c6e95e1 100755 --- a/tests/integration_tests/run_group.sh +++ b/tests/integration_tests/run_group.sh @@ -11,7 +11,7 @@ group_num=${group#G} # Other tests that only support mysql: batch_update_to_no_batch ddl_reentrant # changefeed_fast_fail changefeed_resume_with_checkpoint_ts sequence # multi_cdc_cluster capture_suicide_while_balance_table -mysql_only="bdr_mode capture_suicide_while_balance_table syncpoint syncpoint_check_ts server_config_compatibility changefeed_dup_error_restart safe_mode" +mysql_only="bdr_mode capture_suicide_while_balance_table syncpoint syncpoint_check_ts server_config_compatibility changefeed_dup_error_restart safe_mode overwrite_resume_with_syncpoint" mysql_only_http="http_api http_api_tls api_v2 http_api_tls_with_user_auth cli_tls_with_auth" mysql_only_consistent_replicate="consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table" From 67e924390bf54f2dd8a0aa28d1a022a37e025dd7 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Sat, 8 Feb 2025 19:56:15 +0800 Subject: [PATCH 7/8] update --- tests/integration_tests/overwrite_resume_with_syncpoint/run.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh b/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh index 37704a38e2b..9fa464d0a52 100644 --- a/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh +++ b/tests/integration_tests/overwrite_resume_with_syncpoint/run.sh @@ -64,4 +64,4 @@ function run() { trap stop_tidb_cluster EXIT run $* check_logs $WORK_DIR -echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>" \ No newline at end of file +echo "[$(date)] <<<<<< run test case $TEST_NAME success! >>>>>>" From eb1b90a9bb577227e5476a9b67a9beed5c6505e0 Mon Sep 17 00:00:00 2001 From: hongyunyan <649330952@qq.com> Date: Wed, 12 Feb 2025 10:04:18 +0800 Subject: [PATCH 8/8] update --- tests/integration_tests/run_group.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/integration_tests/run_group.sh b/tests/integration_tests/run_group.sh index 5297c6e95e1..5f825afc581 100755 --- a/tests/integration_tests/run_group.sh +++ b/tests/integration_tests/run_group.sh @@ -11,7 +11,7 @@ group_num=${group#G} # Other tests that only support mysql: batch_update_to_no_batch ddl_reentrant # changefeed_fast_fail changefeed_resume_with_checkpoint_ts sequence # multi_cdc_cluster capture_suicide_while_balance_table -mysql_only="bdr_mode capture_suicide_while_balance_table syncpoint syncpoint_check_ts server_config_compatibility changefeed_dup_error_restart safe_mode overwrite_resume_with_syncpoint" +mysql_only="bdr_mode capture_suicide_while_balance_table syncpoint syncpoint_check_ts server_config_compatibility changefeed_dup_error_restart safe_mode" mysql_only_http="http_api http_api_tls api_v2 http_api_tls_with_user_auth cli_tls_with_auth" mysql_only_consistent_replicate="consistent_replicate_ddl consistent_replicate_gbk consistent_replicate_nfs consistent_replicate_storage_file consistent_replicate_storage_file_large_value consistent_replicate_storage_s3 consistent_partition_table" @@ -52,7 +52,7 @@ groups=( # G09 'gc_safepoint changefeed_pause_resume cli_with_auth savepoint synced_status' # G10 - 'default_value simple cdc_server_tips event_filter sql_mode' + 'default_value simple cdc_server_tips event_filter sql_mode overwrite_resume_with_syncpoint' # G11 'resolve_lock move_table autorandom generate_column' # G12