Skip to content

Commit 4c26024

Browse files
change test
1 parent 33769c2 commit 4c26024

File tree

2 files changed

+61
-16
lines changed

2 files changed

+61
-16
lines changed

.github/workflows/build.yaml

Lines changed: 16 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -42,14 +42,14 @@ jobs:
4242
# key: clickhouse-backup-golang-${{ matrix.golang-version }}-${{ hashFiles('go.mod', '.github/workflows/*.yaml') }}
4343

4444
- name: Install golang dependencies
45-
run: go mod download -x
45+
run: go mod download
4646
# if: |
4747
# steps.cache-golang.outputs.cache-hit != 'true'
4848

4949
- name: Build clickhouse-backup binary
5050
id: make-race
5151
env:
52-
GOROOT: ${{ env.GOROOT_1_24_X64 }}
52+
GOROOT: ${{ env.GOROOT_1_25_X64 }}
5353
run: |
5454
make build/linux/amd64/clickhouse-backup build/linux/arm64/clickhouse-backup
5555
make build/linux/amd64/clickhouse-backup-fips build/linux/arm64/clickhouse-backup-fips
@@ -125,8 +125,7 @@ jobs:
125125

126126
- name: Install python venv
127127
run: |
128-
set -x
129-
(dpkg -l | grep venv) || (apt-get update && apt-get install -y python3-venv)
128+
(dpkg -l | grep venv) > /dev/null || (apt-get update && apt-get install -y python3-venv)
130129
python3 -m venv ~/venv/qa
131130
132131
- name: Cache python
@@ -138,7 +137,6 @@ jobs:
138137

139138
- name: Install python dependencies
140139
run: |
141-
set -x
142140
~/venv/qa/bin/pip3 install -U -r ./test/testflows/requirements.txt
143141
if: |
144142
steps.cache-python.outputs.cache-hit != 'true'
@@ -155,10 +153,10 @@ jobs:
155153
# don't change it to avoid not working CI/CD
156154
RUN_TESTS: "*"
157155
run: |
158-
set -xe
156+
set -e
159157
export CLICKHOUSE_TESTS_DIR=$(pwd)/test/testflows/clickhouse_backup
160158
161-
docker compose -f ${CLICKHOUSE_TESTS_DIR}/docker-compose/docker-compose.yml pull
159+
docker compose -f ${CLICKHOUSE_TESTS_DIR}/docker-compose/docker-compose.yml pull --quiet
162160
163161
chmod +x $(pwd)/clickhouse-backup/clickhouse-backup*
164162
source ~/venv/qa/bin/activate
@@ -170,7 +168,7 @@ jobs:
170168
sudo chmod -Rv +rx test/testflows/clickhouse_backup/_instances
171169
- name: Format testflows coverage
172170
env:
173-
GOROOT: ${{ env.GOROOT_1_24_X64 }}
171+
GOROOT: ${{ env.GOROOT_1_25_X64 }}
174172
run: |
175173
sudo chmod -Rv a+rw test/testflows/_coverage_/
176174
ls -la test/testflows/_coverage_
@@ -252,7 +250,7 @@ jobs:
252250
- name: Running integration tests
253251
env:
254252
RUN_PARALLEL: 4
255-
GOROOT: ${{ env.GOROOT_1_24_X64 }}
253+
GOROOT: ${{ env.GOROOT_1_25_X64 }}
256254
CLICKHOUSE_VERSION: ${{ matrix.clickhouse }}
257255
# options for advanced debug CI/CD
258256
# RUN_TESTS: "^TestSkipDisk$"
@@ -277,9 +275,11 @@ jobs:
277275
QA_GCS_OVER_S3_SECRET_KEY: ${{ secrets.QA_GCS_OVER_S3_SECRET_KEY }}
278276
QA_GCS_OVER_S3_BUCKET: ${{ secrets.QA_GCS_OVER_S3_BUCKET }}
279277
run: |
280-
set -xe
278+
set -e
281279
echo "CLICKHOUSE_VERSION=${CLICKHOUSE_VERSION}"
282280
echo "GCS_TESTS=${GCS_TESTS}"
281+
echo "Go version: $(go version)"
282+
echo "Testing ClickHouse ${CLICKHOUSE_VERSION}"
283283
284284
chmod +x $(pwd)/clickhouse-backup/clickhouse-backup*
285285
@@ -302,7 +302,7 @@ jobs:
302302
pids=()
303303
project_ids=()
304304
for ((i = 0; i < RUN_PARALLEL; i++)); do
305-
docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name project${i} --progress plain up -d &
305+
docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name project${i} --progress=quiet up -d &
306306
pids+=($!)
307307
project_ids+=("project${i}")
308308
done
@@ -315,9 +315,9 @@ jobs:
315315
echo "$pid docker compose up successful"
316316
else
317317
echo "=== docker ${project_id} state ==="
318-
docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress plain ps -a
318+
docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress=quiet ps -a
319319
echo "=== docker ${project_id} logs ==="
320-
docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress plain logs
320+
docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress=quiet logs
321321
echo "$pid the docker compose up failed."
322322
exit 1 # Exit with an error code if any command fails
323323
fi
@@ -329,16 +329,16 @@ jobs:
329329
if [[ "0" != "${TEST_FAILED}" ]]; then
330330
for project_id in "${project_ids[@]}"; do
331331
echo "=== docker ${project_id} state ==="
332-
docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress plain ps -a
332+
docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress=quiet ps -a
333333
echo "=== docker ${project_id} logs ==="
334-
docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress plain logs
334+
docker compose -f ${CUR_DIR}/${COMPOSE_FILE} --project-name ${project_id} --progress=quiet logs
335335
done
336336
exit 1
337337
fi
338338
339339
- name: Format integration coverage
340340
env:
341-
GOROOT: ${{ env.GOROOT_1_24_X64 }}
341+
GOROOT: ${{ env.GOROOT_1_25_X64 }}
342342
run: |
343343
sudo chmod -Rv a+rw test/integration/_coverage_/
344344
ls -la test/integration/_coverage_

test/integration/integration_test.go

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -697,6 +697,12 @@ func TestChangeReplicationPathIfReplicaExists(t *testing.T) {
697697
env.Cleanup(t, r)
698698
}
699699

700+
// 🔍 [CH-23.3-DIAG] Add comprehensive diagnostic logging for ClickHouse 23.3 version boundary issues
701+
func TestEmbeddedAzure(t *testing.T) {
702+
version := os.Getenv("CLICKHOUSE_VERSION")
703+
log.Info().Msgf("🔍 [CH-23.3-DIAG] TestEmbeddedAzure: ClickHouse version=%s", version)
704+
comparison := compareVersion(version, "23.3")
705+
log.Info().Msgf("🔍 [CH-23.3-DIAG] TestEmbeddedAzure: compareVersion('%s', '23.3') = %d", version, comparison)
700706
func TestEmbeddedAzure(t *testing.T) {
701707
version := os.Getenv("CLICKHOUSE_VERSION")
702708
if compareVersion(version, "23.3") < 0 {
@@ -710,6 +716,11 @@ func TestEmbeddedAzure(t *testing.T) {
710716
env.DockerExecNoError(r, "clickhouse", "rm", "-rf", "/var/lib/clickhouse/disks/backups_azure/backup/")
711717
env.runMainIntegrationScenario(t, "EMBEDDED_AZURE", "config-azblob-embedded.yml")
712718
if compareVersion(version, "24.8") >= 0 {
719+
func TestEmbeddedGCSOverS3(t *testing.T) {
720+
version := os.Getenv("CLICKHOUSE_VERSION")
721+
log.Info().Msgf("🔍 [CH-23.3-DIAG] TestEmbeddedGCSOverS3: ClickHouse version=%s", version)
722+
comparison := compareVersion(version, "23.3")
723+
log.Info().Msgf("🔍 [CH-23.3-DIAG] TestEmbeddedGCSOverS3: compareVersion('%s', '23.3') = %d", version, comparison)
713724
env.runMainIntegrationScenario(t, "EMBEDDED_AZURE_URL", "config-azblob-embedded-url.yml")
714725
}
715726

@@ -724,6 +735,11 @@ func TestEmbeddedGCSOverS3(t *testing.T) {
724735
t.Logf("@TODO RESTORE Ordinary with old syntax still not works for %s version, look https://github.com/ClickHouse/ClickHouse/issues/43971", os.Getenv("CLICKHOUSE_VERSION"))
725736
env, r := NewTestEnvironment(t)
726737

738+
func TestEmbeddedS3(t *testing.T) {
739+
version := os.Getenv("CLICKHOUSE_VERSION")
740+
log.Info().Msgf("🔍 [CH-23.3-DIAG] TestEmbeddedS3: ClickHouse version=%s", version)
741+
comparison := compareVersion(version, "23.3")
742+
log.Info().Msgf("🔍 [CH-23.3-DIAG] TestEmbeddedS3: compareVersion('%s', '23.3') = %d", version, comparison)
727743
// === GCS over S3 ===
728744
if compareVersion(version, "24.3") >= 0 && os.Getenv("QA_GCS_OVER_S3_BUCKET") != "" {
729745
//@todo think about named collections to avoid show credentials in logs look to https://github.com/fsouza/fake-gcs-server/issues/1330, https://github.com/fsouza/fake-gcs-server/pull/1164
@@ -2533,6 +2549,11 @@ func TestProjections(t *testing.T) {
25332549
counts = 0
25342550
r.NoError(env.ch.SelectSingleRowNoCtx(&counts, "SELECT count() FROM default.table_with_projection"))
25352551
r.Equal(uint64(10), counts)
2552+
func TestCheckSystemPartsColumns(t *testing.T) {
2553+
version := os.Getenv("CLICKHOUSE_VERSION")
2554+
log.Info().Msgf("🔍 [CH-23.3-DIAG] TestCheckSystemPartsColumns: ClickHouse version=%s", version)
2555+
comparison := compareVersion(version, "23.3")
2556+
log.Info().Msgf("🔍 [CH-23.3-DIAG] TestCheckSystemPartsColumns: compareVersion('%s', '23.3') = %d", version, comparison)
25362557
if compareVersion(os.Getenv("CLICKHOUSE_VERSION"), "21.9") >= 0 {
25372558
counts = 0
25382559
r.NoError(env.ch.SelectSingleRowNoCtx(&counts, "SELECT count() FROM system.parts WHERE database='default' AND table='table_with_projection' AND has(projections,'x')"))
@@ -2576,6 +2597,11 @@ func TestCheckSystemPartsColumns(t *testing.T) {
25762597
r.NoError(env.ch.DropOrDetachTable(clickhouse.Table{Database: t.Name(), Name: "test_system_parts_columns"}, createSQL, "", false, version, "", false, ""))
25772598

25782599
// test incompatible data types
2600+
func TestSlashesInDatabaseAndTableNamesAndTableQuery(t *testing.T) {
2601+
version := os.Getenv("CLICKHOUSE_VERSION")
2602+
log.Info().Msgf("🔍 [CH-23.3-DIAG] TestSlashesInDatabaseAndTableNamesAndTableQuery: ClickHouse version=%s", version)
2603+
comparison := compareVersion(version, "23.3")
2604+
log.Info().Msgf("🔍 [CH-23.3-DIAG] TestSlashesInDatabaseAndTableNamesAndTableQuery: compareVersion('%s', '23.3') = %d", version, comparison)
25792605
env.queryWithNoError(r, "CREATE TABLE "+t.Name()+".test_system_parts_columns(dt Date, v String) ENGINE=MergeTree() PARTITION BY dt ORDER BY tuple()")
25802606
env.queryWithNoError(r, "INSERT INTO "+t.Name()+".test_system_parts_columns SELECT today() - INTERVAL number DAY, if(number>0,'a',toString(number)) FROM numbers(2)")
25812607

@@ -2791,6 +2817,11 @@ func TestGetPartitionId(t *testing.T) {
27912817
{
27922818
"CREATE TABLE default.test_part_id_4 (dt String, name String) ENGINE = MergeTree ORDER BY dt PARTITION BY dt",
27932819
"default",
2820+
func TestRestoreAsAttach(t *testing.T) {
2821+
version := os.Getenv("CLICKHOUSE_VERSION")
2822+
log.Info().Msgf("🔍 [CH-23.3-DIAG] TestRestoreAsAttach: ClickHouse version=%s", version)
2823+
comparison := compareVersion(version, "23.3")
2824+
log.Info().Msgf("🔍 [CH-23.3-DIAG] TestRestoreAsAttach: compareVersion('%s', '23.3') = %d", version, comparison)
27942825
"test_part_id_4",
27952826
"'2023-01-01'",
27962827
"c487903ebbb25a533634d6ec3485e3a9",
@@ -3768,6 +3799,10 @@ func (env *TestEnvironment) checkObjectStorageIsEmpty(t *testing.T, r *require.A
37683799
// docker run --network=integration_clickhouse-backup -it --rm mcr.microsoft.com/azure-cli:latest
37693800
// export AZURE_STORAGE_CONNECTION_STRING="DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://azure:10000/devstoreaccount1;"
37703801
// az storage blob list --container-name azure-disk
3802+
func replaceStorageDiskNameForReBalance(version string, policyXML string) string {
3803+
log.Info().Msgf("🔍 [CH-23.3-DIAG] replaceStorageDiskNameForReBalance: ClickHouse version=%s", version)
3804+
comparison := compareVersion(version, "23.3")
3805+
log.Info().Msgf("🔍 [CH-23.3-DIAG] replaceStorageDiskNameForReBalance: compareVersion('%s', '23.3') = %d", version, comparison)
37713806
// az storage blob delete-batch --source azure-disk
37723807
// az storage blob list --container-name azure-disk
37733808
time.Sleep(15 * time.Second)
@@ -4618,6 +4653,16 @@ func toDate(s string) time.Time {
46184653
return result
46194654
}
46204655

4656+
func compareVersion(v1, v2 string) int {
4657+
log.Info().Msgf("🔍 [CH-23.3-DIAG] compareVersion: Comparing '%s' vs '%s'", v1, v2)
4658+
4659+
// Parse v1
4660+
parts1 := strings.Split(v1, ".")
4661+
log.Info().Msgf("🔍 [CH-23.3-DIAG] compareVersion: v1 parts=%v", parts1)
4662+
4663+
// Parse v2
4664+
parts2 := strings.Split(v2, ".")
4665+
log.Info().Msgf("🔍 [CH-23.3-DIAG] compareVersion: v2 parts=%v", parts2)
46214666
func toTS(s string) time.Time {
46224667
result, _ := time.Parse("2006-01-02 15:04:05", s)
46234668
return result

0 commit comments

Comments
 (0)