[wip](E2E)Update Doris E2E Test Container to Use Latest Version

.github/workflows/build-connector.yml

@@ -25,57 +25,68 @@ jobs:
   build-extension:
     name: "Build Connector"
     runs-on: ubuntu-latest
+    if: ${{ !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) }}
     defaults:
       run:
         shell: bash
     steps:
-    - name: Checkout
-      uses: actions/checkout@master
+      - name: Checkout
+        uses: actions/checkout@v4
 
-    - name: Setup java
-      uses: actions/setup-java@v2
-      with:
-        distribution: adopt
-        java-version: '8'
+      - name: Setup Java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+          cache: 'maven'
 
-    - name: Build flink connector 1.15
-      run: |
-        cd flink-doris-connector && mvn clean package \
-          -Dflink.version=1.15.0 \
-          -Dflink.minor.version=1.15 \
-          -Dflink.python.id=flink-python_2.12
+      - name: Cache Maven Repository
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+            ${{ runner.os }}-maven-
 
-    - name: Build flink connector 1.16
-      run: |
-        cd flink-doris-connector && mvn clean package \
+      - name: Build flink connector 1.15
+        run: |
+          cd flink-doris-connector && mvn clean package \
+            -Dflink.version=1.15.0 \
+            -Dflink.minor.version=1.15 \
+            -Dflink.python.id=flink-python_2.12
+
+      - name: Build flink connector 1.16
+        run: |
+          cd flink-doris-connector && mvn clean package \
           -Dflink.version=1.16.0 \
           -Dflink.minor.version=1.16 \
           -Dflink.python.id=flink-python
 
-    - name: Build flink connector 1.17
-      run: |
-        cd flink-doris-connector && mvn clean package \
-          -Dflink.version=1.17.0 \
-          -Dflink.minor.version=1.17 \
-          -Dflink.python.id=flink-python
+      - name: Build flink connector 1.17
+        run: |
+          cd flink-doris-connector && mvn clean package \
+            -Dflink.version=1.17.0 \
+            -Dflink.minor.version=1.17 \
+            -Dflink.python.id=flink-python
 
-    - name: Build flink connector 1.18
-      run: |
-        cd flink-doris-connector && mvn clean package \
-          -Dflink.version=1.18.0 \
-          -Dflink.minor.version=1.18 \
-          -Dflink.python.id=flink-python
+      - name: Build flink connector 1.18
+        run: |
+          cd flink-doris-connector && mvn clean package \
+            -Dflink.version=1.18.0 \
+            -Dflink.minor.version=1.18 \
+            -Dflink.python.id=flink-python
 
-    - name: Build flink connector 1.19
-      run: |
-        cd flink-doris-connector && mvn clean package \
-          -Dflink.version=1.19.0 \
-          -Dflink.minor.version=1.19 \
-          -Dflink.python.id=flink-python
+      - name: Build flink connector 1.19
+        run: |
+          cd flink-doris-connector && mvn clean package \
+            -Dflink.version=1.19.0 \
+            -Dflink.minor.version=1.19 \
+            -Dflink.python.id=flink-python
 
-    - name: Build flink connector 1.20
-      run: |
-        cd flink-doris-connector && mvn clean package \
-          -Dflink.version=1.20.0 \
-          -Dflink.minor.version=1.20 \
-          -Dflink.python.id=flink-python
+      - name: Build flink connector 1.20
+        run: |
+          cd flink-doris-connector && mvn clean package \
+            -Dflink.version=1.20.0 \
+            -Dflink.minor.version=1.20 \
+            -Dflink.python.id=flink-python

.github/workflows/checkstyle.yaml

@@ -27,17 +27,25 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout
-        uses: actions/checkout@v3
+        uses: actions/checkout@v4
         with:
           persist-credentials: false
           submodules: recursive
 
       - name: Setup java
-        uses: actions/setup-java@v2
+        uses: actions/setup-java@v4
         with:
-          distribution: adopt
+          distribution: 'temurin'
           java-version: '8'
 
+      - name: Cache Maven Repository
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-checkstyle
+          restore-keys: |
+            ${{ runner.os }}-maven-
+
       - name: Run java checkstyle
-        run:
-          cd flink-doris-connector && mvn clean compile checkstyle:checkstyle
+        run: |
+          cd flink-doris-connector && mvn clean compile checkstyle:checkstyle -Dcheckstyle.skip=false -Dspotless.check.skip=false

.github/workflows/license-eyes.yml

@@ -28,8 +28,11 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: "Checkout ${{ github.ref }} ( ${{ github.sha }} )"
-        uses: actions/checkout@v2
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+
       - name: Check License
-        uses: apache/skywalking-eyes@v0.2.0
+        uses: apache/skywalking-eyes@v0.5.0
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/run-e2ecase.yml

@@ -25,20 +25,44 @@ jobs:
   build-extension:
     name: "Run E2ECases"
     runs-on: ubuntu-latest
+    if: ${{ !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) }}
+    env:
+      TZ: Asia/Shanghai
+      DORIS_IMAGE: "yagagagaga/doris-standalone:2.1.7"
     defaults:
       run:
         shell: bash
     steps:
-    - name: Checkout
-      uses: actions/checkout@master
+      - name: Checkout
+        uses: actions/checkout@v4
 
-    - name: Setup java
-      uses: actions/setup-java@v2
-      with:
-        distribution: adopt
-        java-version: '8'
+      - name: Setup java
+        uses: actions/setup-java@v4
+        with:
+          distribution: 'temurin'
+          java-version: '8'
+          cache: 'maven'
 
-    - name: Run E2ECases
-      run: |
-        cd flink-doris-connector && mvn test -Dtest="*E2ECase" -Dimage="apache/doris:doris-all-in-one-2.1.0"
+      - name: Cache Maven Repository
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}-e2e
+          restore-keys: |
+            ${{ runner.os }}-maven-
 
+      - name: Free Disk Space
+        run: |
+          df -h
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          df -h
+
+      - name: Pull Doris Docker Image
+        run: docker pull ${DORIS_IMAGE}
+
+      - name: Run E2ECases
+        run: |
+          cd flink-doris-connector && mvn test -Dtest="*E2ECase" -Dimage="yagagagaga/doris-standalone:2.1.7"

.github/workflows/run-itcase.yml

@@ -25,20 +25,48 @@ jobs:
   build-extension:
     name: "Run ITCases"
     runs-on: ubuntu-latest
+    if: ${{ !(github.event_name == 'pull_request' && github.event.pull_request.head.repo.full_name == github.repository) }}
+    strategy:
+      matrix:
+        java-version: [8]
+    env:
+      TZ: Asia/Shanghai
+      DORIS_IMAGE: "yagagagaga/doris-standalone:2.1.7"
     defaults:
       run:
         shell: bash
+
     steps:
-    - name: Checkout
-      uses: actions/checkout@master
+      - name: Checkout
+        uses: actions/checkout@v4
+
+      - name: Setup java
+        uses: actions/setup-java@v4
+        with:
+          java-version: ${{ matrix.java-version }}
+          distribution: 'temurin'
+          cache: 'maven'
+
+      - name: Cache Maven Repository
+        uses: actions/cache@v4
+        with:
+          path: ~/.m2/repository
+          key: ${{ runner.os }}-maven-${{ hashFiles('**/pom.xml') }}
+          restore-keys: |
+            ${{ runner.os }}-maven-
 
-    - name: Setup java
-      uses: actions/setup-java@v2
-      with:
-        distribution: adopt
-        java-version: '8'
+      - name: Free Disk Space
+        run: |
+          df -h
+          sudo rm -rf /usr/share/dotnet
+          sudo rm -rf /usr/local/lib/android
+          sudo rm -rf /opt/ghc
+          sudo rm -rf /opt/hostedtoolcache/CodeQL
+          df -h
 
-    - name: Run ITCases
-      run: |
-        cd flink-doris-connector && mvn test -Dtest="*ITCase" -Dimage="apache/doris:doris-all-in-one-2.1.0"
+      - name: Pull Doris Docker Image
+        run: docker pull ${DORIS_IMAGE}
 
+      - name: Run ITCases
+        run: |
+          cd flink-doris-connector && mvn test -Dtest="*ITCase" -Dimage="yagagagaga/doris-standalone:2.1.7" -B -V -Dfail-fast=true -DfailIfNoTests=true

flink-doris-connector/src/main/java/org/apache/doris/flink/sink/BackendUtil.java

@@ -119,7 +119,7 @@ public static boolean tryHttpConnection(String host) {
                     responseMessage);
             return false;
         } catch (Exception ex) {
-            LOG.warn("Failed to connect to host:{}", host, ex);
+            LOG.warn("Failed to connect to host: {} ", host, ex);
             return false;
         }
     }

flink-doris-connector/src/test/java/org/apache/doris/flink/container/AbstractITCaseService.java

@@ -46,17 +46,21 @@ public abstract class AbstractITCaseService extends AbstractContainerTestBase {
 
     protected static void waitForJobStatus(
             JobClient client, List<JobStatus> expectedStatus, Deadline deadline) throws Exception {
+        LOG.info("Waiting for job to reach one of the expected states: {}", expectedStatus);
         waitUntilCondition(
                 () -> {
                     JobStatus currentStatus;
                     try {
                         currentStatus = (JobStatus) client.getJobStatus().get();
                     } catch (IllegalStateException e) {
-                        LOG.warn("Failed to get state, cause " + e.getMessage());
+                        LOG.warn("Failed to get job status: {}", e.getMessage());
                         currentStatus = JobStatus.FINISHED;
                     }
 
+                    LOG.debug("Current job status: {}", currentStatus);
+
                     if (expectedStatus.contains(currentStatus)) {
+                        LOG.info("Job reached expected status: {}", currentStatus);
                         return true;
                     } else if (currentStatus.isTerminalState()) {
                         try {

flink-doris-connector/src/test/java/org/apache/doris/flink/container/config/DorisPorts.java

@@ -0,0 +1,40 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements.  See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership.  The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License.  You may obtain a copy of the License at
+//
+//   http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied.  See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+package org.apache.doris.flink.container.config;
+
+/**
+ * DorisPorts class contains default port configurations for Apache Doris Can be used in E2E testing
+ * and other scenarios requiring access to Doris services
+ */
+public class DorisPorts {
+    /** FE (Frontend) related ports */
+    public static final class FE {
+
+        public static final int HTTP_PORT = 8030;
+        public static final int QUERY_PORT = 9030;
+        public static final int FLIGHT_SQL_PORT = 10030;
+    }
+
+    /** BE (Backend) related ports */
+    public static final class BE {
+
+        public static final int THRIFT_PORT = 9060;
+        public static final int WEBSERVICE_PORT = 8040;
+        public static final int FLIGHT_SQL_PORT = 10040;
+    }
+}

flink-doris-connector/src/test/java/org/apache/doris/flink/container/e2e/Doris2DorisE2ECase.java

@@ -23,6 +23,7 @@
 
 import org.apache.doris.flink.container.AbstractContainerTestBase;
 import org.apache.doris.flink.container.ContainerUtils;
+import org.apache.doris.flink.container.config.DorisPorts.FE;
 import org.apache.doris.flink.table.DorisConfigOptions;
 import org.junit.Test;
 import org.junit.runner.RunWith;
@@ -60,7 +61,6 @@ public void testDoris2Doris() throws Exception {
         env.setParallelism(2);
         env.setRuntimeMode(RuntimeExecutionMode.BATCH);
         final StreamTableEnvironment tEnv = StreamTableEnvironment.create(env);
-
         String sourceDDL =
                 String.format(
                         "CREATE TABLE doris_source ("
@@ -96,13 +96,14 @@ public void testDoris2Doris() throws Exception {
                                 + " 'username' = '%s',"
                                 + " 'password' = '%s',"
                                 + " 'source.use-flight-sql' = '%s',\n"
-                                + " 'source.flight-sql-port' = '9611'"
+                                + " 'source.flight-sql-port' = '%s'"
                                 + ")",
                         getFenodes(),
                         DATABASE_SOURCE + "." + TABLE,
                         getDorisUsername(),
                         getDorisPassword(),
-                        useFlightRead);
+                        useFlightRead,
+                        FE.FLIGHT_SQL_PORT);
         tEnv.executeSql(sourceDDL);
 
         String sinkDDL =
@@ -150,9 +151,9 @@ public void testDoris2Doris() throws Exception {
 
         List<String> excepted =
                 Arrays.asList(
-                        "1,true,127,32767,2147483647,9223372036854775807,170141183460469231731687303715884105727,3.14,2.71828,12345.6789,2025-03-11,2025-03-11T12:34:56,A,Hello, Doris!,This is a string,[\"Alice\", \"Bob\"],{\"key1\":\"value1\", \"key2\":\"value2\"},{\"name\": \"Tom\", \"age\": 30},{\"key\":\"value\"},{\"data\":123,\"type\":\"variant\"}",
-                        "2,false,-128,-32768,-2147483648,-9223372036854775808,-170141183460469231731687303715884105728,-1.23,1.0E-4,-9999.9999,2024-12-25,2024-12-25T23:59:59,B,Doris Test,Another string!,[\"Charlie\", \"David\"],{\"k1\":\"v1\", \"k2\":\"v2\"},{\"name\": \"Jerry\", \"age\": 25},{\"status\":\"ok\"},{\"data\":[1,2,3]}",
-                        "3,true,0,0,0,0,0,0.0,0.0,0.0000,2023-06-15,2023-06-15T08:00,C,Test Doris,Sample text,[\"Eve\", \"Frank\"],{\"alpha\":\"beta\"},{\"name\": \"Alice\", \"age\": 40},{\"nested\":{\"key\":\"value\"}},{\"variant\":\"test\"}",
+                        "1,true,127,32767,2147483647,9223372036854775807,170141183460469231731687303715884105727,3.14,2.71828,12345.6789,2025-03-11,2025-03-11T12:34:56,A,Hello, Doris!,This is a string,[\"Alice\", \"Bob\"],{\"key1\":\"value1\", \"key2\":\"value2\"},{\"name\":\"Tom\", \"age\":30},{\"key\":\"value\"},{\"data\":123,\"type\":\"variant\"}",
+                        "2,false,-128,-32768,-2147483648,-9223372036854775808,-170141183460469231731687303715884105728,-1.23,1.0E-4,-9999.9999,2024-12-25,2024-12-25T23:59:59,B,Doris Test,Another string!,[\"Charlie\", \"David\"],{\"k1\":\"v1\", \"k2\":\"v2\"},{\"name\":\"Jerry\", \"age\":25},{\"status\":\"ok\"},{\"data\":[1,2,3]}",
+                        "3,true,0,0,0,0,0,0.0,0.0,0.0000,2023-06-15,2023-06-15T08:00,C,Test Doris,Sample text,[\"Eve\", \"Frank\"],{\"alpha\":\"beta\"},{\"name\":\"Alice\", \"age\":40},{\"nested\":{\"key\":\"value\"}},{\"variant\":\"test\"}",
                         "4,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null,null");
 
         String query = String.format("SELECT * FROM %s.%s", DATABASE_SINK, TABLE);