From a3f4206d392ac95276a0754fe693e998e00a1c04 Mon Sep 17 00:00:00 2001 From: Jakub Stejskal Date: Fri, 8 Aug 2025 09:42:26 +0200 Subject: [PATCH] Make http client for fabric8 configurable Signed-off-by: Jakub Stejskal --- .github/workflows/template.flink-ci.yml | 14 + .../template.flink-kubernetes-ci.yml | 98 +++++++ flink-kubernetes/pom.xml | 251 +++++++++++++++++- 3 files changed, 362 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/template.flink-kubernetes-ci.yml diff --git a/.github/workflows/template.flink-ci.yml b/.github/workflows/template.flink-ci.yml index 4ee0f08d6df4b..bd992d8cf510c 100644 --- a/.github/workflows/template.flink-ci.yml +++ b/.github/workflows/template.flink-ci.yml @@ -286,6 +286,20 @@ jobs: if: ${{ !cancelled() && (failure() || !steps.docker-cache.cache.hit) }} run: ./tools/azure-pipelines/cache_docker_images.sh save + flink-kubernetes: + name: "Build & test flink-kubernetes with multiple HTTP clients" + needs: compile + strategy: + fail-fast: false + matrix: + http-client: [ "okhttp", "jdk", "jetty", "vertx" ] + uses: ./.github/workflows/template.flink-kubernetes-ci.yml + with: + http-client: ${{ matrix.http-client }} + jdk_version: ${{ inputs.jdk_version }} + cache-name: ${{ needs.compile.outputs.stringified-workflow-name }}-${{ github.run_number }} + environment: "${{ inputs.environment }}" + e2e: name: "E2E (group ${{ matrix.group }})" needs: compile diff --git a/.github/workflows/template.flink-kubernetes-ci.yml b/.github/workflows/template.flink-kubernetes-ci.yml new file mode 100644 index 0000000000000..1241d542b9cec --- /dev/null +++ b/.github/workflows/template.flink-kubernetes-ci.yml @@ -0,0 +1,98 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Workflow template for triggering the Flink's test suite. + +name: "Apache Flink Test Workflow Template - for module flink-kubernetes" + +on: + workflow_call: + inputs: + http-client: + description: "Type of fabric8 HTTP client implementation (okhttp, vertx, jetty, jdk)" + default: okhttp + type: string + jdk_version: + description: "Java version" + default: "17" + type: string + cache-name: + description: "Suffix for artifacts name in cache" + default: "" + type: string + environment: + description: "Defines environment variables for downstream scripts." + required: true + type: string + +permissions: read-all + +env: + # The following environment variables need to be overwritten here because the e2e tests do not + # run in containers. + MAVEN_REPO_FOLDER: ${{ github.workspace }}/.m2/repository + MAVEN_ARGS: -Dmaven.repo.local=${{ github.workspace }}/.m2/repository + FLINK_ARTIFACT_DIR: ${{ github.workspace }} + DOCKER_IMAGES_CACHE_FOLDER: ${{ github.workspace }}/.docker-cache + +jobs: + build-and-test: + runs-on: ubuntu-22.04 + steps: + - name: "Flink Checkout" + uses: actions/checkout@v4 + with: + persist-credentials: false + + - name: "Initialize job" + uses: "./.github/actions/job_init" + with: + jdk_version: ${{ inputs.jdk_version }} + maven_repo_folder: ${{ env.MAVEN_REPO_FOLDER }} + + - name: "Download build artifacts from compile job" + uses: actions/download-artifact@v4 + with: + name: build-artifacts-${{ inputs.cache-name }} + path: ${{ env.FLINK_ARTIFACT_DIR }} + + - name: "Unpack build artifact" + run: ./tools/azure-pipelines/unpack_build_artifact.sh + + - name: "Build Flink" + uses: "./.github/actions/run_mvn" + timeout-minutes: 20 + with: + maven-parameters: "install -DskipTests -Dfast $PROFILE -Pskip-webui-build -Phttpclient-${{ inputs.http-client }}" + env: "${{ inputs.environment }}" + + - name: "Build flink-kubernetes with ${{ inputs.http-client }} with unit tests" + timeout-minutes: 20 + run: | + ./mvnw install -pl flink-kubernetes -Phttpclient-${{ inputs.http-client }} + + - name: "Run ITs" + timeout-minutes: 20 + run: | + FLINK_DIR=`pwd`/build-target ./flink-end-to-end-tests/run-single-test.sh ./flink-end-to-end-tests/test-scripts/test_kubernetes_itcases.sh + env: + MAVEN_ARGS: "-Phttpclient-${{ inputs.http-client }}" + + - name: "Run e2e smoke" + timeout-minutes: 20 + run: | + FLINK_DIR=`pwd`/build-target ./flink-end-to-end-tests/run-single-test.sh ./flink-end-to-end-tests/test-scripts/test_kubernetes_application.sh + env: + MAVEN_ARGS: "-Phttpclient-${{ inputs.http-client }}" diff --git a/flink-kubernetes/pom.xml b/flink-kubernetes/pom.xml index 888b2305f7757..f27d8cb5c040c 100644 --- a/flink-kubernetes/pom.xml +++ b/flink-kubernetes/pom.xml @@ -32,6 +32,10 @@ under the License. 7.3.1 + + + okhttp + 4.1.108.Final io.fabric8 - kubernetes-httpclient-okhttp + kubernetes-httpclient-${kubernetes.client.httpclient.impl} ${kubernetes.client.version} ${flink.markBundledAsOptional} + + + io.fabric8 + kubernetes-httpclient-vertx + + @@ -252,4 +262,243 @@ under the License. + + + + + httpclient-okhttp + true + + okhttp + + + + io.fabric8 + kubernetes-httpclient-${kubernetes.client.httpclient.impl} + ${kubernetes.client.version} + ${flink.markBundledAsOptional} + + + io.fabric8 + kubernetes-httpclient-vertx + + + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + ban-non-okhttp-providers + enforce + + + + true + + + io.fabric8:kubernetes-httpclient-vertx + io.fabric8:kubernetes-httpclient-jdk + io.fabric8:kubernetes-httpclient-jetty + + + + true + + + + + + + + + + + httpclient-vertx + + vertx + + + + + io.fabric8 + kubernetes-httpclient-${kubernetes.client.httpclient.impl} + ${kubernetes.client.version} + ${flink.markBundledAsOptional} + + + com.squareup.okhttp3 + okhttp + + + + + io.netty + netty-codec-http + ${netty.override.version} + compile + + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-flink + + + + + io.vertx:* + io.netty:* + + + + + + io.vertx + org.apache.flink.kubernetes.shaded.io.vertx + + + io.netty + org.apache.flink.kubernetes.shaded.io.netty + + + + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + ban-okhttp-when-vertx + enforce + + + + + com.squareup.okhttp3:okhttp + io.fabric8:kubernetes-httpclient-jetty + io.fabric8:kubernetes-httpclient-jdk + + + + true + + + + + + + + + + + httpclient-jetty + + jetty + + + + + + org.apache.maven.plugins + maven-shade-plugin + + + shade-flink + + + + + org.eclipse.jetty:* + org.eclipse.jetty.http2:* + org.eclipse.jetty.websocket:* + + + + + + org.eclipse.jetty + org.apache.flink.kubernetes.shaded.org.eclipse.jetty + + + + + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + ban-okhttp-when-jetty + enforce + + + + + com.squareup.okhttp3:okhttp + io.fabric8:kubernetes-httpclient-vertx + io.fabric8:kubernetes-httpclient-jdk + + + + true + + + + + + + + + + + httpclient-jdk + + jdk + + + + + + org.apache.maven.plugins + maven-enforcer-plugin + + + ban-okhttp-when-jdk + enforce + + + + + com.squareup.okhttp3:okhttp + io.fabric8:kubernetes-httpclient-vertx + io.fabric8:kubernetes-httpclient-jetty + + + + true + + + + + + + +