Merge branch 'trunk' into KAFKA-18530

# Conflicts:
#	core/src/test/scala/unit/kafka/server/ClientQuotasRequestTest.scala

Author: m1a2st

.github/scripts/develocity_reports.py

@@ -889,7 +889,7 @@ def get_develocity_class_link(class_name: str, threshold_days: int, test_type: s
         threshold_days: Number of days to look back in search
         test_type: Type of test (e.g., "quarantinedTest", "test")
     """
-    base_url = "https://ge.apache.org/scans/tests"
+    base_url = "https://develocity.apache.org/scans/tests"
     params = {
         "search.rootProjectNames": "kafka",
         "search.tags": "github,trunk",
@@ -913,7 +913,7 @@ def get_develocity_method_link(class_name: str, method_name: str, threshold_days
         threshold_days: Number of days to look back in search
         test_type: Type of test (e.g., "quarantinedTest", "test")
     """
-    base_url = "https://ge.apache.org/scans/tests"
+    base_url = "https://develocity.apache.org/scans/tests"
 
     # Extract just the method name without the class prefix
     if '.' in method_name:
@@ -1186,7 +1186,7 @@ def main():
         exit(1)
 
     # Configuration
-    BASE_URL = "https://ge.apache.org"
+    BASE_URL = "https://develocity.apache.org"
     PROJECT = "kafka"
     QUARANTINE_THRESHOLD_DAYS = 7
     MIN_FAILURE_RATE = 0.1

.github/scripts/junit.py

@@ -91,7 +91,10 @@ class TestSuite:
 def clean_test_name(test_name: str) -> str:
     cleaned = test_name.strip("\"").rstrip("()")
     m = method_matcher.match(cleaned)
-    return m.group(1)
+    if m is None:
+        raise ValueError(f"Could not parse test name '{test_name}'. Expected a valid Java method name.")
+    else:
+        return m.group(1)
 
 
 class TestCatalogExporter:

.github/workflows/build.yml

@@ -103,7 +103,7 @@ jobs:
           java-version: 23
           gradle-cache-read-only: ${{ !inputs.is-trunk }}
           gradle-cache-write-only: ${{ inputs.is-trunk }}
-          develocity-access-key: ${{ secrets.GE_ACCESS_TOKEN }}
+          develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
       - name: Compile and validate
         env:
           SCAN_ARG: ${{ inputs.is-public-fork && '--no-scan' || '--scan' }}
@@ -168,7 +168,7 @@ jobs:
           java-version: ${{ matrix.java }}
           gradle-cache-read-only: ${{ !inputs.is-trunk }}
           gradle-cache-write-only: ${{ inputs.is-trunk }}
-          develocity-access-key: ${{ secrets.GE_ACCESS_TOKEN }}
+          develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
 
       # If the load-catalog job failed, we won't be able to download the artifact. Since we don't want this to fail
       # the overall workflow, so we'll continue here without a test catalog.

.github/workflows/ci-complete.yml

@@ -30,7 +30,7 @@ run-name: Build Scans for ${{ github.event.workflow_run.display_title}}
 #
 # This "workflow_run" triggered workflow is run in a privileged context and so does have access to
 # the repository secrets. Here we can download the build scan files produced by a PR and publish
-# them to ge.apache.org.
+# them to develocity.apache.org.
 #
 # If we need to do things like comment on, label, or otherwise modify PRs from public forks. This
 # workflow is the place to do it. PR number is ${{ github.event.workflow_run.pull_requests[0].number }}
@@ -59,7 +59,7 @@ jobs:
         uses: ./.github/actions/setup-gradle
         with:
           java-version: ${{ matrix.java }}
-          develocity-access-key: ${{ secrets.GE_ACCESS_TOKEN }}
+          develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
       - name: Download build scan archive
         id: download-build-scan
         uses: actions/download-artifact@v4

.github/workflows/deflake.yml

@@ -50,7 +50,7 @@ jobs:
         with:
           java-version: ${{ inputs.java-version }}
           gradle-cache-read-only: true
-          develocity-access-key: ${{ secrets.GE_ACCESS_TOKEN }}
+          develocity-access-key: ${{ secrets.DEVELOCITY_ACCESS_KEY }}
       - name: Test
         timeout-minutes: 60
         id: junit-test

.github/workflows/generate-reports.yml

@@ -23,7 +23,6 @@ on:
 jobs:
   flaky-test-report:
     name: Flaky Test Report
-    if : github.event.repository.fork == 'false'
     permissions:
       contents: read
     runs-on: ubuntu-latest
@@ -37,6 +36,7 @@ jobs:
       - name: Setup Python
         uses: ./.github/actions/setup-python
       - name: Run Report
+        if : ${{ ! github.event.repository.fork }}
         env:
           DEVELOCITY_ACCESS_TOKEN: ${{ secrets.DV_API_ACCESS }}
         run: |

README.md

@@ -110,8 +110,8 @@ fail due to code changes. You can just run:
 Using compiled files:
 
     KAFKA_CLUSTER_ID="$(./bin/kafka-storage.sh random-uuid)"
-    ./bin/kafka-storage.sh format --standalone -t $KAFKA_CLUSTER_ID -c config/kraft/reconfig-server.properties
-    ./bin/kafka-server-start.sh config/kraft/reconfig-server.properties
+    ./bin/kafka-storage.sh format --standalone -t $KAFKA_CLUSTER_ID -c config/server.properties
+    ./bin/kafka-server-start.sh config/server.properties
 
 Using docker image:
 

build.gradle

@@ -49,7 +49,7 @@ ext {
   gradleVersion = versions.gradle
   minClientJavaVersion = 11
   minNonClientJavaVersion = 17
-  modulesNeedingJava11 = [":clients", ":generator", ":streams", ":streams:test-utils", ":streams-scala", ":test-common:test-common-runtime"]
+  modulesNeedingJava11 = [":clients", ":generator", ":streams", ":streams:test-utils", ":streams-scala", ":test-common:test-common-util"]
 
   buildVersionFileName = "kafka-version.properties"
 
@@ -139,10 +139,11 @@ ext {
   runtimeTestLibs = [
     libs.slf4jLog4j2,
     libs.junitPlatformLanucher,
-    project(":test-common:test-common-runtime")
+    libs.jacksonDatabindYaml,
+    project(":test-common:test-common-util")
   ]
 
-  log4jRuntimeLibs = [
+  log4jReleaseLibs = [
     libs.slf4jLog4j2,
     libs.log4j1Bridge2Api,
     libs.jacksonDatabindYaml
@@ -1059,7 +1060,7 @@ project(':core') {
   }
 
   dependencies {
-    releaseOnly log4jRuntimeLibs
+    releaseOnly log4jReleaseLibs
     // `core` is often used in users' tests, define the following dependencies as `api` for backwards compatibility
     // even though the `core` module doesn't expose any public API
     api project(':clients')
@@ -1102,8 +1103,9 @@ project(':core') {
     testImplementation project(':server-common').sourceSets.test.output
     testImplementation project(':storage:storage-api').sourceSets.test.output
     testImplementation project(':server').sourceSets.test.output
-    testImplementation project(':test-common')
-    testImplementation project(':test-common:test-common-api')
+    testImplementation project(':test-common:test-common-runtime')
+    testImplementation project(':test-common:test-common-internal-api')
+    testImplementation project(':test-common:test-common-util')
     testImplementation libs.bcpkix
     testImplementation libs.mockitoCore
     testImplementation(libs.apacheda) {
@@ -1535,21 +1537,17 @@ project(':group-coordinator') {
   srcJar.dependsOn 'processMessages'
 }
 
-project(':test-common') {
-  // Test framework stuff. Implementations that support test-common-api
+
+project(':test-common:test-common-internal-api') {
+  // Interfaces, config classes, and other test APIs. Java 17 only
   base {
-    archivesName = "kafka-test-common"
+    archivesName = "kafka-test-common-internal-api"
   }
 
   dependencies {
-    implementation project(':core')
-    implementation project(':metadata')
-    implementation project(':server')
-    implementation project(':raft')
-    implementation project(':storage')
-    implementation project(':server-common')
-    implementation libs.jacksonDatabindYaml
-    implementation libs.slf4jApi
+    implementation project(':server-common') // Only project dependency allowed
+
+    implementation libs.junitJupiterApi
 
     testImplementation libs.junitJupiter
     testImplementation libs.mockitoCore
@@ -1559,41 +1557,30 @@ project(':test-common') {
   }
 
   checkstyle {
-    configProperties = checkstyleConfigProperties("import-control-test-common.xml")
+    configProperties = checkstyleConfigProperties("import-control-test-common-internal-api.xml")
   }
 
   javadoc {
     enabled = false
   }
 }
 
-project(':test-common:test-common-api') {
-  // Interfaces, config classes, and other test APIs
+project(':test-common:test-common-util') {
+  // Runtime-only JUnit extensions for entire project. Java 11 only
   base {
-    archivesName = "kafka-test-common-api"
+    archivesName = "kafka-test-common-util"
   }
 
   dependencies {
-    implementation project(':clients')
-    implementation project(':core')
-    implementation project(':group-coordinator')
-    implementation project(':metadata')
-    implementation project(':raft')
-    implementation project(':server')
-    implementation project(':server-common')
-    implementation project(':storage')
-    implementation project(':test-common')
+    implementation libs.junitPlatformLanucher
     implementation libs.junitJupiterApi
-
-    testImplementation libs.junitJupiter
-    testImplementation libs.mockitoCore
+    implementation libs.junitJupiter
+    implementation libs.slf4jApi
     testImplementation testLog4j2Libs
-
-    testRuntimeOnly runtimeTestLibs
   }
 
   checkstyle {
-    configProperties = checkstyleConfigProperties("import-control-test-common-api.xml")
+    configProperties = checkstyleConfigProperties("import-control-test-common-util.xml")
   }
 
   javadoc {
@@ -1602,21 +1589,36 @@ project(':test-common:test-common-api') {
 }
 
 project(':test-common:test-common-runtime') {
-  // Runtime-only test code including JUnit extentions
+  // Runtime-only JUnit extensions for integration tests. Java 17 only
   base {
     archivesName = "kafka-test-common-runtime"
   }
 
   dependencies {
+    implementation project(':test-common:test-common-internal-api')
+    implementation project(':clients')
+    implementation project(':core')
+    implementation project(':group-coordinator')
+    implementation project(':metadata')
+    implementation project(':raft')
+    implementation project(':server')
+    implementation project(':server-common')
+    implementation project(':storage')
+
     implementation libs.junitPlatformLanucher
-    implementation libs.junitJupiterApi
     implementation libs.junitJupiter
+    implementation libs.jacksonDatabindYaml
     implementation libs.slf4jApi
+
+    testImplementation libs.junitJupiter
+    testImplementation libs.mockitoCore
     testImplementation testLog4j2Libs
+
+    testRuntimeOnly runtimeTestLibs
   }
 
   checkstyle {
-    configProperties = checkstyleConfigProperties("import-control-test-common-api.xml")
+    configProperties = checkstyleConfigProperties("import-control-test-common-runtime.xml")
   }
 
   javadoc {
@@ -1644,8 +1646,8 @@ project(':transaction-coordinator') {
     testImplementation libs.junitJupiter
     testImplementation libs.mockitoCore
     testImplementation project(':clients').sourceSets.test.output
-    testImplementation project(':test-common')
-    testImplementation project(':test-common:test-common-api')
+    testImplementation project(':test-common:test-common-runtime')
+    testImplementation project(':test-common:test-common-internal-api')
 
     testRuntimeOnly runtimeTestLibs
 
@@ -1866,6 +1868,7 @@ project(':clients') {
     compileOnly libs.jose4j          // for SASL/OAUTHBEARER JWT validation; only used by broker
 
 
+    testImplementation project(':test-common:test-common-util')
     testImplementation libs.bcpkix
     testImplementation libs.jacksonJakartarsJsonProvider
     testImplementation libs.jose4j
@@ -1880,7 +1883,6 @@ project(':clients') {
     testRuntimeOnly libs.jacksonDatabind
     testRuntimeOnly libs.jacksonJDK8Datatypes
     testRuntimeOnly runtimeTestLibs
-    testRuntimeOnly log4jRuntimeLibs
 
     generator project(':generator')
   }
@@ -2267,7 +2269,8 @@ project(':storage') {
     testImplementation project(':clients').sourceSets.test.output
     testImplementation project(':core')
     testImplementation project(':core').sourceSets.test.output
-    testImplementation project(':test-common:test-common-api')
+    testImplementation project(':test-common:test-common-internal-api')
+    testImplementation project(':test-common:test-common-runtime')
     testImplementation project(':server')
     testImplementation project(':server-common')
     testImplementation project(':server-common').sourceSets.test.output
@@ -2424,7 +2427,7 @@ project(':tools') {
   }
 
   dependencies {
-    releaseOnly log4jRuntimeLibs
+    releaseOnly log4jReleaseLibs
 
     implementation project(':clients')
     implementation project(':metadata')
@@ -2456,7 +2459,8 @@ project(':tools') {
     testImplementation project(':server').sourceSets.test.output
     testImplementation project(':core')
     testImplementation project(':core').sourceSets.test.output
-    testImplementation project(':test-common:test-common-api')
+    testImplementation project(':test-common:test-common-internal-api')
+    testImplementation project(':test-common:test-common-runtime')
     testImplementation project(':server-common')
     testImplementation project(':server-common').sourceSets.test.output
     testImplementation project(':connect:api')
@@ -2467,7 +2471,6 @@ project(':tools') {
     testImplementation project(':streams')
     testImplementation project(':streams').sourceSets.test.output
     testImplementation project(':streams:integration-tests').sourceSets.test.output
-    testImplementation project(':test-common')
     testImplementation libs.junitJupiter
     testImplementation libs.mockitoCore
     testImplementation libs.mockitoJunitJupiter // supports MockitoExtension
@@ -2648,7 +2651,6 @@ project(':streams') {
 
     testRuntimeOnly project(':streams:test-utils')
     testRuntimeOnly runtimeTestLibs
-    testRuntimeOnly log4jRuntimeLibs
 
     generator project(':generator')
   }
@@ -2839,7 +2841,7 @@ project(':streams:integration-tests') {
     testImplementation project(':storage')
     testImplementation project(':streams').sourceSets.test.output
     testImplementation project(':streams:streams-scala')
-    testImplementation project(':test-common')
+    testImplementation project(':test-common:test-common-runtime')
     testImplementation project(':tools')
     testImplementation project(':transaction-coordinator')
     testImplementation libs.bcpkix
@@ -3515,14 +3517,15 @@ project(':connect:runtime') {
     testImplementation project(':server')
     testImplementation project(':metadata')
     testImplementation project(':server-common')
-    testImplementation project(':test-common')
+    testImplementation project(':test-common:test-common-internal-api')
+    testImplementation project(':test-common:test-common-util')
+    testImplementation project(':test-common:test-common-runtime')
     testImplementation project(':server-common')
     testImplementation project(':server')
     testImplementation project(':group-coordinator')
     testImplementation project(':storage')
     testImplementation project(':connect:test-plugins')
     testImplementation project(':server-common').sourceSets.test.output
-    testImplementation project(':test-common:test-common-api')
 
     testImplementation libs.jacksonDatabindYaml
     testImplementation libs.junitJupiter
@@ -3636,7 +3639,7 @@ project(':connect:file') {
     testImplementation project(':connect:runtime')
     testImplementation project(':connect:runtime').sourceSets.test.output
     testImplementation project(':core')
-    testImplementation project(':test-common')
+    testImplementation project(':test-common:test-common-runtime')
     testImplementation project(':server-common').sourceSets.test.output
 
     testRuntimeOnly runtimeTestLibs
@@ -3740,7 +3743,7 @@ project(':connect:mirror') {
     testImplementation project(':clients').sourceSets.test.output
     testImplementation project(':connect:runtime').sourceSets.test.output
     testImplementation project(':core')
-    testImplementation project(':test-common')
+    testImplementation project(':test-common:test-common-runtime')
     testImplementation project(':server')
     testImplementation project(':server-common').sourceSets.test.output
 

checkstyle/import-control-coordinator-common.xml

@@ -37,6 +37,7 @@
     <allow pkg="org.apache.kafka.common.security" />
     <allow pkg="org.apache.kafka.common.serialization" />
     <allow pkg="org.apache.kafka.common.utils" />
+    <allow pkg="org.apache.kafka.common.test.api" />
 
     <subpackage name="coordinator">
         <subpackage name="common">