diff --git a/.github/trigger_files/beam_PostCommit_Java_Examples_Dataflow_Java.json b/.github/trigger_files/beam_PostCommit_Java_Examples_Dataflow_Java.json new file mode 100644 index 000000000000..e69de29bb2d1 diff --git a/.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml b/.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml index a4a1c3ea04f6..67a48b105955 100644 --- a/.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml +++ b/.github/workflows/beam_PostCommit_Java_Hadoop_Versions.yml @@ -73,11 +73,21 @@ jobs: - name: Setup environment uses: ./.github/actions/setup-environment-action with: - java-version: default - - name: run validatesRunner script + java-version: | + 8 + 11 + - name: run javaHadoopVersionsTest script uses: ./.github/actions/gradle-command-self-hosted-action with: gradle-command: :javaHadoopVersionsTest + # TODO(https://github.com/apache/beam/issues/32189) remove when embedded hive supports Java11 + - name: run java8HadoopVersionsTest script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:hcatalog:hadoopVersionsTest + arguments: | + -PtestJavaVersion=8 \ + -Pjava8Home=$JAVA_HOME_8_X64 \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml b/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml index 47e1bca5571e..66ca5c0650e2 100644 --- a/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml +++ b/.github/workflows/beam_PreCommit_Java_HCatalog_IO_Direct.yml @@ -87,6 +87,10 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action + with: + java-version: | + 8 + 11 - name: run HCatalog IO build script uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -94,6 +98,17 @@ jobs: arguments: | -PdisableSpotlessCheck=true \ -PdisableCheckStyle=true \ + # TODO(https://github.com/apache/beam/issues/32189) remove when embedded hive supports Java11 + - name: Test HCatalog IO on Java8 + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:hcatalog:test + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + -Dfile.encoding=UTF-8 \ + -PtestJavaVersion=8 \ + -Pjava17Home=$JAVA_HOME_8_X64 \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml b/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml index e16955fa46ba..b084c48de049 100644 --- a/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml +++ b/.github/workflows/beam_PreCommit_Java_IOs_Direct.yml @@ -86,6 +86,9 @@ jobs: github_job: ${{ matrix.job_name }} (${{ matrix.job_phrase }}) - name: Setup environment uses: ./.github/actions/setup-environment-action + java-version: | + 8 + 11 - name: run Java IOs PreCommit script uses: ./.github/actions/gradle-command-self-hosted-action with: @@ -94,6 +97,17 @@ jobs: -PdisableSpotlessCheck=true \ -PdisableCheckStyle=true \ -Dfile.encoding=UTF-8 \ + # TODO(https://github.com/apache/beam/issues/32189) remove when embedded hive supports Java11 + - name: run Java8 IOs PreCommit script + uses: ./.github/actions/gradle-command-self-hosted-action + with: + gradle-command: :sdks:java:io:hcatalog:build + arguments: | + -PdisableSpotlessCheck=true \ + -PdisableCheckStyle=true \ + -Dfile.encoding=UTF-8 \ + -PtestJavaVersion=8 \ + -Pjava17Home=$JAVA_HOME_8_X64 \ - name: Archive JUnit Test Results uses: actions/upload-artifact@v4 if: ${{ !success() }} diff --git a/runners/google-cloud-dataflow-java/examples/build.gradle b/runners/google-cloud-dataflow-java/examples/build.gradle index aef68e1ec703..f0898fefc885 100644 --- a/runners/google-cloud-dataflow-java/examples/build.gradle +++ b/runners/google-cloud-dataflow-java/examples/build.gradle @@ -147,15 +147,15 @@ task javaPostCommit() { dependsOn postCommitLegacyWorker } -task postCommitLegacyWorkerJava11(type: Test) { +task postCommitLegacyWorkerJava8(type: Test) { dependsOn ":runners:google-cloud-dataflow-java:worker:shadowJar" def dataflowWorkerJar = project.findProperty('dataflowWorkerJar') ?: project(":runners:google-cloud-dataflow-java:worker").shadowJar.archivePath - systemProperty "java.specification.version", "11" + systemProperty "java.specification.version", "8" with commonConfig(dataflowWorkerJar: dataflowWorkerJar, runWordCount: 'only') } -task java11PostCommit() { - dependsOn postCommitLegacyWorkerJava11 +task java8PostCommit() { + dependsOn postCommitLegacyWorkerJava8 } task postCommitLegacyWorkerJava17(type: Test) { diff --git a/sdks/java/io/hcatalog/build.gradle b/sdks/java/io/hcatalog/build.gradle index c333ad3bcc08..c4f1b76ec390 100644 --- a/sdks/java/io/hcatalog/build.gradle +++ b/sdks/java/io/hcatalog/build.gradle @@ -90,9 +90,20 @@ task hadoopVersionsTest(group: "Verification") { } hadoopVersions.each { kv -> - task "hadoopVersion${kv.key}Test"(type: Test, group: "Verification") { + tasks.create(name: "hadoopVersion${kv.key}Test", type: Test, group: "Verification") { description = "Runs HCatalog tests with Hadoop version $kv.value" classpath = configurations."hadoopVersion$kv.key" + sourceSets.test.runtimeClasspath include '**/*Test.class' } } + +project.tasks.withType(Test).configureEach { + if (JavaVersion.VERSION_1_8.compareTo(JavaVersion.current()) < 0 && project.findProperty('testJavaVersion') != '8') { + useJUnit { + filter { + excludeTestsMatching "org.apache.beam.sdk.io.hcatalog.HCatalogIOTest" + excludeTestsMatching "org.apache.beam.sdk.io.hcatalog.HCatalogBeamSchemaTest" + } + } + } +} diff --git a/sdks/python/test-suites/portable/common.gradle b/sdks/python/test-suites/portable/common.gradle index c87696793f0f..4f232c5b104f 100644 --- a/sdks/python/test-suites/portable/common.gradle +++ b/sdks/python/test-suites/portable/common.gradle @@ -1,3 +1,5 @@ +import static org.apache.beam.gradle.BeamModulePlugin.getSupportedJavaVersion + import org.apache.tools.ant.taskdefs.condition.Os /* @@ -39,7 +41,8 @@ def createFlinkRunnerTestTask(String workerType) { task.configure { dependsOn ":runners:flink:${latestFlinkVersion}:job-server:shadowJar" // The Java SDK worker is required to execute external transforms. - dependsOn ':sdks:java:container:java8:docker' + def suffix = getSupportedJavaVersion() + dependsOn ":sdks:java:container:${suffix}:docker" if (workerType == 'DOCKER') { dependsOn pythonContainerTask } else if (workerType == 'PROCESS') {