diff --git a/.circleci/config.yml b/.circleci/config.yml index ac783c1af..fcb86903c 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -55,6 +55,9 @@ commands: sstable_format: type: string default: "big" + cassandra: + type: string + default: "" description: Build and test against Spark <> Scala <> <> steps: - run: @@ -62,11 +65,12 @@ commands: SPARK_VERSION: "<>" SCALA_VERSION: "<>" JDK_VERSION: "<>" - INTEGRATION_MAX_PARALLEL_FORKS: 1 - INTEGRATION_MAX_HEAP_SIZE: "1500M" - CORE_MAX_PARALLEL_FORKS: 2 + INTEGRATION_MAX_PARALLEL_FORKS: 2 + INTEGRATION_MAX_HEAP_SIZE: "3072m" + CORE_MAX_PARALLEL_FORKS: 3 CORE_TEST_MAX_HEAP_SIZE: "2048m" CASSANDRA_USE_JDK11: <> + CASSANDRA_VERSION: "<>" command: | export GRADLE_OPTS="-Xmx2g -Dorg.gradle.jvmargs=-Xmx2g" # Run compile/unit tests, skipping integration tests @@ -92,7 +96,7 @@ commands: SPARK_VERSION: "<>" SCALA_VERSION: "<>" JDK_VERSION: "<>" - INTEGRATION_MAX_PARALLEL_FORKS: 1 + INTEGRATION_MAX_PARALLEL_FORKS: 2 INTEGRATION_MAX_HEAP_SIZE: "2500M" CASSANDRA_USE_JDK11: <> command: | @@ -143,7 +147,7 @@ jobs: - "*.jar" - "org/**/*" - spark3-2_12-jdk11-big: + spark3-2_12-jdk11-big-c40: docker: - image: cimg/openjdk:11.0 resource_class: large @@ -158,132 +162,156 @@ jobs: jdk: "11" use_jdk11: "true" sstable_format: "big" + cassandra: "4.0" - store_artifacts: + when: always path: build/test-reports destination: test-reports - store_artifacts: + when: always path: build/reports destination: reports - store_test_results: + when: always path: build/test-reports - int-c4-spark3-2_12-jdk11: - parallelism: 8 + spark3-2_12-jdk11-big-c41: docker: - image: cimg/openjdk:11.0 resource_class: large steps: - - setup_remote_docker - install_common - checkout - attach_workspace: at: dependencies - - run_integration: + - run_build: spark: "3" scala: "2.12" jdk: "11" use_jdk11: "true" - cassandra: "4.1.4" + sstable_format: "big" + cassandra: "4.1" - store_artifacts: + when: always path: build/test-reports destination: test-reports - store_artifacts: + when: always path: build/reports destination: reports - - store_artifacts: - path: cassandra-analytics-integration-tests - destination: int-tests-misc - - store_test_results: + when: always path: build/test-reports - spark3-2_13-jdk11-big: + int-c4-spark3-2_12-jdk11: + parallelism: 8 docker: - image: cimg/openjdk:11.0 resource_class: large steps: + - setup_remote_docker - install_common - checkout - attach_workspace: at: dependencies - - run_build: + - run_integration: spark: "3" - scala: "2.13" + scala: "2.12" jdk: "11" use_jdk11: "true" - sstable_format: "big" + cassandra: "4.0.17" - store_artifacts: + when: always path: build/test-reports destination: test-reports - store_artifacts: + when: always path: build/reports destination: reports + - store_artifacts: + when: always + path: cassandra-analytics-integration-tests + destination: int-tests-misc + - store_test_results: + when: always path: build/test-reports - spark3-2_13-jdk11-bti: + int-c41-spark3-2_12-jdk11: + parallelism: 8 docker: - image: cimg/openjdk:11.0 resource_class: large steps: + - setup_remote_docker - install_common - checkout - attach_workspace: at: dependencies - - run_build: + - run_integration: spark: "3" - scala: "2.13" + scala: "2.12" jdk: "11" use_jdk11: "true" - sstable_format: "bti" + cassandra: "4.1.4" - store_artifacts: + when: always path: build/test-reports destination: test-reports - store_artifacts: + when: always path: build/reports destination: reports + - store_artifacts: + when: always + path: cassandra-analytics-integration-tests + destination: int-tests-misc + - store_test_results: + when: always path: build/test-reports - int-c4-spark3-2_13-jdk11: - parallelism: 8 + spark3-2_13-jdk11-bti-c50: docker: - image: cimg/openjdk:11.0 resource_class: large steps: - - setup_remote_docker - install_common - checkout - attach_workspace: at: dependencies - - run_integration: + - run_build: spark: "3" scala: "2.13" jdk: "11" use_jdk11: "true" - cassandra: "4.1.4" + sstable_format: "bti" + cassandra: "5.0" - store_artifacts: + when: always path: build/test-reports destination: test-reports - store_artifacts: + when: always path: build/reports destination: reports - store_test_results: + when: always path: build/test-reports int-c5-spark3-2_13-jdk11: @@ -305,14 +333,17 @@ jobs: cassandra: "5.0.5" - store_artifacts: + when: always path: build/test-reports destination: test-reports - store_artifacts: + when: always path: build/reports destination: reports - store_test_results: + when: always path: build/test-reports workflows: @@ -320,19 +351,23 @@ workflows: build-and-test: jobs: - build-deps-jdk11 - - spark3-2_12-jdk11-big: + + # Unit tests: split by Cassandra version to reduce per-process memory pressure + - spark3-2_12-jdk11-big-c40: requires: - build-deps-jdk11 - - spark3-2_13-jdk11-big: + - spark3-2_12-jdk11-big-c41: requires: - build-deps-jdk11 - - spark3-2_13-jdk11-bti: + - spark3-2_13-jdk11-bti-c50: requires: - build-deps-jdk11 + + # Integration tests - int-c4-spark3-2_12-jdk11: requires: - build-deps-jdk11 - - int-c4-spark3-2_13-jdk11: + - int-c41-spark3-2_12-jdk11: requires: - build-deps-jdk11 - int-c5-spark3-2_13-jdk11: diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 953632909..66bfce6c4 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -73,16 +73,21 @@ jobs: key: build-${{ github.sha }} unit-test: - name: Unit test + name: Unit test - Scala ${{ matrix.scala }} ${{ matrix.sstable-format }} C${{ matrix.cassandra }} needs: build runs-on: ubuntu-latest strategy: matrix: - scala: [ '2.12', '2.13' ] - sstable-format: [ 'big', 'bti' ] - exclude: - - scala: "2.12" - sstable-format: "bti" + include: + - scala: '2.13' + sstable-format: 'bti' + cassandra: '5.0' + - scala: '2.12' + sstable-format: 'big' + cassandra: '4.1' + - scala: '2.12' + sstable-format: 'big' + cassandra: '4.0' fail-fast: false steps: - name: Setup JDK @@ -112,29 +117,35 @@ jobs: export SPARK_VERSION="3" export SCALA_VERSION="${{ matrix.scala }}" export JDK_VERSION="11" - export INTEGRATION_MAX_PARALLEL_FORKS=1 - export INTEGRATION_MAX_HEAP_SIZE="1500M" + export INTEGRATION_MAX_PARALLEL_FORKS=3 + export INTEGRATION_MAX_HEAP_SIZE="2048M" export CASSANDRA_USE_JDK11=true - + export CASSANDRA_VERSION="${{ matrix.cassandra }}" + ./gradlew --stacktrace clean assemble check -x cassandra-analytics-integration-tests:test -Dcassandra.analytics.bridges.sstable_format=${{ matrix.sstable-format }} integration-test: - name: Integration test + name: Integration test - ${{ matrix.config }} (${{ matrix.job_index }}) needs: build runs-on: ubuntu-latest strategy: + # GHA generates a cross-product of 'config' × 'job_index' (3 × 5 = 15 jobs). + # The 'include' entries don't add new combinations — they augment existing ones + # by matching on 'config' and injecting 'scala' and 'cassandra' into each match. + # To add a new version: add one entry to 'config' and one to 'include'. matrix: - scala: [ '2.12', '2.13' ] - cassandra: [ '4.0.17', '4.1.4', '5.0.5' ] - job_index: [ 0, 1, 2, 3, 4 ] - job_total: [ 5 ] - exclude: - - scala: "2.12" - cassandra: "5.0.5" - - scala: "2.12" - cassandra: "4.1.4" - - scala: "2.13" - cassandra: "4.0.17" + config: ['s2.13-c5.0.5', 's2.12-c4.1.4', 's2.12-c4.0.17'] + job_index: [0, 1, 2, 3, 4] + include: + - config: 's2.13-c5.0.5' + scala: '2.13' + cassandra: '5.0.5' + - config: 's2.12-c4.1.4' + scala: '2.12' + cassandra: '4.1.4' + - config: 's2.12-c4.0.17' + scala: '2.12' + cassandra: '4.0.17' fail-fast: false steps: - name: Setup JDK @@ -174,7 +185,11 @@ jobs: ./gradlew --stacktrace clean assemble cd cassandra-analytics-integration-tests/src/test/java - CLASSNAMES=$(find . -name '*Test.java' | sort | cut -c 3- | sed 's@/@.@g' | sed 's/.\{5\}$//' | awk 'NR % ${{ matrix.job_total }} == ${{ matrix.job_index }}') + # Shuffle test classes using the commit SHA as seed for reproducible randomization, + # then shard across runners via round-robin on the shuffled order. + CLASSNAMES=$(find . -name '*Test.java' | cut -c 3- | sed 's@/@.@g' | sed 's/.\{5\}$//' \ + | python3 -c "import random,sys; lines=sys.stdin.read().splitlines(); random.seed('$GITHUB_SHA'); random.shuffle(lines); print('\n'.join(lines))" \ + | awk 'NR % 5 == ${{ matrix.job_index }}') cd ../../../.. EXIT_STATUS=0 diff --git a/build.gradle b/build.gradle index c556f9948..69b451308 100644 --- a/build.gradle +++ b/build.gradle @@ -59,6 +59,39 @@ if (!actualJdkVersion.startsWith(ext.jdkLabel)) ext.dependencyLocation = (System.getenv("CASSANDRA_DEP_DIR") ?: "${rootDir}/dependencies") + "/" +// Cassandra version maps used by subproject test tasks to filter parameterized tests +// to a single version when CASSANDRA_VERSION env var is set (e.g. in CI). +// NOTE: Both maps below must stay in sync with the defaults in CassandraVersion.java +// (cassandra-analytics-common/.../bridge/CassandraVersion.java): +// - cassandraVersionEnumMap values must match the implemented_versions default +// - cassandraFullVersionMap values must match the supported_versions default +ext.cassandraVersionEnumMap = ["4.0": "FOURZERO", "4.1": "FOURONE", "5.0": "FIVEZERO"] +ext.cassandraFullVersionMap = ["4.0": "4.0.17", "4.1": "4.1.4", "5.0": "5.0.5"] + +// Shared helper: sets implemented_versions and supported_versions system properties on a Test task. +// When majorMinor is provided (e.g. "4.0"), uses that version directly. +// When majorMinor is null, falls back to the CASSANDRA_VERSION env var (CI path). +// Returns a map with version info [majorMinor, enumName, fullVersion], or null when no version is set. +ext.applyCassandraVersionFilter = { Test task, String majorMinor = null -> + if (!majorMinor) { + def ver = System.getenv("CASSANDRA_VERSION") + if (!ver) return null + majorMinor = ver.split("\\.").take(2).join(".") + } + + def enumName = cassandraVersionEnumMap[majorMinor] + if (!enumName) { + throw new GradleException("Unknown CASSANDRA_VERSION: ${majorMinor}. " + + "Valid versions: ${cassandraVersionEnumMap.keySet()}") + } + def fullVersion = cassandraFullVersionMap[majorMinor] + + task.systemProperty "cassandra.analytics.bridges.implemented_versions", enumName + task.systemProperty "cassandra.analytics.bridges.supported_versions", "cassandra-${fullVersion}" + + return [majorMinor: majorMinor, enumName: enumName, fullVersion: fullVersion] +} + def buildDir = layout.buildDirectory.get().asFile.toPath() def ratExcludeFilePath = buildDir.resolve(".rat-excludes.txt") System.out.println("Rat exclude file:" + ratExcludeFilePath) @@ -214,7 +247,7 @@ subprojects { } } - test { + tasks.withType(Test).configureEach { def heapDumpPath = "${project.rootProject.rootDir}/build/${project.name}/heapDumps" Files.createDirectories(Paths.get(heapDumpPath)) if (JavaVersion.current().isJava11Compatible()) { diff --git a/cassandra-analytics-cdc/build.gradle b/cassandra-analytics-cdc/build.gradle index 38c284065..e413b242d 100644 --- a/cassandra-analytics-cdc/build.gradle +++ b/cassandra-analytics-cdc/build.gradle @@ -141,27 +141,50 @@ jar { } } -test { - systemProperty "cassandra.sidecar.versions_to_test", (System.getenv("CASSANDRA_VERSION") ?: "4.0.17,5.0.5") +// Assuming container w/8g ram, 2x3G == 6G + overhead; override via CDC_MAX_PARALLEL_FORKS for local dev +def cdcMaxHeap = '3072m' +def cdcMaxParallelForks = (System.getenv("CDC_MAX_PARALLEL_FORKS") ?: "2") as int + +// Shared test configuration for CDC tests. When majorMinor is specified (e.g. "4.0"), +// tests run against that single Cassandra version. When null, uses env var filtering +// (CI) or falls back to all supported versions (local dev). +// Using a closure (not a method) so it captures the local variables above. +def configureCdcTestTask = { Test task, String majorMinor = null -> + def versionInfo = rootProject.ext.applyCassandraVersionFilter(task, majorMinor) + if (versionInfo) { + task.systemProperty "cassandra.sidecar.versions_to_test", versionInfo.fullVersion + } else { + // Full version format to match CDC's TestVersionSupplier; tests both versions for backward compat. + // 4.1 intentionally excluded from gradlew defaults to keep local iteration fast; + // use testCassandra41 for targeted 4.1 runs. CI covers 4.1 via CASSANDRA_VERSION env var. + task.systemProperty "cassandra.sidecar.versions_to_test", "4.0.17,5.0.5" + } - minHeapSize = '1024m' - maxHeapSize = '3072m' - maxParallelForks = Math.max(Runtime.runtime.availableProcessors() * 2, 8) - forkEvery = 1 // Enables different end-to-end test classes use Spark contexts with different configurations + task.minHeapSize = '1024m' + task.maxHeapSize = cdcMaxHeap + task.maxParallelForks = cdcMaxParallelForks + task.forkEvery = 1 // Enables different end-to-end test classes use Spark contexts with different configurations // Make it so unit tests run on a Jar with Cassandra bridge implementations built in - dependsOn(tasks.jar) - classpath += files(jar.archiveFile) - useJUnitPlatform() - reports { - def destDir = Paths.get(rootProject.rootDir.absolutePath, "build", "test-reports", "cdc").toFile() - junitXml { - enabled true - destination = destDir - } - html { - enabled true - destination = destDir - } + task.dependsOn(tasks.jar) + task.classpath += files(jar.archiveFile) + task.useJUnitPlatform() + def destDir = Paths.get(rootProject.rootDir.absolutePath, "build", "test-reports", "cdc").toFile() + task.reports.junitXml.outputLocation = destDir + task.reports.html.outputLocation = destDir +} + +test { + configureCdcTestTask(delegate) +} + +// Per-Cassandra-version test tasks for IDE convenience. +// These show up in IntelliJ's Gradle panel alongside 'test'. +rootProject.ext.cassandraVersionEnumMap.keySet().each { majorMinor -> + def suffix = majorMinor.replace(".", "") + tasks.register("testCassandra${suffix}", Test) { + description = "Run CDC tests against Cassandra ${majorMinor}" + group = 'verification' + configureCdcTestTask(delegate, majorMinor) } } diff --git a/cassandra-analytics-cdc/src/test/java/org/apache/cassandra/cdc/test/TestCdcBridgeProvider.java b/cassandra-analytics-cdc/src/test/java/org/apache/cassandra/cdc/test/TestCdcBridgeProvider.java index e5e1044ff..8805d29b9 100644 --- a/cassandra-analytics-cdc/src/test/java/org/apache/cassandra/cdc/test/TestCdcBridgeProvider.java +++ b/cassandra-analytics-cdc/src/test/java/org/apache/cassandra/cdc/test/TestCdcBridgeProvider.java @@ -120,6 +120,19 @@ public CassandraVersion version() BRIDGES.put(version, CdcBridgeFactory.get(version)); MESSAGE_CONVERTERS.put(version, new JdkMessageConverter(BRIDGES.get(version).cassandraTypes())); + + // When the bridge jar is shared across versions (e.g. FOURONE uses the four-zero bridge, + // whose getVersion() returns FOURZERO), register under the bridge's reported version too. + // This ensures lookups via bridge.getVersion() find the correct entries. + CassandraBridge bridgeInstance = BRIDGES.get(version); + CassandraVersion bridgeVersion = bridgeInstance.getVersion(); + if (bridgeVersion != version) + { + OPTIONS.putIfAbsent(bridgeVersion, OPTIONS.get(version)); + COMMIT_LOG_DIRS.putIfAbsent(bridgeVersion, commitLogDir); + BRIDGES.putIfAbsent(bridgeVersion, bridgeInstance); + MESSAGE_CONVERTERS.putIfAbsent(bridgeVersion, MESSAGE_CONVERTERS.get(version)); + } } public static CdcOptions getCdcOptions(CassandraVersion version) diff --git a/cassandra-analytics-common/src/main/java/org/apache/cassandra/bridge/CassandraVersion.java b/cassandra-analytics-common/src/main/java/org/apache/cassandra/bridge/CassandraVersion.java index 167039efd..bfc3c96ac 100644 --- a/cassandra-analytics-common/src/main/java/org/apache/cassandra/bridge/CassandraVersion.java +++ b/cassandra-analytics-common/src/main/java/org/apache/cassandra/bridge/CassandraVersion.java @@ -76,6 +76,9 @@ public String jarBaseName() { sstableFormat = System.getProperty("cassandra.analytics.bridges.sstable_format", "big"); + // NOTE: These default enum names must stay in sync with cassandraVersionEnumMap in build.gradle. + // FOURONE is intentionally excluded from local-dev defaults to keep iteration fast; + // CI covers 4.1 via explicit CASSANDRA_VERSION env var or per-version Gradle tasks (e.g. testCassandra41). String providedVersionsOrDefault = System.getProperty("cassandra.analytics.bridges.implemented_versions", String.join(",", FOURZERO.name(), FIVEZERO.name())); implementedVersions = Arrays.stream(providedVersionsOrDefault.split(",")) @@ -83,6 +86,7 @@ public String jarBaseName() .filter(v -> v.sstableFormats.contains(sstableFormat)) .toArray(CassandraVersion[]::new); + // NOTE: These default versions must stay in sync with cassandraFullVersionMap in build.gradle. String providedSupportedVersionsOrDefault = System.getProperty("cassandra.analytics.bridges.supported_versions", "cassandra-4.0.17,cassandra-5.0.5"); supportedVersions = Arrays.stream(providedSupportedVersionsOrDefault.split(",")) diff --git a/cassandra-analytics-core/build.gradle b/cassandra-analytics-core/build.gradle index ffce66600..8df6b1399 100644 --- a/cassandra-analytics-core/build.gradle +++ b/cassandra-analytics-core/build.gradle @@ -154,10 +154,11 @@ jar { } } -// Tests tagged 'Sequential' do not run in parallel, to prevent OOM errors -// when running in CI environment. -tasks.register('testSequential', Test) { - doFirst { +// Shared test configuration for core tests. When majorMinor is specified (e.g. "4.0"), +// tests run against that single Cassandra version. When null, uses env var filtering +// (CI) or falls back to all supported versions (local dev). +def configureCoreTestTask(Test task, String majorMinor = null) { + task.doFirst { if (JavaVersion.current().isJava11Compatible()) { def JDK11_OPTIONS = ['-Djdk.attach.allowAttachSelf=true', '--add-exports', 'java.base/jdk.internal.misc=ALL-UNNAMED', @@ -175,39 +176,41 @@ tasks.register('testSequential', Test) { '--add-opens', 'java.base/jdk.internal.module=ALL-UNNAMED', '--add-opens', 'java.base/jdk.internal.util.jar=ALL-UNNAMED', '--add-opens', 'jdk.management/com.sun.management.internal=ALL-UNNAMED'] - jvmArgs(JDK11_OPTIONS) + task.jvmArgs(JDK11_OPTIONS) println("JVM arguments for $project.name are $allJvmArgs") } } - systemProperty "cassandra.analytics.bridges.sstable_format", System.getProperty("cassandra.analytics.bridges.sstable_format", "big") - minHeapSize = '1024m' - maxHeapSize = System.getenv('CORE_TEST_MAX_HEAP_SIZE') ?: '3072m' - maxParallelForks = 1 - forkEvery = 1 // Enables different end-to-end test classes use Spark contexts with different configurations + task.systemProperty "cassandra.analytics.bridges.sstable_format", System.getProperty("cassandra.analytics.bridges.sstable_format", "big") + // Core tests use implemented_versions/supported_versions (set by applyCassandraVersionFilter) + // rather than versions_to_test (which drives TestVersionSupplier in CDC and integration tests). + rootProject.ext.applyCassandraVersionFilter(task, majorMinor) + task.minHeapSize = '1024m' + task.maxHeapSize = '2048m' + task.maxParallelForks = 3 + task.forkEvery = 1 // Enables different end-to-end test classes use Spark contexts with different configurations // Make it so unit tests run on a Jar with Cassandra bridge implementations built in - dependsOn(tasks.jar) - classpath += files(jar.archiveFile) + task.dependsOn(tasks.jar) + task.classpath += files(jar.archiveFile) - useJUnitPlatform { - includeTags 'Sequential' - } + def destDir = Paths.get(rootProject.rootDir.absolutePath, "build", "test-reports", "core").toFile() + task.reports.junitXml.outputLocation = destDir + task.reports.html.outputLocation = destDir - reports { - def destDir = Paths.get(rootProject.rootDir.absolutePath, "build", "test-reports", "core").toFile() - junitXml { - enabled true - destination = destDir - } - html { - enabled true - destination = destDir - } + task.testLogging { + events "started", "passed", "skipped", "failed" } +} - testLogging { - events "started", "passed", "skipped", "failed" +// Tests tagged 'Sequential' do not run in parallel, to prevent OOM errors +// when running in CI environment. +tasks.register('testSequential', Test) { + configureCoreTestTask(delegate) + maxParallelForks = 1 + + useJUnitPlatform { + includeTags 'Sequential' } shouldRunAfter test @@ -215,37 +218,45 @@ tasks.register('testSequential', Test) { } test { - systemProperty "cassandra.analytics.bridges.sstable_format", System.getProperty("cassandra.analytics.bridges.sstable_format", "big") - minHeapSize = '1024m' - maxHeapSize = System.getenv('CORE_TEST_MAX_HEAP_SIZE') ?: '3072m' + configureCoreTestTask(delegate) maxParallelForks = System.getenv('CORE_MAX_PARALLEL_FORKS')?.toInteger() ?: Math.max(Runtime.runtime.availableProcessors() * 2, 8) - forkEvery = 1 // Enables different end-to-end test classes use Spark contexts with different configurations - - // Make it so unit tests run on a Jar with Cassandra bridge implementations built in - dependsOn(tasks.jar) - classpath += files(jar.archiveFile) useJUnitPlatform { excludeTags 'Sequential' } - reports { - def destDir = Paths.get(rootProject.rootDir.absolutePath, "build", "test-reports", "core").toFile() - junitXml { - enabled true - destination = destDir - } - html { - enabled true - destination = destDir + + finalizedBy jacocoTestReport // report is always generated after tests run +} + +// Per-Cassandra-version test tasks for IDE convenience. +// These show up in IntelliJ's Gradle panel alongside 'test' and 'testSequential'. +rootProject.ext.cassandraVersionEnumMap.keySet().each { majorMinor -> + def suffix = majorMinor.replace(".", "") + + tasks.register("testCassandra${suffix}", Test) { + description = "Run core tests (non-Sequential) against Cassandra ${majorMinor}" + group = 'verification' + configureCoreTestTask(delegate, majorMinor) + maxParallelForks = System.getenv('CORE_MAX_PARALLEL_FORKS')?.toInteger() + ?: Math.max(Runtime.runtime.availableProcessors() * 2, 8) + useJUnitPlatform { + excludeTags 'Sequential' } + finalizedBy jacocoTestReport } - testLogging { - events "started", "passed", "skipped", "failed" + tasks.register("testSequentialCassandra${suffix}", Test) { + description = "Run core Sequential tests against Cassandra ${majorMinor}" + group = 'verification' + configureCoreTestTask(delegate, majorMinor) + maxParallelForks = 1 + useJUnitPlatform { + includeTags 'Sequential' + } + shouldRunAfter tasks.named("testCassandra${suffix}") + finalizedBy jacocoTestReport } - - finalizedBy jacocoTestReport // report is always generated after tests run } /* Start: JaCoCo check */ diff --git a/cassandra-analytics-core/src/test/java/org/apache/cassandra/spark/TestUtils.java b/cassandra-analytics-core/src/test/java/org/apache/cassandra/spark/TestUtils.java index 99f05f8ea..dabf69ff1 100644 --- a/cassandra-analytics-core/src/test/java/org/apache/cassandra/spark/TestUtils.java +++ b/cassandra-analytics-core/src/test/java/org/apache/cassandra/spark/TestUtils.java @@ -308,7 +308,7 @@ public static Gen tombstoneVersions() public static List tombstoneTestableVersions() { // Tombstone SSTable writing and SSTable-to-JSON conversion are not implemented for Cassandra version 3.0 - List tombstoneTestableVersions = ImmutableList.of(CassandraVersion.FOURZERO, CassandraVersion.FIVEZERO); + List tombstoneTestableVersions = ImmutableList.of(CassandraVersion.FOURZERO, CassandraVersion.FOURONE, CassandraVersion.FIVEZERO); return filterTestableVersions(tombstoneTestableVersions); } diff --git a/cassandra-analytics-core/src/test/java/org/apache/cassandra/spark/bulkwriter/SortedSSTableWriterTest.java b/cassandra-analytics-core/src/test/java/org/apache/cassandra/spark/bulkwriter/SortedSSTableWriterTest.java index 6328f1252..4bb3f7dcb 100644 --- a/cassandra-analytics-core/src/test/java/org/apache/cassandra/spark/bulkwriter/SortedSSTableWriterTest.java +++ b/cassandra-analytics-core/src/test/java/org/apache/cassandra/spark/bulkwriter/SortedSSTableWriterTest.java @@ -133,8 +133,14 @@ public void canCreateWriterForVersion(String version) throws IOException { dataFileStream.forEach(dataFilePath -> { dataFilePaths.add(dataFilePath); - assertThat(SSTables.cassandraVersionFromTable(dataFilePath).getMajorVersion()) - .isEqualTo(CassandraVersionFeatures.cassandraVersionFeaturesFromCassandraVersion(version).getMajorVersion()); + int sstableVersion = SSTables.cassandraVersionFromTable(dataFilePath).getMajorVersion(); + int expectedVersion = CassandraVersionFeatures.cassandraVersionFeaturesFromCassandraVersion(version).getMajorVersion(); + // 4.0 and 4.1 share the same SSTable format (nb), so the SSTable-level version is always 40 + if (expectedVersion == 41) + { + expectedVersion = 40; + } + assertThat(sstableVersion).isEqualTo(expectedVersion); }); } // no exception should be thrown from both the validate methods diff --git a/cassandra-analytics-integration-tests/build.gradle b/cassandra-analytics-integration-tests/build.gradle index d8d469f66..855a2192d 100644 --- a/cassandra-analytics-integration-tests/build.gradle +++ b/cassandra-analytics-integration-tests/build.gradle @@ -32,10 +32,11 @@ if (propertyWithDefault("artifactType", null) == "spark") apply from: "$rootDir/gradle/common/publishing.gradle" } -def integrationMaxHeapSize = System.getenv("INTEGRATION_MAX_HEAP_SIZE") ?: "3000M" +def integrationMaxHeapSize = System.getenv("INTEGRATION_MAX_HEAP_SIZE") ?: "2048M" println("Using ${integrationMaxHeapSize} maxHeapSize") -def integrationMaxParallelForks = (System.getenv("INTEGRATION_MAX_PARALLEL_FORKS") ?: "4") as int +// Large in circle and other env assumed to be 8G total heap available; we restrict to 3x2 at 6G and leave headroom for system +def integrationMaxParallelForks = (System.getenv("INTEGRATION_MAX_PARALLEL_FORKS") ?: "3") as int println("Using ${integrationMaxParallelForks} maxParallelForks") def integrationEnableMtls = (System.getenv("INTEGRATION_MTLS_ENABLED") ?: "true") as boolean @@ -82,26 +83,37 @@ dependencies { testRuntimeOnly 'com.fasterxml.jackson.core:jackson-annotations:2.14.2' } -test { - // Because system properties aren't passed from the command line through to tests, we need to specifically - // set them again here. - systemProperty "cassandra.test.dtest_jar_path", dependencyLocation - systemProperty "cassandra.sidecar.versions_to_test", (System.getenv("CASSANDRA_VERSION") ?: "5.0") - systemProperty "SKIP_STARTUP_VALIDATIONS", "true" - systemProperty "logback.configurationFile", "src/test/resources/logback-test.xml" - systemProperty "cassandra.integration.sidecar.test.enable_mtls", integrationEnableMtls - minHeapSize = '1g' - maxHeapSize = integrationMaxHeapSize - maxParallelForks = integrationMaxParallelForks - forkEvery = 1 // Enables different end-to-end test classes use Spark contexts with different configurations - - beforeTest { descriptor -> - // println("Setting test tags to ${descriptor.className}-${descriptor.name}") - systemProperty "cassandra.testtag", "${descriptor.className}" - systemProperty "suitename", "${descriptor.name}" +// Shared test configuration for integration tests. When majorMinor is specified (e.g. "4.0"), +// tests run against that single Cassandra version. When null, uses env var filtering +// (CI) or falls back to default version (local dev). +// Using a closure (not a method) so it captures the local variables above. +def configureIntegrationTestTask = { Test task, String majorMinor = null -> + task.systemProperty "cassandra.test.dtest_jar_path", dependencyLocation + def versionInfo = rootProject.ext.applyCassandraVersionFilter(task, majorMinor) + if (versionInfo) { + task.systemProperty "cassandra.sidecar.versions_to_test", versionInfo.fullVersion + } else { + // Major.minor format to match integration framework's TestVersionSupplier; defaults to latest version only. + // 4.0 and 4.1 intentionally excluded from gradlew defaults to keep local iteration fast; + // use testCassandra40/testCassandra41 for targeted runs. CI covers all versions via CASSANDRA_VERSION. + task.systemProperty "cassandra.sidecar.versions_to_test", "5.0" + } + task.systemProperty "SKIP_STARTUP_VALIDATIONS", "true" + task.systemProperty "logback.configurationFile", "src/test/resources/logback-test.xml" + task.systemProperty "cassandra.integration.sidecar.test.enable_mtls", integrationEnableMtls + task.minHeapSize = '1g' + task.maxHeapSize = integrationMaxHeapSize + task.maxParallelForks = integrationMaxParallelForks + task.forkEvery = 1 // Enables different end-to-end test classes use Spark contexts with different configurations + + // NOTE: Setting system properties in beforeTest only works because forkEvery = 1 ensures a fresh + // JVM per test class. If forkEvery is changed, these properties would stop varying per test. + task.beforeTest { descriptor -> + task.systemProperty "cassandra.testtag", "${descriptor.className}" + task.systemProperty "suitename", "${descriptor.name}" } - testLogging { + task.testLogging { events "passed", "skipped", "failed" showExceptions true @@ -129,30 +141,40 @@ test { '--add-opens', 'java.base/jdk.internal.module=ALL-UNNAMED', '--add-opens', 'java.base/jdk.internal.util.jar=ALL-UNNAMED', '--add-opens', 'jdk.management/com.sun.management.internal=ALL-UNNAMED'] - jvmArgs(JDK11_OPTIONS) - println("JVM arguments for $project.name are $allJvmArgs") + task.jvmArgs(JDK11_OPTIONS) + println("JVM arguments for $project.name are ${task.allJvmArgs}") + } + + // Some test classes are skipped entirely via assumeThat (e.g. MultipleTokens tests on C* 4.0). + // When CI runs individual classes with --tests, a skipped @BeforeAll leaves zero discovered tests, + // which Gradle treats as an error unless we tell it otherwise. + task.filter { + setFailOnNoMatchingTests(false) } // container test does not run on java 8 if ("true" == System.getenv("skipContainerTest") || JavaVersion.current().isJava8()) { - exclude("**/testcontainer/**") - - filter { - setFailOnNoMatchingTests(false) // do not fail the build if no matching test found, since in CI we run individual test class - } + task.exclude("**/testcontainer/**") } - useJUnitPlatform() + task.useJUnitPlatform() def destDir = Paths.get(rootProject.rootDir.absolutePath, "build", "test-reports", "integration").toFile() - reports { - junitXml { - enabled true - destination = destDir - } - html { - enabled true - destination = destDir - } + task.reports.junitXml.outputLocation = destDir + task.reports.html.outputLocation = destDir +} + +test { + configureIntegrationTestTask(delegate) +} + +// Per-Cassandra-version test tasks for IDE convenience. +// These show up in IntelliJ's Gradle panel alongside 'test'. +rootProject.ext.cassandraVersionEnumMap.keySet().each { majorMinor -> + def suffix = majorMinor.replace(".", "") + tasks.register("testCassandra${suffix}", Test) { + description = "Run integration tests against Cassandra ${majorMinor}" + group = 'verification' + configureIntegrationTestTask(delegate, majorMinor) } } diff --git a/cassandra-five-zero-bridge/src/main/java/org/apache/cassandra/bridge/CdcBridgeImplementation.java b/cassandra-five-zero-bridge/src/main/java/org/apache/cassandra/bridge/CdcBridgeImplementation.java index 6062f8cc2..cfcea225b 100644 --- a/cassandra-five-zero-bridge/src/main/java/org/apache/cassandra/bridge/CdcBridgeImplementation.java +++ b/cassandra-five-zero-bridge/src/main/java/org/apache/cassandra/bridge/CdcBridgeImplementation.java @@ -67,7 +67,7 @@ protected static synchronized void setCDC(Path path, int commitLogSegmentSize, b DatabaseDescriptor.setCommitLogSyncGroupWindow(30); DatabaseDescriptor.setCommitLogSegmentSize(commitLogSegmentSize); DatabaseDescriptor.getRawConfig().commitlog_total_space = new DataStorageSpec.IntMebibytesBound(1024); - DatabaseDescriptor.setCommitLogWriteDiskAccessMode(Config.DiskAccessMode.direct); + DatabaseDescriptor.setCommitLogWriteDiskAccessMode(Config.DiskAccessMode.mmap); DatabaseDescriptor.setCDCTotalSpaceInMiB(1024); DatabaseDescriptor.setCommitLogSegmentMgrProvider((commitLog -> new CommitLogSegmentManagerCDC(commitLog, commitLogPath.toString()))); setup = true;