Skip to content

CASSANALYTICS144: Split testing pipelines out #198

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
jmckenzie-dev wants to merge 14 commits into
base: trunk
Choose a base branch
from
Open

CASSANALYTICS144: Split testing pipelines out #198

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
c8826e8
CASSANALYTICS-144: Split out CI pipelines by Cassandra Version
jmckenzie-dev Apr 9, 2026
6153763
Move commitlog from direct to standard
jmckenzie-dev Apr 15, 2026
655ebf3
Have cdc tasks inherit base JVM params; move to DiskAccessMode mmap
jmckenzie-dev Apr 15, 2026
0ee4c10
slim down to 3 'diagonal' matrixed pipelines for precommit
jmckenzie-dev Apr 16, 2026
c7a2bf8
Move to 3 full combos across all GA; randomize test order based on SH…
jmckenzie-dev Apr 16, 2026
cfbb2b2
Simplify gh job config
jmckenzie-dev Apr 16, 2026
c3f2502
Comment on 4.1 exclusion; default 2G/3 fork
jmckenzie-dev Apr 16, 2026
69e2570
make CDC resource alloc in CI pluggable as well
jmckenzie-dev Apr 16, 2026
55cb1a1
tidy up cdc build.gradle a touch to make heap decl adjacent to parall…
jmckenzie-dev Apr 16, 2026
ad8df9e
Add bridge jar mapping when shared across versions
jmckenzie-dev Apr 16, 2026
7360125
Bump integration to 3G heap
jmckenzie-dev Apr 17, 2026
7a49fde
Allow integration tests to gracefully skip when version assumptions fail
jmckenzie-dev Apr 17, 2026
5590039
Always report artifacts in circle even in failure case
jmckenzie-dev Apr 17, 2026
83b894c
Make 4.0 and 4.1 both use same nb format
jmckenzie-dev Apr 17, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
97 changes: 66 additions & 31 deletions .circleci/config.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -55,18 +55,22 @@ commands:
sstable_format:
type: string
default: "big"
cassandra:
type: string
default: ""
description: Build and test against Spark <<parameters.spark>> Scala <<parameters.scala>> <<parameters.jdk>>
steps:
- run:
environment:
SPARK_VERSION: "<<parameters.spark>>"
SCALA_VERSION: "<<parameters.scala>>"
JDK_VERSION: "<<parameters.jdk>>"
INTEGRATION_MAX_PARALLEL_FORKS: 1
INTEGRATION_MAX_HEAP_SIZE: "1500M"
CORE_MAX_PARALLEL_FORKS: 2
INTEGRATION_MAX_PARALLEL_FORKS: 2
INTEGRATION_MAX_HEAP_SIZE: "3072m"
CORE_MAX_PARALLEL_FORKS: 3
CORE_TEST_MAX_HEAP_SIZE: "2048m"
CASSANDRA_USE_JDK11: <<parameters.use_jdk11>>
CASSANDRA_VERSION: "<<parameters.cassandra>>"
command: |
export GRADLE_OPTS="-Xmx2g -Dorg.gradle.jvmargs=-Xmx2g"
# Run compile/unit tests, skipping integration tests
Expand All @@ -92,7 +96,7 @@ commands:
SPARK_VERSION: "<<parameters.spark>>"
SCALA_VERSION: "<<parameters.scala>>"
JDK_VERSION: "<<parameters.jdk>>"
INTEGRATION_MAX_PARALLEL_FORKS: 1
INTEGRATION_MAX_PARALLEL_FORKS: 2
INTEGRATION_MAX_HEAP_SIZE: "2500M"
CASSANDRA_USE_JDK11: <<parameters.use_jdk11>>
command: |
Expand Down Expand Up @@ -143,7 +147,7 @@ jobs:
- "*.jar"
- "org/**/*"

spark3-2_12-jdk11-big:
spark3-2_12-jdk11-big-c40:
docker:
- image: cimg/openjdk:11.0
resource_class: large
Expand All @@ -158,132 +162,156 @@ jobs:
jdk: "11"
use_jdk11: "true"
sstable_format: "big"
cassandra: "4.0"

- store_artifacts:
when: always
path: build/test-reports
destination: test-reports

- store_artifacts:
when: always
path: build/reports
destination: reports

- store_test_results:
when: always
path: build/test-reports

int-c4-spark3-2_12-jdk11:
parallelism: 8
spark3-2_12-jdk11-big-c41:
docker:
- image: cimg/openjdk:11.0
resource_class: large
steps:
- setup_remote_docker
- install_common
- checkout
- attach_workspace:
at: dependencies
- run_integration:
- run_build:
spark: "3"
scala: "2.12"
jdk: "11"
use_jdk11: "true"
cassandra: "4.1.4"
sstable_format: "big"
cassandra: "4.1"

- store_artifacts:
when: always
path: build/test-reports
destination: test-reports

- store_artifacts:
when: always
path: build/reports
destination: reports

- store_artifacts:
path: cassandra-analytics-integration-tests
destination: int-tests-misc

- store_test_results:
when: always
path: build/test-reports

spark3-2_13-jdk11-big:
int-c4-spark3-2_12-jdk11:
parallelism: 8
docker:
- image: cimg/openjdk:11.0
resource_class: large
steps:
- setup_remote_docker
- install_common
- checkout
- attach_workspace:
at: dependencies
- run_build:
- run_integration:
spark: "3"
scala: "2.13"
scala: "2.12"
jdk: "11"
use_jdk11: "true"
sstable_format: "big"
cassandra: "4.0.17"

- store_artifacts:
when: always
path: build/test-reports
destination: test-reports

- store_artifacts:
when: always
path: build/reports
destination: reports

- store_artifacts:
when: always
path: cassandra-analytics-integration-tests
destination: int-tests-misc

- store_test_results:
when: always
path: build/test-reports

spark3-2_13-jdk11-bti:
int-c41-spark3-2_12-jdk11:
parallelism: 8
docker:
- image: cimg/openjdk:11.0
resource_class: large
steps:
- setup_remote_docker
- install_common
- checkout
- attach_workspace:
at: dependencies
- run_build:
- run_integration:
spark: "3"
scala: "2.13"
scala: "2.12"
jdk: "11"
use_jdk11: "true"
sstable_format: "bti"
cassandra: "4.1.4"

- store_artifacts:
when: always
path: build/test-reports
destination: test-reports

- store_artifacts:
when: always
path: build/reports
destination: reports

- store_artifacts:
when: always
path: cassandra-analytics-integration-tests
destination: int-tests-misc

- store_test_results:
when: always
path: build/test-reports

int-c4-spark3-2_13-jdk11:
parallelism: 8
spark3-2_13-jdk11-bti-c50:
docker:
- image: cimg/openjdk:11.0
resource_class: large
steps:
- setup_remote_docker
- install_common
- checkout
- attach_workspace:
at: dependencies
- run_integration:
- run_build:
spark: "3"
scala: "2.13"
jdk: "11"
use_jdk11: "true"
cassandra: "4.1.4"
sstable_format: "bti"
cassandra: "5.0"

- store_artifacts:
when: always
path: build/test-reports
destination: test-reports

- store_artifacts:
when: always
path: build/reports
destination: reports

- store_test_results:
when: always
path: build/test-reports

int-c5-spark3-2_13-jdk11:
Expand All @@ -305,34 +333,41 @@ jobs:
cassandra: "5.0.5"

- store_artifacts:
when: always
path: build/test-reports
destination: test-reports

- store_artifacts:
when: always
path: build/reports
destination: reports

- store_test_results:
when: always
path: build/test-reports

workflows:
version: 2
build-and-test:
jobs:
- build-deps-jdk11
- spark3-2_12-jdk11-big:

# Unit tests: split by Cassandra version to reduce per-process memory pressure
- spark3-2_12-jdk11-big-c40:
requires:
- build-deps-jdk11
- spark3-2_13-jdk11-big:
- spark3-2_12-jdk11-big-c41:
requires:
- build-deps-jdk11
- spark3-2_13-jdk11-bti:
- spark3-2_13-jdk11-bti-c50:
requires:
- build-deps-jdk11

# Integration tests
- int-c4-spark3-2_12-jdk11:
requires:
- build-deps-jdk11
- int-c4-spark3-2_13-jdk11:
- int-c41-spark3-2_12-jdk11:
requires:
- build-deps-jdk11
- int-c5-spark3-2_13-jdk11:
Expand Down
59 changes: 37 additions & 22 deletions .github/workflows/test.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,16 +73,21 @@ jobs:
key: build-${{ github.sha }}

unit-test:
name: Unit test
name: Unit test - Scala ${{ matrix.scala }} ${{ matrix.sstable-format }} C${{ matrix.cassandra }}
needs: build
runs-on: ubuntu-latest
strategy:
matrix:
scala: [ '2.12', '2.13' ]
sstable-format: [ 'big', 'bti' ]
exclude:
- scala: "2.12"
sstable-format: "bti"
include:
- scala: '2.13'
sstable-format: 'bti'
cassandra: '5.0'
- scala: '2.12'
sstable-format: 'big'
cassandra: '4.1'
- scala: '2.12'
sstable-format: 'big'
cassandra: '4.0'
fail-fast: false
steps:
- name: Setup JDK
Expand Down Expand Up @@ -112,29 +117,35 @@ jobs:
export SPARK_VERSION="3"
export SCALA_VERSION="${{ matrix.scala }}"
export JDK_VERSION="11"
export INTEGRATION_MAX_PARALLEL_FORKS=1
export INTEGRATION_MAX_HEAP_SIZE="1500M"
export INTEGRATION_MAX_PARALLEL_FORKS=3
export INTEGRATION_MAX_HEAP_SIZE="2048M"
export CASSANDRA_USE_JDK11=true

export CASSANDRA_VERSION="${{ matrix.cassandra }}"

./gradlew --stacktrace clean assemble check -x cassandra-analytics-integration-tests:test -Dcassandra.analytics.bridges.sstable_format=${{ matrix.sstable-format }}

integration-test:
name: Integration test
name: Integration test - ${{ matrix.config }} (${{ matrix.job_index }})
needs: build
runs-on: ubuntu-latest
strategy:
# GHA generates a cross-product of 'config' × 'job_index' (3 × 5 = 15 jobs).
# The 'include' entries don't add new combinations — they augment existing ones
# by matching on 'config' and injecting 'scala' and 'cassandra' into each match.
# To add a new version: add one entry to 'config' and one to 'include'.
matrix:
scala: [ '2.12', '2.13' ]
cassandra: [ '4.0.17', '4.1.4', '5.0.5' ]
job_index: [ 0, 1, 2, 3, 4 ]
job_total: [ 5 ]
exclude:
- scala: "2.12"
cassandra: "5.0.5"
- scala: "2.12"
cassandra: "4.1.4"
- scala: "2.13"
cassandra: "4.0.17"
config: ['s2.13-c5.0.5', 's2.12-c4.1.4', 's2.12-c4.0.17']
job_index: [0, 1, 2, 3, 4]
include:
- config: 's2.13-c5.0.5'
scala: '2.13'
cassandra: '5.0.5'
- config: 's2.12-c4.1.4'
scala: '2.12'
cassandra: '4.1.4'
- config: 's2.12-c4.0.17'
scala: '2.12'
cassandra: '4.0.17'
fail-fast: false
steps:
- name: Setup JDK
Expand Down Expand Up @@ -174,7 +185,11 @@ jobs:
./gradlew --stacktrace clean assemble

cd cassandra-analytics-integration-tests/src/test/java
CLASSNAMES=$(find . -name '*Test.java' | sort | cut -c 3- | sed 's@/@.@g' | sed 's/.\{5\}$//' | awk 'NR % ${{ matrix.job_total }} == ${{ matrix.job_index }}')
# Shuffle test classes using the commit SHA as seed for reproducible randomization,
# then shard across runners via round-robin on the shuffled order.
CLASSNAMES=$(find . -name '*Test.java' | cut -c 3- | sed 's@/@.@g' | sed 's/.\{5\}$//' \
| python3 -c "import random,sys; lines=sys.stdin.read().splitlines(); random.seed('$GITHUB_SHA'); random.shuffle(lines); print('\n'.join(lines))" \
| awk 'NR % 5 == ${{ matrix.job_index }}')
cd ../../../..

EXIT_STATUS=0
Expand Down
Loading
Loading