Skip to content

Commit 2c5a888

Browse files
TerrenceMcGuinness-NOAATerrenceMcGuinness-NOAA
authored
Ctest case updates (#4118)
This adds a few new cases to the CTest framework for the NOAA Global Workflow. - Move verbose details of the framework to [read-the-docs](https://global-workflow--4118.org.readthedocs.build/en/4118/testing.html) - Simplified **[README.md](https://github.com/TerrenceMcGuinness-NOAA/global-workflow/tree/ctest_case_updates/dev/ctests#readme)** to how-to - Fully tested in development pipeline Resolves #3871
1 parent 0e3ebcd commit 2c5a888

12 files changed

+1369
-41
lines changed

‎dev/ci/gitlab-ci-hosts.yml‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -248,7 +248,13 @@ finalize_success-ursa:
248248
stage: run_tests
249249
parallel:
250250
matrix:
251-
- CTEST_NAME: ['C48_ATM_gfs_fcst_seg0', 'C48_S2SW_gfs_fcst_seg0']
251+
- CTEST_NAME:
252+
- C48_ATM-gfs_fcst_seg0
253+
- C48_ATM-gfs_atmos_prod_f000-f002
254+
- C48_S2SW-gfs_fcst_seg0
255+
- C48_S2SW-gfs_ocean_prod_f006
256+
- C48_S2SW-gfs_ice_prod_f006
257+
- C48_S2SWA_gefs-gefs_fcst_mem001_seg0
252258

253259
# Host-specific CTest setup jobs
254260
setup_ctests-hera:

‎dev/ctests/CMakeLists.txt‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -72,30 +72,30 @@ function(AddJJOBTest)
7272
"${novals}" "${singlevals}" "${multivals}"
7373
${ARGN})
7474

75-
set(TEST_NAME ${ARG_CASE}_${ARG_JOB})
75+
set(TEST_NAME ${ARG_CASE}-${ARG_JOB})
7676
set(CASE_PATH ${HOMEgfs}/dev/ci/cases/pr)
7777
set(CASE_YAML ${CASE_PATH}/${ARG_CASE}.yaml)
7878

7979
add_test(NAME test_${TEST_NAME}_setup
8080
COMMAND ./setup.sh ${TEST_NAME} ${CASE_YAML} ${ARG_TEST_DATE}
8181
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
82-
set_tests_properties(test_${TEST_NAME}_setup PROPERTIES LABELS "${ARG_CASE}_${ARG_JOB}")
82+
set_tests_properties(test_${TEST_NAME}_setup PROPERTIES LABELS "${ARG_CASE}-${ARG_JOB}")
8383

8484
add_test(NAME test_${TEST_NAME}_stage
8585
COMMAND ./stage.sh ${ARG_CASE} ${TEST_NAME} ${ARG_TEST_DATE}
8686
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
87-
set_tests_properties(test_${TEST_NAME}_stage PROPERTIES DEPENDS test_${TEST_NAME}_setup LABELS "${ARG_CASE}_${ARG_JOB}")
87+
set_tests_properties(test_${TEST_NAME}_stage PROPERTIES DEPENDS test_${TEST_NAME}_setup LABELS "${ARG_CASE}-${ARG_JOB}")
8888

8989
add_test(NAME test_${TEST_NAME}_execute
9090
COMMAND ./execute.sh ${TEST_NAME} ${ARG_JOB} ${ARG_TEST_DATE}
9191
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
92-
set_tests_properties(test_${TEST_NAME}_execute PROPERTIES DEPENDS test_${TEST_NAME}_stage LABELS "${ARG_CASE}_${ARG_JOB}")
92+
set_tests_properties(test_${TEST_NAME}_execute PROPERTIES DEPENDS test_${TEST_NAME}_stage LABELS "${ARG_CASE}-${ARG_JOB}")
9393

9494
# TODO - This is a stub for the validation step
9595
add_test(NAME test_${TEST_NAME}_validate
9696
COMMAND ./validate.sh ${ARG_CASE} ${TEST_NAME} ${ARG_TEST_DATE}
9797
WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
98-
set_tests_properties(test_${TEST_NAME}_validate PROPERTIES DEPENDS test_${TEST_NAME}_execute LABELS "${ARG_CASE}_${ARG_JOB}")
98+
set_tests_properties(test_${TEST_NAME}_validate PROPERTIES DEPENDS test_${TEST_NAME}_execute LABELS "${ARG_CASE}-${ARG_JOB}")
9999
endfunction()
100100

101101
AddJJOBTest(
@@ -104,8 +104,32 @@ AddJJOBTest(
104104
TEST_DATE "2021032312"
105105
)
106106

107+
AddJJOBTest(
108+
CASE "C48_ATM"
109+
JOB "gfs_atmos_prod_f000-f002"
110+
TEST_DATE "2021032312"
111+
)
112+
107113
AddJJOBTest(
108114
CASE "C48_S2SW"
109115
JOB "gfs_fcst_seg0"
110116
TEST_DATE "2021032312"
111117
)
118+
119+
AddJJOBTest(
120+
CASE "C48_S2SW"
121+
JOB "gfs_ice_prod_f006"
122+
TEST_DATE "2021032312"
123+
)
124+
125+
AddJJOBTest(
126+
CASE "C48_S2SW"
127+
JOB "gfs_ocean_prod_f006"
128+
TEST_DATE "2021032312"
129+
)
130+
131+
AddJJOBTest(
132+
CASE "C48_S2SWA_gefs"
133+
JOB "gefs_fcst_mem001_seg0"
134+
TEST_DATE "2021032312"
135+
)

‎dev/ctests/README.md‎

Lines changed: 235 additions & 35 deletions
Original file line numberDiff line numberDiff line change
@@ -1,63 +1,263 @@
1-
# CTest Framework for NOAA Global Workflow
1+
# CTest Framework Quick Start
22

3-
This directory contains the CTest framework for testing Rocoto JJOBS. The framework allows you to stage, execute, and validate individual JJOBS independently from other jobs in the workflow. Each test requires its own YAML definition of inputs and configurations.
3+
This directory contains the CTest framework for testing Rocoto workflow jobs (JJOBS) independently. Each test runs in an isolated environment with staged input files from nightly baseline runs.
44

5-
## Overview
5+
> **📖 Complete Documentation**: See the [comprehensive testing documentation](../../docs/source/testing.rst) for detailed information on framework architecture, YAML configuration, test patterns, CI/CD integration, and troubleshooting.
66
7-
The CTest framework consists of the following scripts:
8-
- **setup.sh.in**: Prepares the environment and creates the experiment.
9-
- **stage.sh.in**: Stages the input files needed to run a JJOB.
10-
- **execute.sh.in**: Executes the JJOB and monitors its status.
11-
- **validate.sh.in**: Validates the results of the JJOB.
7+
## Quick Start Guide
128

13-
**NOTE:** So far only test C48_ATM *gfs_fcst_set0* has `output_files` for the validation step using a basic chksum for testing. Further development using grib and NETCDF comparison tools is pending.
9+
### Prerequisites
1410

15-
## Usage
11+
The following environment variables must be set (either in your environment or via platform configuration):
1612

17-
### CMake Configuration
13+
```bash
14+
HPC_ACCOUNT # Your HPC allocation account
15+
STAGED_CTESTS # Path to nightly baseline COMROOT
16+
ICSDIR_ROOT # Path to initial condition files
17+
```
1818

19-
To configure the **CTest** framework using **CMake**, you need to provide several environment variables. Here is an example of how to configure and build the project:
19+
These are typically defined in `$HOMEgfs/dev/ci/platforms/config.$MACHINE_ID`.
2020

21-
**NOTE**: The the specific values for these three enviroment variables can be found in `$HOMEgfs/dev/ci/platforms/config.$MACHINE_ID` and may also be added to the `cmake` command line with the `-D` option
21+
### Configure and Build
2222

23-
# Run CMake to configure the ctest framework
24-
```shell
23+
```bash
2524
cd $HOMEgfs/dev/ctests
26-
mkdir build
25+
mkdir -p build
2726
cd build
28-
cmake ../..
29-
```
3027

31-
### Running Tests with CTest
28+
# Configure using environment variables from config.$MACHINE_ID
29+
cmake ../../..
3230

33-
Once the project is configured, you can run the tests using CTest. Here are some examples:
31+
# Or configure with explicit command-line options
32+
cmake -DHPC_ACCOUNT=myaccount \
33+
-DSTAGED_CTESTS=/path/to/baselines/RUNTESTS \
34+
-DICSDIR_ROOT=/path/to/ics \
35+
../../..
36+
```
3437

35-
#### Run All Tests
38+
### Run Tests
3639

3740
```bash
38-
cd /path/to/build
41+
# Run all tests
3942
ctest
43+
44+
# Run tests for a specific configuration case
45+
ctest -L C48_ATM
46+
47+
# Run test for a specific JJOB
48+
cest -L C48_ATM-gfs_atmos_prod_f000-f002
49+
50+
# Run a specific test with verbose output
51+
ctest -R test_C48_ATM-gfs_fcst_seg0_execute -V
52+
53+
# Run tests in parallel (4 concurrent tests)
54+
ctest -j 4
55+
56+
# Show test list without running
57+
ctest -N
4058
```
4159

42-
#### Run Tests for a Specific Case
60+
### Common CTest Options
4361

44-
You can use the `-L` option with CTest to run tests for a specific case. For example, to run tests for the `C48_ATM` case:
62+
| Option | Description |
63+
|--------|-------------|
64+
| `-V` | Verbose output |
65+
| `-VV` | Extra verbose output |
66+
| `-N` | Dry run (list tests without executing) |
67+
| `-L <label>` | Run tests matching label (e.g., `-L C48_ATM`) |
68+
| `-R <regex>` | Run tests matching regex pattern |
69+
| `-j <N>` | Run N tests in parallel |
70+
| `--output-on-failure` | Show output only for failed tests |
71+
| `--rerun-failed` | Rerun only previously failed tests |
72+
73+
## Test Structure
74+
75+
Each test consists of four phases executed sequentially:
76+
77+
1. **Setup**: Creates experiment directory and configuration
78+
2. **Stage**: Stages input files from baseline into test COMROOT
79+
3. **Execute**: Runs the job script and monitors execution
80+
4. **Validate**: Compares outputs against baseline results
81+
82+
Test phases are automatically chained via CMake dependencies.
83+
84+
## Validation Modes
85+
86+
Control validation behavior with the `CTEST_VALIDATION_MODE` environment variable:
4587

4688
```bash
47-
cd /path/to/build
48-
ctest -L C48_ATM
49-
```
50-
Or simply use the '-R' switch to run any individual test:
51-
```
52-
ctest -R test_C48_S2SW_gfs_fcst_seg0_execute -V
89+
# PRESENCE_ONLY (default): Verify files exist, no checksum validation
90+
export CTEST_VALIDATION_MODE=PRESENCE_ONLY
91+
ctest -R validate
92+
93+
# STRICT: All files must exist AND checksums must match
94+
export CTEST_VALIDATION_MODE=STRICT
95+
ctest -R validate
96+
97+
# CHECKSUM_ONLY: Validate checksums for existing files, ignore missing files
98+
export CTEST_VALIDATION_MODE=CHECKSUM_ONLY
99+
ctest -R validate
53100
```
54101

55-
To add a new test use the **AddJJOBTest()** function at the end of the `$HOMEgfs/dev/ctest/CMakeLists.txt` file as follows:
102+
## Available Tests
103+
104+
Current test cases include:
105+
106+
| Test Name | Configuration | Component | Description |
107+
|-----------|--------------|-----------|-------------|
108+
| `C48_ATM-gfs_fcst_seg0` | C48_ATM | Atmosphere | Atmosphere-only forecast |
109+
| `C48_ATM-gfs_atmos_prod_f000-f002` | C48_ATM | Products | Atmosphere product generation |
110+
| `C48_S2SW-gfs_fcst_seg0` | C48_S2SW | Coupled | Coupled forecast (atmos-ocean-ice-wave) |
111+
| `C48_S2SW-gfs_ocean_prod_f006` | C48_S2SW | Products | Ocean product generation |
112+
| `C48_S2SW-gfs_ice_prod_f006` | C48_S2SW | Products | Ice product generation |
113+
| `C48_S2SWA_gefs-gefs_fcst_mem001_seg0` | C48_S2SWA_gefs | Ensemble | GEFS ensemble member forecast |
114+
115+
## Adding New Tests
116+
117+
### 1. Add test definition to `CMakeLists.txt`:
118+
56119
```cmake
57120
AddJJOBTest(
58-
CASE "C48_ATM"
59-
JOB "gfs_fcst_seg0"
60-
TEST_DATE "2021032312"
121+
CASE "C48_ATM" # Configuration case
122+
JOB "gfs_analysis" # Job identifier
123+
TEST_DATE "2021032312" # Test cycle (YYYYMMDDHH)
61124
)
62125
```
63-
Then create a new YAML file with the required staged input files as is done with this example found in `$HOMEgfs/dev/ctests/cases/C48_ATM_gfs_fcts_seg0.yaml`
126+
127+
### 2. Create YAML file `cases/C48_ATM-gfs_analysis.yaml`:
128+
129+
```yaml
130+
{% set cyc = TEST_DATE | strftime('%H') %}
131+
{% set PDY = TEST_DATE | to_YMD %}
132+
{% set SRC_DIR = STAGED_CTESTS + '/COMROOT/' + PSLOT %}
133+
{% set DST_DIR = RUNTESTS + '/COMROOT/' + TEST_NAME %}
134+
135+
input_files:
136+
mkdir:
137+
- {{ DST_DIR }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input
138+
139+
copy:
140+
- [{{ SRC_DIR }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/gfs_ctrl.nc,
141+
{{ DST_DIR }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/gfs_ctrl.nc]
142+
# Add additional required files...
143+
144+
output_files:
145+
cmpfiles:
146+
- [{{ SRC_DIR }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/analysis/atminc.nc,
147+
{{ DST_DIR }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/analysis/atminc.nc]
148+
# Add additional output files...
149+
```
150+
151+
### 3. Build and test:
152+
153+
```bash
154+
cd build
155+
cmake ../../..
156+
ctest -L C48_S2SW -j 3
157+
```
158+
159+
## Naming Convention
160+
161+
Test names follow the pattern: `CASE-JOB.yaml`
162+
163+
**Examples:**
164+
- `C48_ATM-gfs_fcst_seg0.yaml` - Atmosphere forecast
165+
- `C48_S2SW-gfs_ocean_prod_f006.yaml` - Ocean products
166+
- `C48_S2SWA_gefs-gefs_fcst_mem001_seg0.yaml` - Ensemble member forecast
167+
168+
## Troubleshooting
169+
170+
### Missing Input Files
171+
172+
```bash
173+
# Compare baseline with test environment
174+
ls ${STAGED_CTESTS}/COMROOT/${PSLOT}/gfs.${PDY}/${cyc}/
175+
ls ${RUNTESTS}/COMROOT/${TEST_NAME}/gfs.${PDY}/${cyc}/
176+
```
177+
178+
Add missing files to the YAML `input_files.copy` section.
179+
180+
### Checksum Validation Failures
181+
182+
For development, use presence-only validation:
183+
```bash
184+
export CTEST_VALIDATION_MODE=PRESENCE_ONLY
185+
ctest -R validate
186+
```
187+
188+
### Verbose Debugging
189+
190+
```bash
191+
# Enable debug logging
192+
export LOGGING_LEVEL=DEBUG
193+
194+
# Run with maximum verbosity
195+
ctest -R test_name -VV
196+
197+
# Check test logs
198+
tail -f ${RUNTESTS}/COMROOT/${TEST_NAME}_*/EXPDIR/logs/*.log
199+
```
200+
201+
### Manual Execution
202+
203+
Run test phases manually for debugging:
204+
205+
```bash
206+
cd build/scripts
207+
./setup.sh TEST_NAME CASE_YAML TEST_DATE
208+
./stage.sh CASE_NAME TEST_NAME TEST_DATE
209+
./execute.sh TEST_NAME JOB_NAME TEST_DATE
210+
./validate.sh CASE_NAME TEST_NAME TEST_DATE
211+
```
212+
213+
## Key Directories
214+
215+
```
216+
$HOMEgfs/dev/ctests/ # Framework root
217+
├── build/ # CMake build directory (create this)
218+
├── cases/ # YAML test definitions
219+
├── scripts/ # Test phase scripts
220+
└── CMakeLists.txt # Test configuration
221+
222+
$HOMEgfs/dev/ci/platforms/ # Platform-specific configuration
223+
└── config.$MACHINE_ID # Machine settings (STAGED_CTESTS, HPC_ACCOUNT, etc.)
224+
225+
${STAGED_CTESTS}/COMROOT/ # Nightly baseline outputs (input source)
226+
${RUNTESTS}/COMROOT/ # Test execution environments (created by tests)
227+
```
228+
229+
## Platform Configuration
230+
231+
Platform-specific settings are in `$HOMEgfs/dev/ci/platforms/config.$MACHINE_ID`:
232+
233+
```bash
234+
# Example from config.hera
235+
export GFS_CI_ROOT=/scratch1/NCEPDEV/global/Terry.McGuinness/GFS_CI_ROOT
236+
export GITLAB_BUILDS_DIR=${GFS_CI_ROOT}/BUILDS/GITLAB
237+
export STAGED_CTESTS=${GITLAB_BUILDS_DIR}/stable/RUNTESTS
238+
export ICSDIR_ROOT=/scratch1/NCEPDEV/global/glopara/data/ICSDIR
239+
export HPC_ACCOUNT=nems
240+
```
241+
242+
Source the appropriate configuration before running CMake:
243+
244+
```bash
245+
source $HOMEgfs/ush/detect_machine.sh
246+
source $HOMEgfs/dev/ci/platforms/config.$MACHINE_ID
247+
```
248+
249+
## Additional Resources
250+
251+
- **Complete Documentation**: `docs/source/testing.rst`
252+
- **Test Case Examples**: `cases/*.yaml`
253+
- **CI/CD Pipeline**: `../ci/gitlab-ci-hosts.yml`
254+
- **Job Scripts**: `../../jobs/JGLOBAL_*`
255+
- **Platform Configuration**: `../ci/platforms/config.*`
256+
257+
---
258+
259+
**Framework Version**: 1.0
260+
**Last Updated**: October 2025
261+
**Status**: Active Development
262+
263+
For detailed architecture, YAML templating, CI/CD integration, and comprehensive troubleshooting, see the [full testing documentation](../../docs/source/testing.rst).

0 commit comments

Comments
 (0)