CTests extended validation for C48_ATM and staged C48_S2SW for gfs_fcst and gfs_atmos

ci/platforms/config.orion

@@ -2,7 +2,7 @@
 
 export GFS_CI_ROOT=/work2/noaa/stmp/GFS_CI_ROOT/ORION
 export ICSDIR_ROOT=/work/noaa/global/glopara/data/ICSDIR
-export STAGED_TESTS_DIR=${GFS_CI_ROOT}/STAGED_TESTS_DIR
+export STAGED_TESTS_DIR=/work/noaa/stmp/GFS_CI_ROOT/ORION/STAGED_TESTS_DIR
 export HPC_ACCOUNT=nems
 export max_concurrent_cases=5
 export max_concurrent_pr=4

ctests/CMakeLists.txt

@@ -94,7 +94,7 @@ function(AddJJOBTest)
 
   # TODO - This is a stub for the validation step
   add_test(NAME test_${TEST_NAME}_validate
-      COMMAND ./validate.sh ${TEST_NAME} ${CASE_YAML}
+      COMMAND ./validate.sh ${TEST_NAME} ${ARG_TEST_DATE}
       WORKING_DIRECTORY ${CMAKE_CURRENT_BINARY_DIR}/scripts)
   set_tests_properties(test_${TEST_NAME}_validate PROPERTIES DEPENDS test_${TEST_NAME}_execute LABELS "${ARG_CASE};${ARG_JOB}")
 endfunction()
@@ -104,3 +104,15 @@ AddJJOBTest(
   JOB  "gfs_fcst_seg0"
   TEST_DATE "2021032312"
 )
+
+AddJJOBTest(
+  CASE "C48_S2SW"
+  JOB "gfs_fcst_seg0"
+  TEST_DATE "2021032312"
+)
+
+AddJJOBTest(
+  CASE "C48_S2SW"
+  JOB "gfs_atmos_prod_f000-f003"
+  TEST_DATE "2021032312"
+)

ctests/README.md

@@ -8,25 +8,33 @@ The CTest framework consists of the following scripts:
 - **setup.sh.in**: Prepares the environment and creates the experiment.
 - **stage.sh.in**: Stages the input files needed to run a JJOB.
 - **execute.sh.in**: Executes the JJOB and monitors its status.
-- **validate.sh.in**: (TODO) Validates the results of the JJOB.
+- **validate.sh.in**: Validates the results of the JJOB.
+
+**NOTE:** So far only test C48_ATM *gfs_fcst_set0* has `output_files` for the validation step using a basic chksum for testing. Further development using grib and NETCDF comparison tools is pending. 
 
 ## Usage
 
 ### CMake Configuration
 
-To configure the CTest framework using CMake, you need to provide several environment variables or default values. Here is an example of how to configure and build the project:
+To configure the **CTest** framework using **CMake**, you need to provide several environment variables. Here is an example of how to configure and build the project:
 
 ```bash
 # Set environment variables (may also be include at command line with -D)
 export HPC_ACCOUNT="your_hpc_account"
 export ICSDIR_ROOT="/path/to/icsdir_root"
 export STAGED_TESTS_DIR="/path/to/staged_tests_dir"
+```
+**NOTE**: The the specific values for these three enviroment variables can be found in `$HOMEgfs/ci/platforms/config.$MACHINE_ID` and may also be added to the `cmake` command line with the `-D` option
 
 # Run CMake to configure the ctest framework
-cmake -S /path/to/HOMEgfs -B /path/to/build -DRUNTESTS=/path/to/runtests
-
+```shell
+cd $HOMEgfs/ctests
+mkdir build
+cd build
+cmake ../.. 
 ```
 
+
 ### Running Tests with CTest
 
 Once the project is configured, you can run the tests using CTest. Here are some examples:
@@ -46,6 +54,10 @@ You can use the `-L` option with CTest to run tests for a specific case. For exa
 cd /path/to/build
 ctest -L C48_ATM
 ```
+Or simply use the '-R' switch to run any individual test:
+```
+ctest -R test_C48_S2SW_gfs_fcst_seg0_execute -V
+```
 
 To add a new test use the **AddJJOBTest()** function at the end of the `$HOMEgfs/ctest/CMakeLists.txt` file as follows:
 ```cmake

ctests/cases/C48_S2SW_gfs_fcst_seg0.yaml

@@ -0,0 +1,50 @@
+{% set H_offset = "-6H" %}
+{% set H_timedelta = H_offset | to_timedelta %}
+{% set TEST_DATE_offset = TEST_DATE | add_to_datetime(H_timedelta) %}
+
+{% set cyc = TEST_DATE | strftime('%H') %}
+{% set cyc_offset = TEST_DATE_offset  | strftime('%H') %}
+
+{% set PDY = TEST_DATE | to_YMD %}
+{% set PDY_offset = TEST_DATE_offset |  to_YMD %}
+
+input_files:
+    mkdir:
+        - "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input"
+        - "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/model/wave/prep"
+        - "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gdas.{{ PDY }}/{{cyc_offset}}/model/ocean/restart"
+        - "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gdas.{{ PDY }}/{{cyc_offset}}/model/ice/restart"
+    copy:
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/gfs_ctrl.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/gfs_ctrl.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/gfs_data.tile1.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/gfs_data.tile1.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/gfs_data.tile2.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/gfs_data.tile2.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/gfs_data.tile3.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/gfs_data.tile3.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/gfs_data.tile4.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/gfs_data.tile4.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/gfs_data.tile5.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/gfs_data.tile5.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/gfs_data.tile6.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/gfs_data.tile6.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/sfc_data.tile1.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/sfc_data.tile1.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/sfc_data.tile2.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/sfc_data.tile2.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/sfc_data.tile3.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/sfc_data.tile3.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/sfc_data.tile4.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/sfc_data.tile4.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/sfc_data.tile5.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/sfc_data.tile5.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/sfc_data.tile5.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/sfc_data.tile5.nc"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/sfc_data.tile6.nc", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/model/atmos/input/sfc_data.tile6.nc"]
+
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ PDY }}/{{ cyc }}/model/wave/prep/gfswave.mod_def.glo_200", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/model/wave/prep/gfswave.mod_def.glo_200"]
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gfs.{{ PDY }}/{{ cyc }}/model/wave/prep/gfswave.mod_def.uglo_100km", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/model/wave/prep/gfswave.mod_def.uglo_100km"]
+
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gdas.{{ PDY_offset }}/{{ cyc_offset }}/model/ocean/restart/{{ TEST_DATE | strftime('%Y%m%d.%H0000') }}.MOM.res.nc",
+           "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gdas.{{ PDY_offset }}/{{ cyc_offset }}/model/ocean/restart/{{ TEST_DATE | strftime('%Y%m%d.%H0000') }}.MOM.res.nc" ]
+
+        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/input_files/gdas.{{ PDY_offset }}/{{ cyc_offset }}/model/ice/restart/{{ TEST_DATE | strftime('%Y%m%d.%H0000') }}.cice_model.res.nc",
+           "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gdas.{{ PDY_offset }}/{{ cyc_offset }}/model/ice/restart/{{ TEST_DATE | strftime('%Y%m%d.%H0000') }}.cice_model.res.nc" ]
+
+# TODO - To enable the validation step include specific files to compare against the results from running the test.
+# Note: The below three files is only an example. The cmpfiles tag means it will use checksum to compare the files in the two locations.
+# Other extensions can be added and/or folded into the FileUtils class in wxflow FileUtilities.py
+#
+#output_files:
+#    cmpfiles:
+#        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/output_files/gfs.{{ PDY }}/{{ cyc }}/conf/ufs.diag_table", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/conf/ufs.diag_table"]
+#        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/output_files/gfs.{{ PDY }}/{{ cyc }}/conf/ufs.input.nml", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/conf/ufs.input.nml"]
+#        - ["{{ 'STAGED_TESTS_DIR' | getenv }}/{{ 'TEST_NAME' | getenv }}/output_files/gfs.{{ PDY }}/{{ cyc }}/conf/ufs.model_configure", "{{ 'RUNTESTS' | getenv }}/COMROOT/{{ 'TEST_NAME' | getenv }}/gfs.{{ PDY }}/{{ cyc }}/conf/ufs.model_configure"]

ctests/scripts/execute.sh.in

@@ -8,7 +8,7 @@ JOB=${2:?"Job name is required"}
 idate=$3
 
 #TODO - add rocotoboot_dryrun to repo some how
-rocotoboot_dryrun=/work2/noaa/global/mterry/rocoto_dryrun/bin/rocotoboot
+rocotoboot_dryrun=$(command -v rocotoboot_dryrun) || true
 CASEDIR="@CMAKE_CURRENT_BINARY_DIR@/RUNTESTS/EXPDIR/${TEST_NAME}"
 cd "${CASEDIR}"
 rm -f ./*.db

ctests/scripts/stage.py

@@ -24,7 +24,7 @@
 
 from argparse import ArgumentParser
 from pathlib import Path
-from wxflow import parse_j2yaml, FileHandler, Logger
+from wxflow import parse_j2yaml, FileHandler, Logger, logit, to_datetime
 
 # Initialize logger with environment variable for logging level
 logger = Logger(level=os.environ.get("LOGGING_LEVEL", "DEBUG"), colored_log=False)
@@ -52,16 +52,19 @@ def parse_args():
     return parser.parse_args()
 
 
-if __name__ == '__main__':
-
-    # Parse command line arguments
+@logit(logger)
+def main():
     args = parse_args()
 
     data = {}
     if args.test_date:
         # Parse test date from string to datetime object
-        data['TEST_DATE'] = datetime.datetime.strptime(args.test_date, '%Y%m%d%H')
+        data['TEST_DATE'] = to_datetime(args.test_date)
     # Parse YAML configuration file with optional data
     case_cfg = parse_j2yaml(path=args.yaml, data=data)
     # Synchronize input files as per the parsed configuration
     FileHandler(case_cfg.input_files).sync()
+
+
+if __name__ == '__main__':
+    main()

ctests/scripts/stage.sh.in

@@ -19,12 +19,12 @@ set -x
 PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/sorc/wxflow/src"
 export PYTHONPATH
 
-INPUTS_YAML="${HOMEgfs}/ctests/cases/${TEST_NAME}.yaml"
+YAML_FILE="${HOMEgfs}/ctests/cases/${TEST_NAME}.yaml"
 
 TEST_NAME="${TEST_NAME}" \
 RUNTESTS="${RUNTESTS}" \
 STAGED_TESTS_DIR="${STAGED_TESTS_DIR}" \
-"${HOMEgfs}/ctests/scripts/stage.py" --yaml "${INPUTS_YAML}" --test_date "${TEST_DATE}"
+"${HOMEgfs}/ctests/scripts/stage.py" --yaml "${YAML_FILE}" --test_date "${TEST_DATE}"
 rc=$?
 if [[ "${rc}" -ne 0 ]]; then
     set +x

ctests/scripts/validate.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+"""
+validate.py
+
+Validation script that checks file checksums and verifies test outputs.
+
+Usage
+-----
+validate.py --yaml <path_to_yaml> --test_date <YYYYMMDDHH>
+
+Parameters
+----------
+--yaml : str
+    Path to the YAML configuration file.
+--test_date : str
+    Test date in the format YYYYMMDDHH.
+
+"""
+
+import sys
+import argparse
+from pathlib import Path
+import hashlib
+from wxflow import parse_j2yaml, Logger, logit, to_datetime
+
+logger = Logger(level="DEBUG", colored_log=True)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--yaml", required=True)
+    parser.add_argument("--test_date", required=True)
+    return parser.parse_args()
+
+
+def file_checksum(path):
+    hasher = hashlib.md5()
+    with open(path, "rb") as f:
+        for chunk in iter(lambda: f.read(4096), b""):
+            hasher.update(chunk)
+    return hasher.hexdigest()
+
+
+def validate_cmpfiles(config):
+    cmpfiles = config.get("output_files", {}).get("cmpfiles", [])
+    for pair in cmpfiles:
+        file_a, file_b = pair
+        if file_checksum(file_a) != file_checksum(file_b):
+            logger.error(f"Checksum mismatch: {file_a} vs {file_b}")
+            raise ValueError(f"Checksum mismatch: {file_a} vs {file_b}")
+        logger.info(f"checksums match: {file_a} vs {file_b}")
+
+
+@logit(logger)
+def main():
+    args = parse_args()
+
+    data = {}
+    if args.test_date:
+        # Parse test date from string to datetime object
+        data['TEST_DATE'] = to_datetime(args.test_date)
+
+    files = parse_j2yaml(path=args.yaml, data=data)
+    if 'output_files' not in files:
+        logger.info(f"No output files found for test: {args.yaml}")
+        logger.info("Nothing to validate (TODO - Stubbed).")
+        sys.exit(0)
+
+    validate_cmpfiles(files)
+    logger.info(f"All files exist and pass checksum for test: {args.yaml}")
+
+
+if __name__ == "__main__":
+    main()

ctests/scripts/validate.sh.in

@@ -2,8 +2,35 @@
 set -ux
 
 TEST_NAME=${1:?"Name of the test is required"}
-YAML_FILE=${2:?"Name of the CI yaml file for validating the test"}
+TEST_DATE=${2:?"idate of the test is required"}
 
+# CMake to fill these variables
+STAGED_TESTS_DIR="@STAGED_TESTS_DIR@"
+RUNTESTS="@RUNTESTS@"
+HOMEgfs="@PROJECT_SOURCE_DIR@"
+
+# Load the runtime environment for this script (needs wxflow and its dependencies)
+set +x
+source "${HOMEgfs}/workflow/gw_setup.sh"
+rc=$?
+[[ "${rc}" -ne 0 ]] && exit "${status}"
+set -x
+PYTHONPATH="${PYTHONPATH:+${PYTHONPATH}:}${HOMEgfs}/sorc/wxflow/src"
+export PYTHONPATH
+
+YAML_FILE="${HOMEgfs}/ctests/cases/${TEST_NAME}.yaml"
 echo "validating '${TEST_NAME}' with yaml file '${YAML_FILE}'"
+
+TEST_NAME="${TEST_NAME}" \
+RUNTESTS="${RUNTESTS}" \
+STAGED_TESTS_DIR="${STAGED_TESTS_DIR}" \
+"${HOMEgfs}/ctests/scripts/validate.py" --yaml "${YAML_FILE}"  --test_date "${TEST_DATE}"
+rc=$?
+if [[ "${rc}" -ne 0 ]]; then
+    set +x
+    echo "Failed to validate for '${TEST_NAME}' with '${YAML_FILE}'"
+    set -x
+    exit "${rc}"
+fi
 
-exit 0
+exit 0