Merge release/2.38.0 to master

Author: shahramn

.github/ci-hpc-config.yml

@@ -7,3 +7,4 @@ build:
   parallel: 64
   cmake_options:
     - -DENABLE_EXTRA_TESTS=1
+    - -DENABLE_ECCODES_OMP_THREADS=1

.github/ci-nightly-test.sh

@@ -1,6 +1,6 @@
 #!/bin/bash
 
-set -e
+set -eu
 
 # We do not want to come across the ecCodes tools in the toolbox
 module unload ecmwf-toolbox
@@ -14,12 +14,17 @@ module load python3
 
 version=$(cat $TMPDIR/eccodes/VERSION)
 
-# regression_suite_dir=$TMPDIR/eccodes-regression-tests
-# mkdir -p $regression_suite_dir
-# git clone [email protected]:ecmwf/eccodes-regression-tests.git $regression_suite_dir
-# cd $regression_suite_dir
+# Note:
+# The environment variable "GH_TOKEN" needs to be there for the clone to succeed
+#
 
-cd ~masn/REGRESSION_TESTING/ecCodes
+regression_suite_dir=$TMPDIR/eccodes-regression-tests
+mkdir -p $regression_suite_dir
+git clone https://${GH_TOKEN}@github.com/ecmwf/eccodes-regression-tests.git $regression_suite_dir
+cd $regression_suite_dir
+
+# Launch the regression tests in parallel (This script uses GNU parallel)
+# cd ~masn/REGRESSION_TESTING/ecCodes
 ./par-suite.sh -w $TMPDIR/install/eccodes/$version
 
 # For debugging specific test(s)

.github/nightly-ci-hpc-config.yml

@@ -0,0 +1,18 @@
+build:
+  modules:
+    - ecbuild
+    - ninja
+    - aec
+    - netcdf4
+  parallel: 64
+  cmake_options:
+    - -DENABLE_EXTRA_TESTS=1
+    - -DENABLE_PNG=1
+    - -DENABLE_NETCDF=1
+  force_build: true
+  post_script: .github/ci-nightly-test.sh
+  env:
+    - GH_TOKEN=${GH_TOKEN}
+  compiler_cc: gcc
+  compiler_cxx: g++
+  compiler_fc: gfortran

.github/workflows/nightly.yml

@@ -5,15 +5,21 @@ on:
 
   # Run at 20:00 UTC every day (on default branch)
   schedule:
-    - cron: '0 20 * * *'
+    - cron: "0 20 * * *"
 
 jobs:
   test-hpc:
-    uses: ./.github/workflows/reusable-ci-hpc.yml
-    with:
-      eccodes: ecmwf/eccodes@${{ github.event.pull_request.head.sha || github.sha }}
-      nightly_test: true
-    secrets: inherit
+    runs-on: [self-hosted, linux, hpc]
+    env:
+      GH_TOKEN: ${{ secrets.GH_REPO_READ_TOKEN }}
+    steps:
+      - uses: ecmwf-actions/reusable-workflows/ci-hpc@v2
+        with:
+          github_user: ${{ secrets.BUILD_PACKAGE_HPC_GITHUB_USER }}
+          github_token: ${{ secrets.GH_REPO_READ_TOKEN }}
+          troika_user: ${{ secrets.HPC_CI_SSH_USER }}
+          repository: ecmwf/eccodes@${{ github.event.pull_request.head.sha || github.sha }}
+          build_config: .github/nightly-ci-hpc-config.yml
 
   test-macos:
     strategy:
@@ -63,6 +69,8 @@ jobs:
     secrets:
       url_debian_11: ${{ secrets.NEXUS_TEST_REPO_NIGHTLY_URL_DEBIAN_11 }}
       token_debian_11: ${{ secrets.NEXUS_TEST_REPO_UPLOAD_TOKEN }}
+      url_debian_12: ${{ secrets.NEXUS_TEST_REPO_URL_DEBIAN_12 }}
+      token_debian_12: ${{ secrets.NEXUS_TEST_REPO_UPLOAD_TOKEN }}
       url_centos_7: ${{ secrets.NEXUS_TEST_REPO_NIGHTLY_URL_CENTOS_7 }}
       token_centos_7: ${{ secrets.NEXUS_TEST_REPO_UPLOAD_TOKEN }}
       url_rocky_8: ${{ secrets.NEXUS_TEST_REPO_NIGHTLY_URL_ROCKY_8 }}

.github/workflows/notify_new_pr.yml

@@ -18,7 +18,16 @@
 
 cmake_minimum_required( VERSION 3.12 FATAL_ERROR )
 
-find_package( ecbuild 3.7 REQUIRED HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild)
+find_package( ecbuild 3.7 HINTS ${CMAKE_CURRENT_SOURCE_DIR} ${CMAKE_CURRENT_SOURCE_DIR}/../ecbuild)
+if(NOT ecbuild_FOUND)
+    message(STATUS "Fetching ecbuild...")
+    include(FetchContent)
+    FetchContent_Populate(ecbuild
+        GIT_REPOSITORY https://github.com/ecmwf/ecbuild.git
+        GIT_TAG        3.8.5
+    )
+    find_package( ecbuild 3.7 REQUIRED HINTS ${ecbuild_SOURCE_DIR})
+endif()
 
 # Initialise project
 project( eccodes LANGUAGES CXX )
@@ -50,9 +59,9 @@ check_symbol_exists( fdatasync       "unistd.h"   ECCODES_HAVE_FDATASYNC)
 
 check_c_source_compiles(
       " typedef int foo_t;
-      static inline foo_t static_foo(){return 0;}
-      foo_t foo(){return 0;}
-      int main(int argc, char *argv[]){return 0;}
+      static inline foo_t static_foo(void){return 0;}
+      foo_t foo(void){return 0;}
+      int main(int argc, char *argv[]){ return static_foo(); }
       " ECCODES_HAVE_C_INLINE
 )
 
@@ -99,6 +108,10 @@ ecbuild_add_option( FEATURE BUILD_TOOLS
     DESCRIPTION "Build the command line tools"
     DEFAULT ON )
 
+ecbuild_add_option( FEATURE GEOGRAPHY
+    DESCRIPTION "Support for Geoiterator and nearest neighbour"
+    DEFAULT ON )
+
 ecbuild_add_option( FEATURE JPG
     DESCRIPTION "Support for JPG decoding/encoding"
     DEFAULT ON )
@@ -375,6 +388,16 @@ if( HAVE_FORTRAN AND CMAKE_Fortran_COMPILER_ID MATCHES "GNU" AND NOT CMAKE_Fortr
     ecbuild_add_fortran_flags("-fallow-argument-mismatch")
 endif()
 
+if(GIT_FOUND AND NOT ${GIT_EXECUTABLE} STREQUAL "")
+    ecbuild_info("Found git: ${GIT_EXECUTABLE} (found version \"${GIT_VERSION_STRING}\")")
+    execute_process(COMMAND ${GIT_EXECUTABLE} rev-parse --abbrev-ref HEAD
+                    OUTPUT_VARIABLE eccodes_GIT_BRANCH
+                    RESULT_VARIABLE nok ERROR_VARIABLE error
+                    OUTPUT_STRIP_TRAILING_WHITESPACE
+                    WORKING_DIRECTORY "${PROJECT_SOURCE_DIR}" )
+    ecbuild_info("ecCodes branch = ${eccodes_GIT_BRANCH}" )
+endif()
+
 ###############################################################################
 # contents
 
@@ -457,7 +480,7 @@ ecbuild_pkgconfig(
   DESCRIPTION    "The ecCodes library"
   LIBRARIES      eccodes
   IGNORE_INCLUDE_DIRS ${PYTHON_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIRS} ${NETCDF_INCLUDE_DIRS}
-  VARIABLES      HAVE_MEMFS HAVE_JPEG HAVE_LIBJASPER HAVE_LIBOPENJPEG
+  VARIABLES      HAVE_MEMFS HAVE_GEOGRAPHY HAVE_JPEG HAVE_LIBJASPER HAVE_LIBOPENJPEG
                  HAVE_ECCODES_THREADS HAVE_ECCODES_OMP_THREADS
                  HAVE_NETCDF HAVE_FORTRAN HAVE_PNG HAVE_AEC
 )
@@ -469,7 +492,7 @@ if( HAVE_FORTRAN )
     DESCRIPTION         "The ecCodes library for Fortran 90"
     IGNORE_INCLUDE_DIRS ${CMAKE_CURRENT_SOURCE_DIR}/fortran ${PROJECT_BINARY_DIR}/fortran
                         ${PYTHON_INCLUDE_DIRS} ${NUMPY_INCLUDE_DIRS} ${NETCDF_INCLUDE_DIRS}
-    VARIABLES           HAVE_MEMFS HAVE_JPEG HAVE_LIBJASPER HAVE_LIBOPENJPEG
+    VARIABLES           HAVE_MEMFS HAVE_GEOGRAPHY HAVE_JPEG HAVE_LIBJASPER HAVE_LIBOPENJPEG
                         HAVE_ECCODES_THREADS HAVE_ECCODES_OMP_THREADS
                         HAVE_NETCDF HAVE_PNG HAVE_AEC
   )
@@ -494,11 +517,8 @@ ecbuild_info("   |  ecCodes version ${eccodes_VERSION}  |")
 ecbuild_info("   +--------------------------+")
 ecbuild_info("")
 
-ecbuild_info("   +--------------------------------------+")
-ecbuild_info("   | Please note:                         |")
-ecbuild_info("   | For Python3 support, first install   |")
-ecbuild_info("   | ecCodes and then install the Python  |")
-ecbuild_info("   | bindings from PyPI with:             |")
-ecbuild_info("   |   $ pip3 install eccodes             |")
-ecbuild_info("   +--------------------------------------+")
+ecbuild_info("Please note:")
+ecbuild_info(" For Python3 support, you must install the Python bindings.")
+ecbuild_info(" See:")
+ecbuild_info("  https://confluence.ecmwf.int/display/ECC/ecCodes+installation")
 ecbuild_info("")

.github/workflows/reusable-ci-hpc.yml

@@ -1 +1 @@
-2.37.0
+2.38.0

.github/workflows/reusable-ci.yml

@@ -91,4 +91,4 @@ mercator.grib2
 run_length_packing.grib2
 boustrophedonic.grib1
 reduced_gaussian_sub_area.legacy.grib1
-
+grid_complex_spatial_differencing.grib2

CMakeLists.txt

@@ -1,4 +1,7 @@
+Will set values...
 values=2 2 2 2
 max=2 min=2 average=2
+max=2 min=2 average=2
+Will set values...
 values=2 5 2 2
 max=5 min=2 average=2.75

VERSION

@@ -12,8 +12,8 @@ transient missingValue=999999 : hidden;
 transient setToMissingIfOutOfRange=0 : hidden;
 
 # This gets updated twice a year by WMO.
-# See https://community.wmo.int/activity-areas/wmo-codes/manual-codes/latest-version
-constant masterTablesVersionNumberLatest = 41;
+# See https://community.wmo.int/en/wis/latest-version
+constant masterTablesVersionNumberLatest = 42;
 
 #negative value=not used
 transient inputDelayedDescriptorReplicationFactor={-1} : hidden;

data/grib_data_files.txt

@@ -1,4 +1,5 @@
 # RDB stands for REPORT DATA BASE
+# See https://confluence.ecmwf.int/pages/viewpage.action?pageId=24316441
 unsigned[1]   rdbType : dump; 
 unsigned[1]   oldSubtype: dump;
 

data/statistics.out.good

@@ -50,7 +50,7 @@ unsigned[1]     reservedSection3 = 0;
 unsigned[2]     numberOfSubsets : dump;
 alias ls.numberOfSubsets=numberOfSubsets;
 
-if (section2Present && bufrHeaderCentre==98 && section2Length==52) {
+if (section2Present && bufrHeaderCentre == 98 && section2Length == 52) {
     if ( rdbType == 2 || rdbType == 3 || rdbType == 8 || rdbType == 12 || rdbType == 30 ) {
         transient isSatelliteType=1;
     } else {
@@ -69,10 +69,11 @@ if (section2Present && bufrHeaderCentre==98 && section2Length==52) {
         meta localLatitude2 bits(keyMore,32,25,-9000000,100000) : dump,no_copy;
 
         # This rule is taken from BUFRDC. See bufrdc_wmo/buukey.F
-        # Also see ECC-686
-        if (oldSubtype == 255 || numberOfSubsets>255 ||
-            ( oldSubtype>=121 && oldSubtype <=130 )  ||
-            oldSubtype==31) {
+        # See ECC-686
+        # and https://confluence.ecmwf.int/pages/viewpage.action?pageId=24316441
+        if (oldSubtype == 255 || numberOfSubsets > 255 ||
+            ( oldSubtype >= 121 && oldSubtype <= 130 ) ||
+            oldSubtype == 31) {
             meta ls.localNumberOfObservations bits(keySat,0,16) : dump,long_type,no_copy;
             meta ls.satelliteID bits(keySat,16,16) : dump,long_type,no_copy;
         } else {
@@ -114,16 +115,16 @@ meta bufrdcExpandedDescriptors bufrdc_expanded_descriptors(expandedCodes) : no_c
 #smart_table NAME (VALUES,FILE_NAME,MASTER_DIRECTORY,LOCAL_DIRECTORY,WIDTH_OF_CODE_IN_BITS,EXTRA_DIRECTORY,EXTRA_FILE_NAME);
 #Note: The WIDTH_OF_CODE_IN_BITS has to be big enough so 2^width > the highest BUFR descriptor code
 smart_table expandedOriginalCodes (expandedCodes,"element.table",tablesMasterDir,tablesLocalDir,18,rootTablesDir,"operators.table") :no_copy;
-meta expandedAbbreviations         smart_table_column(expandedOriginalCodes,0,1) : string_type,no_copy;
-meta expandedTypes                 smart_table_column(expandedOriginalCodes,1,0) : string_type,no_copy;
-meta expandedNames                 smart_table_column(expandedOriginalCodes,2,0) : string_type,no_copy;
-meta expandedUnits                 smart_table_column(expandedOriginalCodes,3,0) : string_type,no_copy;
-meta expandedOriginalScales        smart_table_column(expandedOriginalCodes,4,0) : long_type,no_copy;
-meta expandedOriginalReferences    smart_table_column(expandedOriginalCodes,5,0) : long_type,no_copy;
-meta expandedOriginalWidths        smart_table_column(expandedOriginalCodes,6,0) : long_type,no_copy;
-meta expandedCrex_units            smart_table_column(expandedOriginalCodes,7,0) : long_type,no_copy;
-meta expandedCrex_scales           smart_table_column(expandedOriginalCodes,8,0) : long_type,no_copy;
-meta expandedCrex_widths           smart_table_column(expandedOriginalCodes,9,0) : long_type,no_copy;
+meta expandedAbbreviations        smart_table_column(expandedOriginalCodes,0,1) : string_type,no_copy;
+meta expandedTypes                smart_table_column(expandedOriginalCodes,1,0) : string_type,no_copy;
+meta expandedNames                smart_table_column(expandedOriginalCodes,2,0) : string_type,no_copy;
+meta expandedUnits                smart_table_column(expandedOriginalCodes,3,0) : string_type,no_copy;
+meta expandedOriginalScales       smart_table_column(expandedOriginalCodes,4,0) : long_type,no_copy;
+meta expandedOriginalReferences   smart_table_column(expandedOriginalCodes,5,0) : long_type,no_copy;
+meta expandedOriginalWidths       smart_table_column(expandedOriginalCodes,6,0) : long_type,no_copy;
+meta expandedCrex_units           smart_table_column(expandedOriginalCodes,7,0) : long_type,no_copy;
+meta expandedCrex_scales          smart_table_column(expandedOriginalCodes,8,0) : long_type,no_copy;
+meta expandedCrex_widths          smart_table_column(expandedOriginalCodes,9,0) : long_type,no_copy;
 
 position endDescriptors;
 section_padding section3Padding;

definitions/bufr/boot.def

@@ -0,0 +1,8 @@
+0 0 ANTARCTICA
+1 1 REGION I
+2 2 REGION II
+3 3 REGION III
+4 4 REGION IV
+5 5 REGION V
+6 6 REGION VI
+7 7 MISSING VALUE

definitions/bufr/rdb_key.def

@@ -0,0 +1,10 @@
+0 0 INCREASING, THEN DECREASING; ATMOSPHERIC PRESSURE THE SAME OR HIGHER THAN THREE HOURS AGO
+1 1 INCREASING, THEN STEADY; OR INCREASING, THEN INCREASING MORE SLOWLY
+2 2 INCREASING (STEADILY OR UNSTEADILY)
+3 3 DECREASING OR STEADY, THEN INCREASING; OR INCREASING, THEN INCREASING MORE RAPIDLY
+4 4 STEADY; ATMOSPHERIC PRESSURE THE SAME AS THREE HOURS AGO
+5 5 DECREASING, THEN INCREASING; ATMOSPHERIC PRESSURE THE SAME OR LOWER THAN THREE HOURS AGO
+6 6 DECREASING, THEN STEADY; OR DECREASING, THEN DECREASING MORE SLOWLY
+7 7 DECREASING (STEADILY OR UNSTEADILY)
+8 8 STEADY OR INCREASING, THEN DECREASING; OR DECREASING, THEN DECREASING MORE RAPIDLY
+15 15 MISSING VALUE

definitions/bufr/section.3.def

@@ -0,0 +1,4 @@
+0 0 SUBSONIC
+1 1 TRANSONIC
+2 2 SUPERSONIC
+7 7 MISSING VALUE