GRIB source feature for NWP emulator

.github/ci-config.yml

@@ -3,6 +3,7 @@ dependencies: |
   ecmwf/eckit
   ecmwf/fckit
   ecmwf/atlas
+  ecmwf/eccodes
 dependency_cmake_options: |
   ecmwf/atlas: "-DENABLE_FORTRAN=ON"
 dependency_branch: develop

.github/ci-hpc-config.yml

@@ -1,11 +1,41 @@
-build:
-  modules:
-    - ninja
-  dependencies:
-    - ecmwf/ecbuild@develop
-    - ecmwf/eckit@develop
-    - ecmwf/fckit@develop
-    - ecmwf/atlas@develop
-  dependency_cmake_options:
-    - ecmwf/atlas:-DENABLE_FORTRAN=ON
-  parallel: 64
+matrix:
+  - mpi_on
+  - mpi_off
+
+mpi_on:
+  build:
+    modules:
+      - ninja
+      - openmpi
+    modules_package:
+      - atlas:openmpi
+      - eckit:openmpi
+    dependencies:
+      - ecmwf/ecbuild@develop
+      - ecmwf/eckit@develop
+      - ecmwf/fckit@develop
+      - ecmwf/atlas@develop
+      - ecmwf/eccodes@develop
+    dependency_cmake_options:
+      - ecmwf/atlas:-DENABLE_FORTRAN=ON
+    parallel: 64
+    ntasks: 16
+    env:
+      - CTEST_PARALLEL_LEVEL=1
+      - OMPI_MCA_rmaps_base_oversubscribe=1
+      - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples
+      - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions
+
+mpi_off:
+  build:
+    modules:
+      - ninja
+    dependencies:
+      - ecmwf/ecbuild@develop
+      - ecmwf/eckit@develop
+      - ecmwf/fckit@develop
+      - ecmwf/atlas@develop
+      - ecmwf/eccodes@develop
+    dependency_cmake_options:
+      - ecmwf/atlas:-DENABLE_FORTRAN=ON
+    parallel: 64

CMakeLists.txt

@@ -20,6 +20,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
 ### dependencies and options
 ecbuild_find_package( NAME eckit  VERSION  1.18 REQUIRED )
 ecbuild_find_package( NAME atlas )
+ecbuild_find_package( NAME eccodes )
 
 ecbuild_add_option( FEATURE BUILD_TOOLS
                     DEFAULT ON
@@ -47,6 +48,15 @@ ecbuild_info("FCKIT_FOUND ${fckit_FOUND}")
 ecbuild_info("FCKIT_LIBRARIES ${FCKIT_LIBRARIES}")
 ecbuild_info("FCKIT_INCLUDE_DIRS ${FCKIT_INCLUDE_DIRS}")
 
+############## PRECISION
+ecbuild_add_option( FEATURE NWP_EMULATOR_SINGLE_PRECISION
+                    DESCRIPTION "Single precision Atlas fields"
+                    DEFAULT OFF )
+
+if( HAVE_NWP_EMULATOR_SINGLE_PRECISION ) 
+  list(APPEND NWP_EMULATOR_DEFINITIONS WITH_NWP_EMULATOR_SINGLE_PRECISION )
+endif()
+
 add_subdirectory( src )
 add_subdirectory( tests )
 add_subdirectory( examples )

src/CMakeLists.txt

@@ -8,3 +8,5 @@
 # does it submit to any jurisdiction.
 
 add_subdirectory(plume)
+
+add_subdirectory(nwp_emulator)

src/nwp_emulator/CMakeLists.txt

@@ -0,0 +1,43 @@
+# (C) Copyright 2025- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation nor
+# does it submit to any jurisdiction.
+ecbuild_add_library(
+    TARGET plume_nwp_emulator
+    INSTALL_HEADERS LISTED
+    HEADER_DESTINATION
+        ${INSTALL_INCLUDE_DIR}/nwp_emulator
+    SOURCES
+        data_reader.h
+        grib_file_reader.h
+        config_reader.h
+        nwp_definitions.h
+        nwp_data_provider.h
+        nwp_data_provider.cc
+        grib_file_reader.cc
+        config_reader.cc
+        config_reader_funcs.cc
+    PUBLIC_INCLUDES
+       $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/src>
+       $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/src>
+    PRIVATE_INCLUDES
+        "${MPI_INCLUDE_DIRS}"
+    DEFINITIONS
+        ${NWP_EMULATOR_DEFINITIONS}
+    PUBLIC_LIBS
+        plume_plugin
+        plume_plugin_manager
+        eccodes
+        eckit
+)
+
+ecbuild_add_executable( TARGET nwp_emulator_run.x
+  SOURCES
+    nwp_emulator.cc
+  LIBS
+    plume_nwp_emulator
+)

src/nwp_emulator/README.md

@@ -0,0 +1,106 @@
+# Plume Numerical Weather Prediction Model Emulator
+
+This tool is here to help you with your Plume plugins development.
+You can use it to emulate model runs with Plume loading the plugins from your configuration.
+
+## Features
+
+- Dry run: The NWP emulator can run without Plume to develop new features, or to test your data generation to ensure it fits the purpose of your plugin test.
+- Plume run: The NWP emulator runs Plume with the passed configuration. It can be used to test the Plume framweork, or test Plume plugins.
+- GRIB as input: The NWP emulator can either take a directory containing GRIB files or a GRIB file as input source. It will attempt to serve the data in each file as if it were a single model step. The number of emulator steps run depend on the number of GRIB files in the source folder, or a user-defined maximum number of steps. This approach allows a developer to test a plugin with real data. *Note:* the emulator is not looking recursively into the source folder, any subdirectory will be ignored.
+- Config as input: The NWP emulator can take a configuration file that sets its run parameters (number of steps, grid, number of levels, fields...). The data is synthetically generated by a set of available functions, see [below](#configuration-source) for more details.
+
+## Usage
+
+Once installed (check out [Plume install guide](../../README.md/#installation) for more info), the emulator tool can be used as follows:
+```bash
+mpirun -np 2 nwp_emulator_run [--grib-src=<path> | --config-src=<path>] [--plume-cfg=<path>] [OPTION]... [--help]
+```
+To make a dry run, omit the `--plume-cfg` flag.
+
+## Quick start
+### GRIB source
+
+The GRIB files are served alphabetically so your source folder should be organised to ensure the data you expect is served in the order you expect. Validation is run on all the source files to make sure they all contain the same fields and levels on the same grid. Check out the [MARS documentation](https://confluence.ecmwf.int/display/UDOC/MARS+user+documentation) if you need a hand to retrieve historical data. 
+
+### Configuration source
+
+Currently, the emulator supports five functions for data generation:
+- Vortex Rollup (provided by the [Atlas library](https://confluence.ecmwf.int/display/ATLAS))
+- Random (uniform or normal distributions)
+- Step
+- Cardinal Sine
+- Gaussian
+
+All functions accept a focus area which can be translated between each time step by a user-defined factor.
+Validation is run on the configuration file to ensure options are valid, but it lies with the user to make sure the selected options achieve their purpose.
+
+Here is an example of YAML configuration file you can use to generate model data, which demonstrate the options that each generation function accept:
+
+```yaml
+emulator:
+  n_steps: 5
+  grid_identifier: "N80"
+  vertical_levels: 5
+  # can provide area here if all population methods should use a non global area by default
+  # range for lon is assumed [-180,180] and lat [-90,90]
+  fields:
+    100u: # use the field short name
+      levtype: "sfc"
+      apply:
+        vortex_rollup:
+          area: [71.5, -25, 34.5, 45]
+          time_variation: 1.1
+    100v:
+      levtype: "sfc"
+      apply:
+        vortex_rollup:
+          time_variation: 1.1
+    u: # no specified levtype means not surface
+      apply: # functions applied in the given order
+        levels:
+          "1":
+            random: # no specified area means global
+              distribution: "uniform"
+            step:
+              area: [71.5, -25, 34.5, 45] # rectangle represented by NW and SE (lat,lon) coordinates
+              value: 10.0
+              variation: 1.0
+              translation: [1.0, 1.0] # degrees of translation of the area per time step (lat, lon)
+          "2":
+            sinc:
+              modes: 3 # number of peaks or sinks in the generation area
+              min: -1.0
+              max: 10.0
+              spread: 10.0
+              sink: false
+          "3:":
+            gaussian:
+              modes: 2
+              min: 0.0
+              max: 1.0
+              max_stddev: 3.0
+              sink: true
+    v: "u"
+    t:
+      apply:
+        levels: # this key should not be used for sfc
+          "2": # when using levels, they should range from 1-<vertical_levels>
+            random:
+              area: [71.5, -25, 34.5, 45]
+              distribution: "uniform"
+              min: 10.0
+              max: 35.0
+          "1,3":
+            step:
+              probability: 0.1
+              value: 45.0
+          "4:":
+            random:
+              distribution: "normal"
+              mean: 20.0
+              stddev: 20.0
+```
+
+## License
+© 2025 ECMWF. All rights reserved.