Added config reader data source

.github/ci-hpc-config.yml

@@ -1,12 +1,41 @@
-build:
-  modules:
-    - ninja
-  dependencies:
-    - ecmwf/ecbuild@develop
-    - ecmwf/eckit@develop
-    - ecmwf/fckit@develop
-    - ecmwf/atlas@develop
-    - ecmwf/eccodes@develop
-  dependency_cmake_options:
-    - ecmwf/atlas:-DENABLE_FORTRAN=ON
-  parallel: 64
+matrix:
+  - mpi_on
+  - mpi_off
+
+mpi_on:
+  build:
+    modules:
+      - ninja
+      - openmpi
+    modules_package:
+      - atlas:openmpi
+      - eckit:openmpi
+    dependencies:
+      - ecmwf/ecbuild@develop
+      - ecmwf/eckit@develop
+      - ecmwf/fckit@develop
+      - ecmwf/atlas@develop
+      - ecmwf/eccodes@develop
+    dependency_cmake_options:
+      - ecmwf/atlas:-DENABLE_FORTRAN=ON
+    parallel: 64
+    ntasks: 16
+    env:
+      - CTEST_PARALLEL_LEVEL=1
+      - OMPI_MCA_rmaps_base_oversubscribe=1
+      - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples
+      - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions
+
+mpi_off:
+  build:
+    modules:
+      - ninja
+    dependencies:
+      - ecmwf/ecbuild@develop
+      - ecmwf/eckit@develop
+      - ecmwf/fckit@develop
+      - ecmwf/atlas@develop
+      - ecmwf/eccodes@develop
+    dependency_cmake_options:
+      - ecmwf/atlas:-DENABLE_FORTRAN=ON
+    parallel: 64

src/nwp_emulator/CMakeLists.txt

@@ -14,10 +14,13 @@ ecbuild_add_library(
     SOURCES
         base_data_reader.h
         grib_file_reader.h
+        config_reader.h
         nwp_definitions.h
         nwp_data_provider.h
         nwp_data_provider.cc
         grib_file_reader.cc
+        config_reader.cc
+        config_reader_funcs.cc
     PUBLIC_INCLUDES
        $<BUILD_INTERFACE:${PROJECT_BINARY_DIR}/src>
        $<BUILD_INTERFACE:${PROJECT_SOURCE_DIR}/src>

src/nwp_emulator/README.md

@@ -0,0 +1,106 @@
+# Plume Numerical Weather Prediction Model Emulator
+
+This tool is here to help you with your Plume plugins development.
+You can use it to emulate model runs with Plume loading the plugins from your configuration.
+
+## Features
+
+- Dry run: The NWP emulator can run without Plume to develop new features, or to test your data generation to ensure it fits the purpose of your plugin test.
+- Plume run: The NWP emulator runs Plume with the passed configuration. It can be used to test the Plume framweork, or test Plume plugins.
+- GRIB as input: The NWP emulator can either take a directory containing GRIB files or a GRIB file as input source. It will attempt to serve the data in each file as if it were a single model step. The number of emulator steps run depend on the number of GRIB files in the source folder, or a user-defined maximum number of steps. This approach allows a developer to test a plugin with real data. *Note:* the emulator is not looking recursively into the source folder, any subdirectory will be ignored.
+- Config as input: The NWP emulator can take a configuration file that sets its run parameters (number of steps, grid, number of levels, fields...). The data is synthetically generated by a set of available functions, see [below](#configuration-source) for more details.
+
+## Usage
+
+Once installed (check out [Plume install guide](../../README.md/#installation) for more info), the emulator tool can be used as follows:
+```bash
+mpirun -np 2 nwp_emulator_run [--grib-src=<path> | --config-src=<path>] [--plume-cfg=<path>] [OPTION]... [--help]
+```
+To make a dry run, omit the `--plume-cfg` flag.
+
+## Quick start
+### GRIB source
+
+The GRIB files are served alphabetically so your source folder should be organised to ensure the data you expect is served in the order you expect. Validation is run on all the source files to make sure they all contain the same fields and levels on the same grid. Check out the [MARS documentation](https://confluence.ecmwf.int/display/UDOC/MARS+user+documentation) if you need a hand to retrieve historical data. 
+
+### Configuration source
+
+Currently, the emulator supports five functions for data generation:
+- Vortex Rollup (provided by the [Atlas library](https://confluence.ecmwf.int/display/ATLAS))
+- Random (uniform or normal distributions)
+- Step
+- Cardinal Sine
+- Gaussian
+
+All functions accept a focus area which can be translated between each time step by a user-defined factor.
+Validation is run on the configuration file to ensure options are valid, but it lies with the user to make sure the selected options achieve their purpose.
+
+Here is an example of YAML configuration file you can use to generate model data, which demonstrate the options that each generation function accept:
+
+```yaml
+emulator:
+  n_steps: 5
+  grid_identifier: "N80"
+  vertical_levels: 5
+  # can provide area here if all population methods should use a non global area by default
+  # range for lon is assumed [-180,180] and lat [-90,90]
+  fields:
+    100u: # use the field short name
+      levtype: "sfc"
+      apply:
+        vortex_rollup:
+          area: [71.5, -25, 34.5, 45]
+          time_variation: 1.1
+    100v:
+      levtype: "sfc"
+      apply:
+        vortex_rollup:
+          time_variation: 1.1
+    u: # no specified levtype means not surface
+      apply: # functions applied in the given order
+        levels:
+          "1":
+            random: # no specified area means global
+              distribution: "uniform"
+            step:
+              area: [71.5, -25, 34.5, 45] # rectangle represented by NW and SE (lat,lon) coordinates
+              value: 10.0
+              variation: 1.0
+              translation: [1.0, 1.0] # degrees of translation of the area per time step (lat, lon)
+          "2":
+            sinc:
+              modes: 3 # number of peaks or sinks in the generation area
+              min: -1.0
+              max: 10.0
+              spread: 10.0
+              sink: false
+          "3:":
+            gaussian:
+              modes: 2
+              min: 0.0
+              max: 1.0
+              max_stddev: 3.0
+              sink: true
+    v: "u"
+    t:
+      apply:
+        levels: # this key should not be used for sfc
+          "2": # when using levels, they should range from 1-<vertical_levels>
+            random:
+              area: [71.5, -25, 34.5, 45]
+              distribution: "uniform"
+              min: 10.0
+              max: 35.0
+          "1,3":
+            step:
+              probability: 0.1
+              value: 45.0
+          "4:":
+            random:
+              distribution: "normal"
+              mean: 20.0
+              stddev: 20.0
+```
+
+## License
+© 2025 ECMWF. All rights reserved.

src/nwp_emulator/base_data_reader.h

@@ -13,6 +13,8 @@
 #include <string>
 #include <vector>
 
+#include "atlas/field/Field.h"
+#include "eckit/exception/Exceptions.h"
 #include "nwp_definitions.h"
 
 namespace nwp_emulator {
@@ -35,6 +37,15 @@ class BaseDataReader {
     BaseDataReader(int stepCountLimit) : stepCountLimit_(stepCountLimit) {}
     virtual ~BaseDataReader() = default;
 
+    /**
+     * @brief Sets the reader lon,lat area if necessary for data generation.
+     *
+     * @param lonlat The lonlat field defininf the partition grid points.
+     */
+    virtual void setReaderArea(const atlas::Field& lonlat) {
+        throw eckit::NotImplemented("This data reader does not support area definition", Here());
+    }
+
     /**
      * @brief Provides the raw values for a single levelin the next message from the data source.
      *