Added tests for the NWP Emulator

.github/ci-hpc-config.yml

@@ -1,12 +1,41 @@
-build:
-  modules:
-    - ninja
-  dependencies:
-    - ecmwf/ecbuild@develop
-    - ecmwf/eckit@develop
-    - ecmwf/fckit@develop
-    - ecmwf/atlas@develop
-    - ecmwf/eccodes@develop
-  dependency_cmake_options:
-    - ecmwf/atlas:-DENABLE_FORTRAN=ON
-  parallel: 64
+matrix:
+  - mpi_on
+  - mpi_off
+
+mpi_on:
+  build:
+    modules:
+      - ninja
+      - openmpi
+    modules_package:
+      - atlas:openmpi
+      - eckit:openmpi
+    dependencies:
+      - ecmwf/ecbuild@develop
+      - ecmwf/eckit@develop
+      - ecmwf/fckit@develop
+      - ecmwf/atlas@develop
+      - ecmwf/eccodes@develop
+    dependency_cmake_options:
+      - ecmwf/atlas:-DENABLE_FORTRAN=ON
+    parallel: 64
+    ntasks: 16
+    env:
+      - CTEST_PARALLEL_LEVEL=1
+      - OMPI_MCA_rmaps_base_oversubscribe=1
+      - ECCODES_SAMPLES_PATH=$ECCODES_DIR/share/eccodes/samples
+      - ECCODES_DEFINITION_PATH=$ECCODES_DIR/share/eccodes/definitions
+
+mpi_off:
+  build:
+    modules:
+      - ninja
+    dependencies:
+      - ecmwf/ecbuild@develop
+      - ecmwf/eckit@develop
+      - ecmwf/fckit@develop
+      - ecmwf/atlas@develop
+      - ecmwf/eccodes@develop
+    dependency_cmake_options:
+      - ecmwf/atlas:-DENABLE_FORTRAN=ON
+    parallel: 64

src/nwp_emulator/README.md

@@ -65,6 +65,7 @@ emulator:
             step:
               area: [71.5, -25, 34.5, 45] # rectangle represented by NW and SE (lat,lon) coordinates
               value: 10.0
+              variation: 1.0
               translation: [1.0, 1.0] # degrees of translation of the area per time step (lat, lon)
           "2":
             sinc:

src/nwp_emulator/grib_file_reader.cc

@@ -44,19 +44,25 @@ GRIBFileReader::GRIBFileReader(const eckit::PathName& inputPath, size_t rank, si
         gridName_ = std::string(gridNameBuffer.begin(), gridNameBuffer.end());
         gridName_.resize(strlen(gridName_.c_str()));
     }
-    // 2. Field names (parameters)
-    size_t paramCount = params_.size();
-    eckit::mpi::comm().broadcast(paramCount, root_);
-    std::vector<char> paramBuffer(64);  // Params short name are typically 1-4 chars
-    for (size_t i = 0; i < paramCount; ++i) {
-        paramBuffer.resize(64, '\0');  // clean param buffer
-        if (rank_ == root_) {
-            paramBuffer.assign(params_[i].begin(), params_[i].end());
+    // 2. Field names & metadata (parameters)
+    std::vector<char> paramBuffer;
+    if (rank_ == root_) {
+        for (size_t i = 0; i < params_.size(); i++) {
+            std::copy(params_[i].begin(), params_[i].end(), std::back_inserter(paramBuffer));
+            paramBuffer.push_back(';');
         }
-        eckit::mpi::comm().broadcast(paramBuffer, root_);
-        if (rank_ != root_) {
-            std::string param(paramBuffer.begin(), paramBuffer.end());
-            param.resize(strlen(param.c_str()));  // Trim trailing '\0's
+        paramBuffer.pop_back(); // remove last ';' separator
+    }
+    size_t paramSize = paramBuffer.size();
+    eckit::mpi::comm().broadcast(paramSize, root_);
+    paramBuffer.resize(paramSize);
+    eckit::mpi::comm().broadcast(paramBuffer, root_); // broadcast a single string to limit communication
+    if (rank_ != root_) {
+        std::string paramBufferStr(paramBuffer.begin(), paramBuffer.end());
+        paramBufferStr.resize(strlen(paramBufferStr.c_str()));
+        std::stringstream ss(paramBufferStr);
+        std::string param;
+        while (std::getline(ss, param, ';')) {
             params_.push_back(param);
         }
     }

tests/CMakeLists.txt

@@ -7,5 +7,10 @@
 # granted to it by virtue of its status as an intergovernmental organisation nor
 # does it submit to any jurisdiction.
 
+if( NOT DEFINED MPI_SLOTS )
+  set( MPI_SLOTS 9999 )
+endif()
+
 add_subdirectory(core)
-add_subdirectory(api)
+add_subdirectory(api)
+add_subdirectory(nwp_emulator)

tests/nwp_emulator/CMakeLists.txt

@@ -0,0 +1,63 @@
+# (C) Copyright 2025- ECMWF.
+#
+# This software is licensed under the terms of the Apache Licence Version 2.0
+# which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+#
+# In applying this licence, ECMWF does not waive the privileges and immunities
+# granted to it by virtue of its status as an intergovernmental organisation nor
+# does it submit to any jurisdiction.
+# Test files repository on Nexus
+set(ECBUILD_DOWNLOAD_BASE_URL https://get.ecmwf.int/repository/plume-test-data)
+
+set(test_nwp_emulator_files_dir ${CMAKE_BINARY_DIR}/tests/nwp_emulator/data)
+add_custom_target(
+    make_nwp_emulator_test_data_dir ALL COMMAND ${CMAKE_COMMAND} -E make_directory ${test_nwp_emulator_files_dir}
+)
+
+ecbuild_get_test_multidata(
+    TARGET test_nwp_emulator_files
+    NAMES
+        model_data_1.grib
+        model_data_2.grib
+    DIRNAME
+        nwp_emulator
+    DIRLOCAL
+        ${test_nwp_emulator_files_dir}
+    NOCHECK
+)
+
+ecbuild_add_test(
+    TARGET      plume_test_nwp_grib
+    SOURCES     test_grib_reader.cc
+    LIBS        plume_nwp_emulator
+    ENVIRONMENT TEST_DATA_DIR=${test_nwp_emulator_files_dir}
+    MPI         3
+    CONDITION   eckit_HAVE_MPI
+    TEST_DEPENDS test_nwp_emulator_files
+)
+
+ecbuild_add_test(
+    TARGET      plume_test_nwp_config
+    SOURCES     test_config_reader.cc
+    LIBS        plume_nwp_emulator
+    ENVIRONMENT TEST_DATA_DIR=${CMAKE_CURRENT_SOURCE_DIR}/data/
+    MPI         3
+    CONDITION   eckit_HAVE_MPI
+)
+
+ecbuild_add_library(
+    TARGET nwp_emulator_test_plugin
+    SOURCES 
+        nwp_emulator_plugin.h
+        nwp_emulator_plugin.cc
+    PRIVATE_LIBS
+        plume_plugin
+)
+
+ecbuild_add_test(
+    TARGET  plume_test_nwp_tool
+    LIBS simple_plugin nwp_emulator_test_plugin
+    COMMAND nwp_emulator_run.x
+    ARGS --config-src=${CMAKE_CURRENT_SOURCE_DIR}/data/valid_config.yml
+         --plume-cfg=${CMAKE_CURRENT_SOURCE_DIR}/data/plume_config.yml
+)

tests/nwp_emulator/data/invalid_config.yml

@@ -0,0 +1 @@
+no_emulator_key: -1

tests/nwp_emulator/data/plume_config.yml

@@ -0,0 +1,14 @@
+{
+  "plugins": [
+    {
+      "name": "SimplePlugin",
+      "lib": "simple_plugin",
+      "plugincore-config": {}
+    },
+    {
+      "name": "NWPEmulatorPlugin",
+      "lib": "nwp_emulator_test_plugin",
+      "plugincore-config": {}
+    }
+  ]
+}

tests/nwp_emulator/data/valid_config.yml

@@ -0,0 +1,39 @@
+emulator:
+  n_steps: 2
+  grid_identifier: "N80"
+  vertical_levels: 5
+  fields:
+    100u:
+      levtype: "sfc"
+      apply:
+        vortex_rollup:
+          area: [71.5, -25, 34.5, 45]
+          time_variation: 1.1
+    u:
+      apply:
+        levels:
+          "2":
+            random:
+              distribution: "uniform"
+              min: 1.0
+              max: 2.0
+            step:
+              area: [71.5, -25, 34.5, 45]
+              value: 10.0
+              variation: 1.0
+              translation: [1.0, 1.0]
+          "1,3":
+            sinc:
+              modes: 3
+              min: -1.0
+              max: 10.0
+              spread: 10.0
+              sink: false
+          "4:":
+            gaussian:
+              modes: 2
+              min: 1.0
+              max: 2.0
+              max_stddev: 3.0
+              sink: true
+    v: "u"

tests/nwp_emulator/nwp_emulator_plugin.cc

@@ -0,0 +1,46 @@
+/*
+ * (C) Copyright 2025- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+#include "nwp_emulator_plugin.h"
+#include <iostream>
+
+
+namespace nwp_emulator_test_plugin {
+
+REGISTER_LIBRARY(NWPEmulatorPlugin)
+
+NWPEmulatorPlugin::NWPEmulatorPlugin() : Plugin("NWPEmulatorPlugin"){};
+
+NWPEmulatorPlugin::~NWPEmulatorPlugin(){};
+
+const NWPEmulatorPlugin& NWPEmulatorPlugin::instance() {
+    static NWPEmulatorPlugin instance;
+    return instance;
+}
+//--------------------------------------------------------------
+
+
+// NWPEmulatorPluginCore
+static plume::PluginCoreBuilder<NWPEmulatorPluginCore> runnable_plugincore_FooBuilder_;
+
+NWPEmulatorPluginCore::NWPEmulatorPluginCore(const eckit::Configuration& conf) : PluginCore(conf) {}
+
+NWPEmulatorPluginCore::~NWPEmulatorPluginCore() {}
+
+void NWPEmulatorPluginCore::run() {
+    eckit::Log::info() << "Consuming parameters " << modelData().getAtlasFieldShared("100u").name() << ", "
+                       << modelData().getAtlasFieldShared("u").name() << ", "
+                       << modelData().getAtlasFieldShared("v").name() << std::endl;
+}
+
+//--------------------------------------------------------------
+
+
+}  // namespace nwp_emulator_test_plugin

tests/nwp_emulator/nwp_emulator_plugin.h

@@ -0,0 +1,55 @@
+/*
+ * (C) Copyright 2025- ECMWF.
+ *
+ * This software is licensed under the terms of the Apache Licence Version 2.0
+ * which can be obtained at http://www.apache.org/licenses/LICENSE-2.0.
+ *
+ * In applying this licence, ECMWF does not waive the privileges and immunities
+ * granted to it by virtue of its status as an intergovernmental organisation nor
+ * does it submit to any jurisdiction.
+ */
+#include <string>
+#include "plume/Plugin.h"
+#include "plume/PluginCore.h"
+
+namespace nwp_emulator_test_plugin {
+
+
+// ------ Foo runnable plugincore that self-registers! -------
+class NWPEmulatorPluginCore : public plume::PluginCore {
+public:
+    NWPEmulatorPluginCore(const eckit::Configuration& conf);
+    ~NWPEmulatorPluginCore();
+    void run() override;
+    constexpr static const char* type() { return "nwpemulator-plugincore"; }
+};
+// ------------------------------------------------------
+
+// ------------------------------------------------------
+class NWPEmulatorPlugin : public plume::Plugin {
+
+public:
+    NWPEmulatorPlugin();
+    ~NWPEmulatorPlugin();
+
+    plume::Protocol negotiate() override {
+        plume::Protocol protocol;
+        protocol.requireAtlasField("100u");
+        protocol.requireAtlasField("u");
+        protocol.requireAtlasField("v");
+
+        return protocol;
+    }
+
+    // Return the static instance
+    static const NWPEmulatorPlugin& instance();
+
+    std::string version() const override { return "0.0.1-NWPEmulator"; }
+
+    std::string gitsha1(unsigned int count) const override { return "undefined"; }
+
+    virtual std::string plugincoreName() const override { return NWPEmulatorPluginCore::type(); }
+};
+// ------------------------------------------------------
+
+}  // namespace nwp_emulator_test_plugin