diff --git a/src/multio/action/Action.h b/src/multio/action/Action.h index 82f89faf7..aaf010eea 100644 --- a/src/multio/action/Action.h +++ b/src/multio/action/Action.h @@ -25,6 +25,7 @@ #include "ActionStatistics.h" #include "eckit/memory/NonCopyable.h" #include "multio/config/ComponentConfiguration.h" +#include "multio/datamod/core/EntryParser.h" #include "multio/message/Message.h" #include "multio/util/FailureHandling.h" @@ -130,4 +131,16 @@ class ActionBuilder final : public ActionBuilderBase { //-------------------------------------------------------------------------------------------------- +template +ParsedConfig parseConfig(const ComponentConfiguration& compConf) { + multio::datamod::ParseOptions opts; + opts.allowAdditionalKeys = false; + + auto conf = compConf.parsedConfig(); + conf.remove("type"); + conf.remove("next"); + + return multio::datamod::readRecordByValue(conf, opts); +} + } // namespace multio::action diff --git a/src/multio/action/encode-mtg2/EncodeMtg2.cc b/src/multio/action/encode-mtg2/EncodeMtg2.cc index 716c32933..9d5d67bd6 100644 --- a/src/multio/action/encode-mtg2/EncodeMtg2.cc +++ b/src/multio/action/encode-mtg2/EncodeMtg2.cc @@ -43,23 +43,9 @@ mars2grib::RawOptions mapOpts(EncodeMtg2Options opts) { return ret; }; -EncodeMtg2Options parseOpts(const ComponentConfiguration& compConf) { - /// TODO(pgeier) With C++20 designators are more useful for inline creation of structs: - /// ParsedOptions{.allowAdditionalKeys=false} - dm::ParseOptions opts; - opts.allowAdditionalKeys = false; - - // TODO(pgeier) Fix after refactoring action - need to remove keys "type" and "next" - auto conf = compConf.parsedConfig(); - conf.remove("type"); - conf.remove("next"); - - return dm::readRecordByValue(conf, opts); -} - EncodeMtg2::EncodeMtg2(const ComponentConfiguration& compConf) : - ChainedAction{compConf}, opts_{parseOpts(compConf)}, mars2grib_{mapOpts(opts_)} {} + ChainedAction{compConf}, opts_{parseConfig(compConf)}, mars2grib_{mapOpts(opts_)} {} void EncodeMtg2::executeImpl(Message msg) { diff --git a/src/multio/action/scale/Scale.cc b/src/multio/action/scale/Scale.cc index 0c01d659e..6a74c23d0 100644 --- a/src/multio/action/scale/Scale.cc +++ b/src/multio/action/scale/Scale.cc @@ -11,7 +11,6 @@ #include "multio/action/scale/Scale.h" #include "eckit/config/LocalConfiguration.h" -#include "eckit/exception/Exceptions.h" #include "multio/LibMultio.h" #include "multio/datamod/ContainerInterop.h" @@ -21,63 +20,46 @@ namespace multio::action::scale { -const Mappings getMappings(const std::string& preset) { - if (preset != "local-to-wmo" && preset != "wmo-to-local") { - throw eckit::UserError("Preset " + preset + " does not exist!", Here()); - } - +const std::vector getPresetMappings(const Preset& preset) { // Load the mapping file eckit::LocalConfiguration mappingConf{eckit::YAMLConfiguration{eckit::PathName{ multio::LibMultio::instance().libraryHome() + "/share/multio/mappings/local-to-wmo.yaml" }}}; - // Read the mappings and put them into the map - // We use the same mapping file for local-to-wmo and wmo-to-local, the - // second is just the reverse mapping of the first! - Mappings mappings; - for (auto& mapping : mappingConf.getSubConfigurations()) { - const auto paramIn = mapping.getInt64("param-in"); - const auto paramOut = mapping.getInt64("param-out"); - const auto scaling = mapping.getDouble("scaling"); - if (preset == "local-to-wmo") { - mappings[paramIn] = {paramOut, scaling}; - } - else { - mappings[paramOut] = {paramIn, 1.0 / scaling}; + auto mappings = datamod::RecordMapper>::parse(mappingConf.getSubConfigurations()); + + if (preset == Preset::WmoToLocal) { + for (auto& mapping : mappings) { + std::swap(mapping.paramIn, mapping.paramOut); + mapping.scaling.set(1.0 / mapping.scaling.get()); } } + return mappings; } -Mappings getMappings(const eckit::LocalConfiguration& config) { +Mappings getMappings(const ScaleConfig& config) { Mappings mappings; // Read the preset mapping from a mappings file - if (config.has("preset-mappings")) { - ASSERT(config.isString("preset-mappings")); - mappings = getMappings(config.getString("preset-mappings")); + if (config.presetMappings.isSet()) { + for (auto& mapping : getPresetMappings(config.presetMappings.get())) { + mappings[mapping.paramIn.get()] = {mapping.paramOut.get(), mapping.scaling.get()}; + } } // Read any user defined mappings from the action configuration - if (config.has("custom-mappings")) { - ASSERT(config.isSubConfigurationList("custom-mappings")); - for (auto& mapping : config.getSubConfigurations("custom-mappings")) { - const auto paramIn = mapping.getInt64("param-in"); - const auto paramOut = mapping.getInt64("param-out"); - const auto scaling = mapping.getDouble("scaling"); - ASSERT(mappings.find(paramIn) == mappings.end()); - mappings[paramIn] = {paramOut, scaling}; + if (config.customMappings.isSet()) { + for (auto& mapping : config.customMappings.get()) { + mappings[mapping.paramIn.get()] = {mapping.paramOut.get(), mapping.scaling.get()}; } } - if (mappings.empty()) { - throw eckit::UserError("No scale mapping was found, set 'preset' or 'mappings' in action configuration!", Here()); - } return mappings; } Scale::Scale(const ComponentConfiguration& compConf) : - ChainedAction(compConf), mappings_{getMappings(compConf.parsedConfig())} {} + ChainedAction(compConf), mappings_{getMappings(parseConfig(compConf))} {} void Scale::executeImpl(message::Message msg) { // Skip non-field messages @@ -133,3 +115,21 @@ void Scale::print(std::ostream& os) const { static ActionBuilder ScaleBuilder("scale"); } // namespace multio::action::scale + + +namespace multio::datamod { + +action::scale::Preset ParseType::parse(const std::string& val) { + if (val == "local-to-wmo") { + return action::scale::Preset::LocalToWmo; + } + if (val == "wmo-to-local") { + return action::scale::Preset::WmoToLocal; + } + throw DataModellingException( + std::string("ParseType::parse Unknown value for PresetMappings: ") + val, + Here() + ); +} + +} // namespace multio::datamod diff --git a/src/multio/action/scale/Scale.h b/src/multio/action/scale/Scale.h index d242ba2ad..201cd3e0c 100644 --- a/src/multio/action/scale/Scale.h +++ b/src/multio/action/scale/Scale.h @@ -1,7 +1,10 @@ #pragma once +#include "eckit/exception/Exceptions.h" + #include "multio/action/ChainedAction.h" #include "multio/datamod/core/EntryDef.h" +#include "multio/datamod/core/NestedRecord.h" #include "multio/datamod/MarsKeys.h" #include "multio/datamod/GribKeys.h" @@ -23,37 +26,51 @@ struct ScaleMetadataKeys { }; //------------------------ Action Configuration Keys ------------------------// -// -// TODO: Support (lists of) sub-records in datamod -// -// struct ScaleMappingKeys { -// dm::Entry paramIn; -// dm::Entry paramOut; -// dm::Entry scaling; -// -// static constexpr std::string_view record_name_ = "scale-mapping-keys"; -// -// using SMK = ScaleMappingKeys; -// static constexpr auto record_entries_ = std::make_tuple( -// dm::entryDef("param-in", &SMK::paramIn), -// dm::entryDef("param-out", &SMK::paramOut), -// dm::entryDef("scaling", &SMK::scaling) -// ); -// }; -// -// struct ScaleConfigurationKeys { -// dm::Entry presetMappings; -// dm::Entry> customMappings; -// -// static constexpr std::string_view record_name_ = "scale-action-configuration"; -// -// using SCK = ScaleConfigurationKeys; -// static constexpr auto record_entries_ = std::make_tuple( -// dm::entryDef("preset-mappings", &SCK::presetMappings).tagOptional(), -// dm::entryDef("custom-mappings", &SCK::customMappings).tagOptional() -// ); -// }; -// + +enum class Preset : std::size_t +{ + LocalToWmo, // "local-to-wmo" + WmoToLocal // "wmo-to-local" +}; + +struct ScaleMappingConfig { + dm::Entry paramIn; + dm::Entry paramOut; + dm::Entry scaling; + + static constexpr std::string_view record_name_ = "scale-mapping-config"; + + static constexpr auto record_entries_ = std::make_tuple( + dm::entryDef("param-in", &ScaleMappingConfig::paramIn), + dm::entryDef("param-out", &ScaleMappingConfig::paramOut), + dm::entryDef("scaling", &ScaleMappingConfig::scaling) + ); +}; + +struct ScaleConfig { + dm::Entry presetMappings; + dm::NestedEntry_t> customMappings; + + static constexpr std::string_view record_name_ = "scale-action-config"; + + static constexpr auto record_entries_ = std::make_tuple( + dm::entryDef("preset-mappings", &ScaleConfig::presetMappings).tagOptional(), + + // Custom mappings are user defined mappings that will be applied on top of a preset, if a mapping from a + // param already exists, it will be overwritten by the custom mapping. + dm::entryDef("custom-mappings", &ScaleConfig::customMappings).tagOptional() + ); + + static void validate(const ScaleConfig& k) { + if (!k.presetMappings.isSet() && (!k.customMappings.isSet() || k.customMappings.get().empty())) { + throw eckit::UserError( + "Either 'preset-mappings' or 'custom-mappings' must be set in scale action configuration!", + Here() + ); + } + } +}; + //---------------------------------------------------------------------------// using Param = std::int64_t; @@ -78,3 +95,28 @@ class Scale final : public ChainedAction { }; } // namespace multio::action::scale + + +namespace multio::util { + +template <> +struct TypeToString { + std::string operator()() const { return std::string("ScaleMappingConfig"); }; +}; + +template <> +struct TypeToString { + std::string operator()() const { return std::string("PresetMappings"); }; +}; + +} // namespace multio::util + + +namespace multio::datamod { + +template <> +struct ParseType { + static action::scale::Preset parse(const std::string& s); +}; + +} // namespace multio::datamod diff --git a/src/multio/datamod/ContainerInterop.h b/src/multio/datamod/ContainerInterop.h index c05e06fc4..99a06fdc4 100644 --- a/src/multio/datamod/ContainerInterop.h +++ b/src/multio/datamod/ContainerInterop.h @@ -269,9 +269,6 @@ struct EntryParser { if (c.isIntegralList(key)) { return std::forward(func)(util::TypeTag>{}); } - // if (c.isList(key)) { - // // Not supported - // } if (c.isString(key)) { return std::forward(func)(util::TypeTag{}); } @@ -281,6 +278,9 @@ struct EntryParser { if (c.isSubConfiguration(key)) { return std::forward(func)(util::TypeTag{}); } + if (c.isSubConfigurationList(key)) { + return std::forward(func)(util::TypeTag>{}); + } return std::forward(func)(); } diff --git a/src/multio/datamod/core/NestedRecord.h b/src/multio/datamod/core/NestedRecord.h index 4bed84784..3bb951b23 100644 --- a/src/multio/datamod/core/NestedRecord.h +++ b/src/multio/datamod/core/NestedRecord.h @@ -10,6 +10,8 @@ #pragma once +#include + #include "multio/datamod/core/EntryDef.h" #include "multio/datamod/core/EntryDumper.h" #include "multio/datamod/core/EntryParser.h" @@ -57,6 +59,26 @@ struct RecordMapper { }; +template +struct RecordMapper> { + // Parse record from other record or implemented container + template < + typename OtherRec, + std::enable_if_t || EntryParserIsSpecialized_v, bool> + = true> + static std::vector parse(const std::vector& vec) { + std::vector res; + res.reserve(vec.size()); + for (const auto& v : vec) { + res.push_back(readRecordByValue(v)); + } + return res; + } + + // Dumping of vector is not implemented +}; + + template using NestedEntry_t = Entry>;