Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace all curly during multio configuration initialization #67

Merged
merged 1 commit into from
Mar 25, 2025
Merged

Replace all curly during multio configuration initialization #67

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 2 additions & 8 deletions src/multio/action/encode/Encode.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -136,20 +136,14 @@ std::unique_ptr<GribEncoder> makeEncoder(const eckit::LocalConfiguration& conf,

if (format == "grib") {
ASSERT(conf.has("template"));
std::string tmplPath = conf.getString("template");
// TODO provide utility to distinguish between relative and absolute paths
eckit::AutoStdFile fin{multioConfig.replaceCurly(tmplPath)};
eckit::AutoStdFile fin{conf.getString("template")};
int err;
auto sample = codes_handle_new_from_file(nullptr, fin, PRODUCT_GRIB, &err);
handleCodesError("eccodes error while reading the grib template: ", err, Here());

if (conf.has("atlas-named-grid")) {
const auto atlasNamedGrid
= util::replaceCurly(conf.getString("atlas-named-grid"), [](std::string_view replace) {
std::string lookUpKey{replace};
char* env = ::getenv(lookUpKey.c_str());
return env ? std::optional<std::string>{env} : std::optional<std::string>{};
});
const auto atlasNamedGrid = conf.getString("atlas-named-grid");

eckit::Log::info() << "REQUESTED ATLAS GRID DEFINITION UPDATE: " << atlasNamedGrid << std::endl;

Expand Down
3 changes: 1 addition & 2 deletions src/multio/action/encode/GribEncoder.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -238,8 +238,7 @@ std::optional<ValueSetter> valueSetter(GribEncoder& g, const std::string& key) {
}

std::string getUnstructuredGridType(const eckit::LocalConfiguration& config) {
std::optional<std::string_view> (*F)(std::string_view) = &multio::util::getEnv;
return multio::util::replaceCurly(config.getString("unstructured-grid-type"), F);
return config.getString("unstructured-grid-type");
}

} // namespace
Expand Down
7 changes: 2 additions & 5 deletions src/multio/action/encode/GridDownloader.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -42,9 +42,7 @@ std::unique_ptr<multio::action::GribEncoder> createEncoder(const multio::config:
<< std::endl;
return nullptr;
}
const auto tmplPath = compConf.parsedConfig().getString("grid-downloader-template");

eckit::AutoStdFile fin{compConf.multioConfig().replaceCurly(tmplPath)};
eckit::AutoStdFile fin{compConf.parsedConfig().getString("grid-downloader-template")};

int err = 0;
auto encoder = std::make_unique<multio::action::GribEncoder>(
Expand All @@ -64,8 +62,7 @@ atlas::Grid readGrid(const std::string& name) {
}

std::string getUnstructuredGridType(const multio::config::ComponentConfiguration& compConf) {
std::optional<std::string_view> (*F)(std::string_view) = &multio::util::getEnv;
return multio::util::replaceCurly(compConf.parsedConfig().getString("unstructured-grid-type"), F);
return compConf.parsedConfig().getString("unstructured-grid-type");
}

} // namespace
Expand Down
8 changes: 3 additions & 5 deletions src/multio/action/interpolate-fesom/InterpolateFesom.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -59,13 +59,11 @@ void fill_metadata(const message::Metadata& in_md, message::Metadata& out_md, si
return;
};

// TODO: Consider removing this function!
// fname comes from an already expanded configuration.
std::string fullFileName(const std::string& fname) {
if (fname != "none") {
const std::string fullFname = util::replaceCurly(fname, [](std::string_view replace) {
std::string lookUpKey{replace};
char* env = ::getenv(lookUpKey.c_str());
return env ? std::optional<std::string>{env} : std::optional<std::string>{};
});
const std::string fullFname = util::replaceCurly(fname);
eckit::PathName tmp{fullFname};
if (!tmp.exists()) {
std::ostringstream os;
Expand Down
36 changes: 7 additions & 29 deletions src/multio/action/interpolate/Interpolate.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -86,15 +86,10 @@ void forwardMetadata(const eckit::LocalConfiguration& cfg, DestType& destination
}
if (cfg.isString(key)) {
if (cfg.isString("grid")) {
const std::string input = util::replaceCurly(cfg.getString("grid"), [](std::string_view replace) {
std::string lookUpKey{replace};
char* env = ::getenv(lookUpKey.c_str());
return env ? std::optional<std::string>{env} : std::optional<std::string>{};
});
destination.set(key, input);
destination.set(key, cfg.getString("grid"));
}
else {
destination.set(key, cfg.getString(key).c_str());
destination.set(key, cfg.getString(key));
}
return;
}
Expand Down Expand Up @@ -214,11 +209,7 @@ void fill_input(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametr

inp.visit([&](auto& v) {
if constexpr (std::is_same_v<std::decay_t<decltype(v)>, std::string>) {
const std::string input = util::replaceCurly(v, [](std::string_view replace) {
std::string lookUpKey{replace};
char* env = ::getenv(lookUpKey.c_str());
return env ? std::optional<std::string>{env} : std::optional<std::string>{};
});
const std::string input = util::replaceCurly(v);

static const std::regex sh("(T|TCO|TL)([1-9][0-9]*)");
static const std::regex gg("([FNO])([1-9][0-9]*)");
Expand Down Expand Up @@ -350,11 +341,7 @@ void fill_job(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametris
std::vector<double> grid(2, 0.0);

if (cfg.isString("grid")) {
const std::string input = util::replaceCurly(cfg.getString("grid"), [](std::string_view replace) {
std::string lookUpKey{replace};
char* env = ::getenv(lookUpKey.c_str());
return env ? std::optional<std::string>{env} : std::optional<std::string>{};
});
const std::string input = cfg.getString("grid");
#define fp "([+]?([0-9]*[.])?[0-9]+([eE][-+][0-9]+)?)"
static const std::regex ll(fp "/" fp);
static const std::regex H("([h|H])([1-9][0-9]*)(_nested)?");
Expand Down Expand Up @@ -409,24 +396,15 @@ void fill_job(const eckit::LocalConfiguration& cfg, mir::param::SimpleParametris

// If no interpolation matrix name is provided, generate one
if (interpolationMatrix) {
const std::string input = util::replaceCurly(inp.get<std::string>(), [](std::string_view replace) {
std::string lookUpKey{replace};
char* env = ::getenv(lookUpKey.c_str());
return env ? std::optional<std::string>{env} : std::optional<std::string>{};
});
const std::string input = util::replaceCurly(inp.get<std::string>());
if (input != "fesom") {
std::ostringstream os;
os << " - interpolation matrix supported only for fesom -> Healpix" << std::endl;
throw eckit::SeriousBug(os.str(), Here());
}
const auto& options = cfg.getSubConfiguration("options");
const auto cache_path = cfg.has("cache-path") ? cfg.getString("cache-path") : "";
const auto expanded_cache_path = util::replaceCurly(cache_path, [](std::string_view replace) {
std::string lookUpKey{replace};
char* env = ::getenv(lookUpKey.c_str());
return env ? std::optional<std::string>{env} : std::optional<std::string>{};
});
const auto weights_file = generateKey<double>(msg, expanded_cache_path, grid[0], gridKind == "HEALPix_nested" ? "nested" : "ring");
const auto cache_path = cfg.getString("cache-path", "");
const auto weights_file = generateKey<double>(msg, cache_path, grid[0], gridKind == "HEALPix_nested" ? "nested" : "ring");
destination.set("interpolation-matrix", weights_file);
}
}
Expand Down
2 changes: 1 addition & 1 deletion src/multio/action/renumber-healpix/HEALPix_ring2nest.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ std::string parseCacheFileName(const ComponentConfiguration& compConf) {
}

// Expand file name
const auto cacheFileName = compConf.multioConfig().replaceCurly(cfg.getString("cache-file-name"));
const auto cacheFileName = cfg.getString("cache-file-name");

// Check existence of the cache file
eckit::PathName tmp{cacheFileName};
Expand Down
22 changes: 9 additions & 13 deletions src/multio/action/statistics/cfg/StatisticsOptions.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -150,14 +150,12 @@ void StatisticsOptions::parseRestartPath(const config::ComponentConfiguration& c
const eckit::LocalConfiguration& cfg) {
// Read the path used to restart statistics
// Default value is "."
if (cfg.has("restart-path")) {
restartPath_ = compConf.multioConfig().replaceCurly(cfg.getString("restart-path", "."));
eckit::PathName path{restartPath_};
if (!path.exists() || !path.isDir()) {
std::ostringstream os;
os << "Restart path does not exist :: " << restartPath_ << std::endl;
throw eckit::UserError{os.str(), Here()};
}
restartPath_ = cfg.getString("restart-path", ".");
eckit::PathName path{restartPath_};
if (!path.exists() || !path.isDir()) {
std::ostringstream os;
os << "Restart path does not exist :: " << restartPath_ << std::endl;
throw eckit::UserError{os.str(), Here()};
}
return;
};
Expand All @@ -167,9 +165,7 @@ void StatisticsOptions::parseRestartTime(const config::ComponentConfiguration& c
const eckit::LocalConfiguration& cfg) {
// Read the path used to restart statistics
// Default value is "latest"
if (cfg.has("restart-time")) {
restartTime_ = compConf.multioConfig().replaceCurly(cfg.getString("restart-time", "latest"));
}
restartTime_ = cfg.getString("restart-time", "latest");
return;
};

Expand All @@ -178,7 +174,7 @@ void StatisticsOptions::parseRestartPrefix(const config::ComponentConfiguration&
const eckit::LocalConfiguration& cfg) {
// Prefix used for the restart file names in order
// to make the file name unique across different plans
restartPrefix_ = compConf.multioConfig().replaceCurly(cfg.getString("restart-prefix", "StatisticsDump"));
restartPrefix_ = cfg.getString("restart-prefix", "StatisticsDump");
return;
};

Expand Down Expand Up @@ -222,7 +218,7 @@ void StatisticsOptions::parseSolverResetAccumulatedFields(const config::Componen

void StatisticsOptions::parseValueCountThreshold(const config::ComponentConfiguration& compConf,
const eckit::LocalConfiguration& cfg) {
long threshold = stol(compConf.multioConfig().replaceCurly(cfg.getString("value-count-threshold", "-1")));
long threshold = cfg.getLong("value-count-threshold", -1);

if (threshold == -1) {
valueCountThreshold_ = std::nullopt;
Expand Down
10 changes: 0 additions & 10 deletions src/multio/config/MetadataMappings.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,16 +34,6 @@ const std::vector<message::MetadataMapping>& MetadataMappings::getMappings(const

std::vector<eckit::LocalConfiguration> sourceList = configFile.content.getSubConfigurations("data");

for (auto& s : sourceList) {
for (auto& key : s.keys()) {
// Replace the value if it is string
if (s.isString(key)) {
s.set(key, multioConf.replaceCurly(s.getString(key)));
}
}
}


// Evaluate mappings block
if (!configFile.content.has("mappings")) {
std::ostringstream oss;
Expand Down
48 changes: 42 additions & 6 deletions src/multio/config/MultioConfiguration.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -122,7 +122,9 @@ ConfigAndPaths configureFromEnv(config::LocalPeerTag tag) {
MultioConfiguration::MultioConfiguration(const eckit::LocalConfiguration& globalConfig,
const eckit::PathName& configDir, const eckit::PathName& configFile,
LocalPeerTag localPeerTag) :
parsedConfig_{globalConfig}, configDir_{configDir}, configFile_{configFile}, localPeerTag_{localPeerTag} {}
configDir_{configDir}, configFile_{configFile}, localPeerTag_{localPeerTag} {
parsedConfig_ = replaceAllCurly(globalConfig);
}

MultioConfiguration::MultioConfiguration(const eckit::PathName& configDir, const eckit::PathName& configFile,
LocalPeerTag localPeerTag) :
Expand All @@ -134,14 +136,16 @@ MultioConfiguration::MultioConfiguration(const eckit::PathName& configFile, Loca
configuration_path_name(configFile), configFile, localPeerTag) {}

MultioConfiguration::MultioConfiguration(const eckit::LocalConfiguration& globalConfig, LocalPeerTag localPeerTag) :
parsedConfig_{globalConfig}, configDir_{}, configFile_{}, localPeerTag_{localPeerTag} {}
configDir_{}, configFile_{}, localPeerTag_{localPeerTag} {
parsedConfig_ = replaceAllCurly(globalConfig);
}


MultioConfiguration::MultioConfiguration(ConfigAndPaths c, LocalPeerTag localPeerTag) :
parsedConfig_{c.parsedConfig},
configDir_{c.paths.configDir},
configFile_{c.paths.configFile},
localPeerTag_{localPeerTag} {}
configDir_{c.paths.configDir}, configFile_{c.paths.configFile}, localPeerTag_{localPeerTag} {
const auto& cfg = c.parsedConfig;
parsedConfig_ = replaceAllCurly(cfg);
}

MultioConfiguration::MultioConfiguration(LocalPeerTag localPeerTag) :
MultioConfiguration(configureFromEnv(localPeerTag), localPeerTag) {}
Expand Down Expand Up @@ -222,6 +226,38 @@ std::string MultioConfiguration::replaceCurly(const std::string& s) const {
});
}

void MultioConfiguration::replaceAllCurly(eckit::LocalConfiguration& cfg) const {
for (auto& key : cfg.keys()) {
if (cfg.isString(key)) {
cfg.set(key, replaceCurly(cfg.getString(key)));
}
if (cfg.isStringList(key)) {
std::vector<std::string> tmp;
for (const auto& str : cfg.getStringVector(key)) {
tmp.push_back(replaceCurly(str));
}
cfg.set(key, tmp);
}
if (cfg.isSubConfiguration(key)) {
const auto& sub = cfg.getSubConfiguration(key);
cfg.set(key, replaceAllCurly(sub));
}
if (cfg.isSubConfigurationList(key)) {
std::vector<eckit::LocalConfiguration> tmp;
for (const auto& sub : cfg.getSubConfigurations(key)) {
tmp.push_back(replaceAllCurly(sub));
}
cfg.set(key, tmp);
}
}
}

eckit::LocalConfiguration MultioConfiguration::replaceAllCurly(const eckit::LocalConfiguration& cfg) const {
eckit::LocalConfiguration tmp = cfg;
replaceAllCurly(tmp);
return tmp;
}

const std::vector<message::MetadataMapping>& MultioConfiguration::getMetadataMappings(
const std::string& mappings) const {
return metadataMappings_.getMappings(*this, mappings);
Expand Down
3 changes: 3 additions & 0 deletions src/multio/config/MultioConfiguration.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -138,6 +138,9 @@ class MultioConfiguration {
MultioConfiguration(const eckit::LocalConfiguration& globalConfig, const eckit::PathName& configDir,
const eckit::PathName& configFile, LocalPeerTag clientOrServer = LocalPeerTag::Client);

void replaceAllCurly(eckit::LocalConfiguration& cfg) const;
eckit::LocalConfiguration replaceAllCurly(const eckit::LocalConfiguration& cfg) const;

eckit::LocalConfiguration parsedConfig_;
eckit::PathName configDir_;
eckit::PathName configFile_;
Expand Down
29 changes: 8 additions & 21 deletions src/multio/config/PlanConfiguration.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -292,29 +292,16 @@ eckit::LocalConfiguration parseDisseminationFile(const eckit::LocalConfiguration
sId = s.str();

// Get templates Path
std::string templatesPath;
if (componentConfig.has("templates-path")) {
templatesPath = multioConf.replaceCurly(componentConfig.getString("templates-path"));
if (!eckit::PathName{templatesPath}.exists()) {
std::ostringstream oss;
oss << "templates path not exists";
throw eckit::UserError(oss.str(), Here());
}
}
else {
templatesPath = ".";
const auto templatesPath = componentConfig.getString("templates-path", ".");
if (!eckit::PathName{templatesPath}.exists()) {
std::ostringstream oss;
oss << "templates path does not exists";
throw eckit::UserError(oss.str(), Here());
}


// Get templates Path
std::string outputPath;
if (componentConfig.has("output-path")) {
outputPath = multioConf.replaceCurly(componentConfig.getString("output-path"));
}
else {
outputPath = ".";
}
outputPath = outputPath + "/" + planName + "/" + sId;
std::string outputPath = componentConfig.getString("output-path", ".") + "/" + planName + "/" + sId;
eckit::PathName{outputPath}.mkdir();

// The objective is to construct something like this starting from a metkit mars request
Expand Down Expand Up @@ -434,7 +421,7 @@ std::vector<eckit::LocalConfiguration> buildDisseminationPlan(const eckit::Local
const MultioConfiguration& multioConf) {

// Get the name of the dissemination file
std::string disseminationFile = multioConf.replaceCurly(componentConfig.getString("dissemination"));
std::string disseminationFile = componentConfig.getString("dissemination");

// Create plans
std::vector<eckit::LocalConfiguration> rawPlans
Expand All @@ -458,7 +445,7 @@ std::vector<eckit::LocalConfiguration> configurePlans(const eckit::LocalConfigur
eckit::LocalConfiguration cfg;
std::string name;
if (componentConfig.has("file")) {
const std::string fname = multioConf.replaceCurly(componentConfig.getString("file"));
const std::string fname = componentConfig.getString("file");
const auto& file = multioConf.getConfigFile(fname);
cfg = file.content;
// Force the name in the plan configuration
Expand Down
17 changes: 0 additions & 17 deletions src/multio/fdb5/FDB5Sink.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,25 +27,8 @@ using config::ComponentConfiguration;

namespace {

void replaceCurly(const ComponentConfiguration& compConf, eckit::LocalConfiguration& cfg) {
for (auto& key : cfg.keys()) {
// Replace the value if it is string
if (cfg.isString(key)) {
cfg.set(key, compConf.multioConfig().replaceCurly(cfg.getString(key)));
}
// Recursive replace of curly brackets
if (cfg.isSubConfiguration(key)) {
auto tmp = cfg.getSubConfiguration(key);
replaceCurly(compConf, tmp);
cfg.set(key, tmp);
}
}
return;
}

fdb5::Config fdb5_configuration(const ComponentConfiguration& compConf) {
auto fdb_configuration = compConf.parsedConfig().getSubConfiguration("config");
replaceCurly(compConf, fdb_configuration);

eckit::LocalConfiguration userConfig;
if (fdb_configuration.has("userConfig")) {
Expand Down
9 changes: 4 additions & 5 deletions src/multio/sink/FileSink.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -30,14 +30,13 @@ namespace {
std::string create_path(const config::ComponentConfiguration& compConf) {
const auto& cfg = compConf.parsedConfig();
auto path = cfg.getString("path");
auto expanded_path = compConf.multioConfig().replaceCurly(path);
eckit::Log::info() << "path = " << expanded_path << std::endl;
eckit::Log::info() << "path = " << path << std::endl;
if (cfg.getBool("per-server", false)) {
eckit::PathName tmp = expanded_path;
auto dirName = tmp.baseName().asString() == expanded_path ? "" : tmp.dirName().asString() + "/";
eckit::PathName tmp = path;
auto dirName = tmp.baseName().asString() == path ? "" : tmp.dirName().asString() + "/";
return dirName + util::filename_prefix() + "-" + tmp.baseName().asString();
}
return expanded_path;
return path;
}
} // namespace

Expand Down
Loading
Loading