diff --git a/CMakeLists.txt b/CMakeLists.txt index 8d2f1632a..e724956fa 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -20,7 +20,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) ### dependencies ecbuild_find_package( NAME eccodes VERSION 2.31.2 REQUIRED ) -ecbuild_find_package( NAME eckit VERSION 1.24 REQUIRED ) +ecbuild_find_package( NAME eckit VERSION 1.25 REQUIRED ) ecbuild_find_package( NAME metkit VERSION 1.10 REQUIRED ) ecbuild_find_package( NAME atlas VERSION 0.35 REQUIRED ) @@ -35,12 +35,17 @@ ecbuild_add_option( FEATURE FDB5 DESCRIPTION "Send data to FDB5" REQUIRED_PACKAGES "NAME fdb5 VERSION 5.11.20" ) -### atlas-io plugin +### eckit-codec plugin -ecbuild_add_option( FEATURE ATLAS_IO +set( eckit_HAVE_ECKIT_CODEC 0 ) +if( TARGET eckit_codec ) + set( eckit_HAVE_ECKIT_CODEC 1 ) +endif() + +ecbuild_add_option( FEATURE ECKIT_CODEC DEFAULT ON - DESCRIPTION "Use atlasIO for the restart" - REQUIRED_PACKAGES "NAME atlas_io VERSION 0.33" ) + DESCRIPTION "Use eckit codec for IO" + CONDITION eckit_HAVE_ECKIT_CODEC ) ### interpolate action diff --git a/src/multio/action/interpolate-fesom/CMakeLists.txt b/src/multio/action/interpolate-fesom/CMakeLists.txt index 5b70ed2f9..30f46ed16 100644 --- a/src/multio/action/interpolate-fesom/CMakeLists.txt +++ b/src/multio/action/interpolate-fesom/CMakeLists.txt @@ -16,11 +16,12 @@ ecbuild_add_library( ${ECKIT_INCLUDE_DIRS} CONDITION - HAVE_ATLAS_IO + HAVE_ECKIT_CODEC PUBLIC_LIBS multio - atlas_io + eckit + eckit_codec ) @@ -38,12 +39,12 @@ ecbuild_add_executable( InterpolateFesom.h CONDITION - HAVE_ATLAS_IO AND HAVE_MULTIO_BUILD_TOOLS + HAVE_ECKIT_CODEC AND HAVE_MULTIO_BUILD_TOOLS LIBS multio - atlas_io eckit + eckit_codec ) ecbuild_add_executable( @@ -59,12 +60,12 @@ ecbuild_add_executable( InterpolateFesom.h CONDITION - HAVE_ATLAS_IO AND HAVE_MULTIO_BUILD_TOOLS + HAVE_ECKIT_CODEC AND HAVE_MULTIO_BUILD_TOOLS LIBS multio - atlas_io eckit + eckit_codec ) ecbuild_add_executable( @@ -80,10 +81,10 @@ ecbuild_add_executable( InterpolateFesom.h CONDITION - HAVE_ATLAS_IO AND HAVE_MULTIO_BUILD_TOOLS + HAVE_ECKIT_CODEC AND HAVE_MULTIO_BUILD_TOOLS LIBS multio - atlas_io eckit + eckit_codec ) diff --git a/src/multio/action/interpolate-fesom/FesomInterpolationWeights.h b/src/multio/action/interpolate-fesom/FesomInterpolationWeights.h index 37f8eb364..33728911d 100644 --- a/src/multio/action/interpolate-fesom/FesomInterpolationWeights.h +++ b/src/multio/action/interpolate-fesom/FesomInterpolationWeights.h @@ -7,7 +7,7 @@ #include #include "InterpolateFesom_debug.h" -#include "atlas_io/atlas-io.h" +#include "eckit/codec/codec.h" #include "eckit/exception/Exceptions.h" #include "multio/LibMultio.h" @@ -143,7 +143,7 @@ class FesomInterpolationWeights { level) << ".atlas"; - atlas::io::RecordWriter record; + eckit::codec::RecordWriter record; record.compression("none"); record.set("version", static_cast(0)); record.set("nside", static_cast(NSide)); @@ -154,10 +154,10 @@ class FesomInterpolationWeights { record.set("nOutRows", static_cast(nOutRows)); record.set("landSeaMask", - atlas::io::ArrayReference(landSeaMask.data(), std::vector{landSeaMask.size()})); - record.set("rowPtr", atlas::io::ArrayReference(rowStart.data(), std::vector{rowStart.size()})); - record.set("colIdx", atlas::io::ArrayReference(colIdx.data(), std::vector{colIdx.size()})); - record.set("weights", atlas::io::ArrayReference(values.data(), std::vector{values.size()})); + eckit::codec::ArrayReference(landSeaMask.data(), std::vector{landSeaMask.size()})); + record.set("rowPtr", eckit::codec::ArrayReference(rowStart.data(), std::vector{rowStart.size()})); + record.set("colIdx", eckit::codec::ArrayReference(colIdx.data(), std::vector{colIdx.size()})); + record.set("weights", eckit::codec::ArrayReference(values.data(), std::vector{values.size()})); record.write(os.str()); INTERPOLATE_FESOM_OUT_STREAM << " - FesomIntermopationWeights: exit dumpCache" diff --git a/src/multio/action/interpolate-fesom/InterpolateFesom.h b/src/multio/action/interpolate-fesom/InterpolateFesom.h index 4c1206306..d7f4aaaca 100644 --- a/src/multio/action/interpolate-fesom/InterpolateFesom.h +++ b/src/multio/action/interpolate-fesom/InterpolateFesom.h @@ -23,7 +23,7 @@ #include "FesomInterpolationWeights.h" #include "InterpolateFesom_debug.h" -#include "atlas_io/atlas-io.h" +#include "eckit/codec/codec.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" #include "multio/LibMultio.h" @@ -68,7 +68,7 @@ class Fesom2HEALPix { size_t version; size_t NSideR; size_t levelR; - atlas::io::RecordReader reader(file); + eckit::codec::RecordReader reader(file); // Read the objects needed for the interpolation reader.read("version", version); reader.wait(); diff --git a/src/multio/action/interpolate/CMakeLists.txt b/src/multio/action/interpolate/CMakeLists.txt index 8cb857f41..828b4b902 100644 --- a/src/multio/action/interpolate/CMakeLists.txt +++ b/src/multio/action/interpolate/CMakeLists.txt @@ -32,11 +32,11 @@ ecbuild_add_executable( ../../tools/MultioTool.cc CONDITION - HAVE_MIR AND HAVE_ATLAS_IO AND HAVE_MULTIO_BUILD_TOOLS + HAVE_MIR AND HAVE_ECKIT_CODEC AND HAVE_MULTIO_BUILD_TOOLS LIBS multio - atlas_io eckit + eckit_codec mir ) diff --git a/src/multio/action/renumber-healpix/CMakeLists.txt b/src/multio/action/renumber-healpix/CMakeLists.txt index 577e15f4e..500b2241a 100644 --- a/src/multio/action/renumber-healpix/CMakeLists.txt +++ b/src/multio/action/renumber-healpix/CMakeLists.txt @@ -12,13 +12,13 @@ ecbuild_add_library( PUBLIC_LIBS multio - atlas_io + eckit_codec ) # TODO: move this to the tools folder ecbuild_add_executable( TARGET multio-generate-healpix-cache SOURCES GenerateCache.cc ${PROJECT_SOURCE_DIR}/src/multio/tools/MultioTool.cc HEALPix.cc HEALPix.h - CONDITION HAVE_ATLAS_IO AND HAVE_MULTIO_BUILD_TOOLS - LIBS multio atlas_io eckit + CONDITION HAVE_ECKIT_CODEC AND HAVE_MULTIO_BUILD_TOOLS + LIBS multio eckit eckit_codec ) diff --git a/src/multio/action/renumber-healpix/GenerateCache.cc b/src/multio/action/renumber-healpix/GenerateCache.cc index a8caf7daa..3cb157cf7 100644 --- a/src/multio/action/renumber-healpix/GenerateCache.cc +++ b/src/multio/action/renumber-healpix/GenerateCache.cc @@ -33,7 +33,7 @@ #include "multio/ifsio/ifsio.h" #include "multio/tools/MultioTool.h" -#include "atlas_io/atlas-io.h" +#include "eckit/codec/codec.h" namespace multio::action { @@ -142,7 +142,7 @@ void CacheGenerator::init(const eckit::option::CmdArgs& args) { void CacheGenerator::execute(const eckit::option::CmdArgs& args) { size_t ref = 1; - atlas::io::RecordWriter record; + eckit::codec::RecordWriter record; record.compression("none"); // for ( size_t i=from_; i<=to_; ++i ){ for (size_t i = 0; i < list_.size(); ++i) { diff --git a/src/multio/action/renumber-healpix/HEALPix_ring2nest.cc b/src/multio/action/renumber-healpix/HEALPix_ring2nest.cc index c15db09eb..88b0a18b2 100644 --- a/src/multio/action/renumber-healpix/HEALPix_ring2nest.cc +++ b/src/multio/action/renumber-healpix/HEALPix_ring2nest.cc @@ -23,7 +23,7 @@ #include "eckit/filesystem/PathName.h" #include "eckit/log/Log.h" -#include "atlas_io/atlas-io.h" +#include "eckit/codec/codec.h" #include "multio/LibMultio.h" #include "multio/util/PrecisionTag.h" @@ -95,7 +95,7 @@ void checkMetadata(const message::Metadata& md) { std::vector makeMapping(size_t Nside, const std::string& cacheFileName) { std::vector map; - atlas::io::RecordReader reader(cacheFileName); + eckit::codec::RecordReader reader(cacheFileName); std::ostringstream os; os << "H" << std::setfill('0') << std::setw(8) << Nside << "_ring2nest"; reader.read(os.str(), map).wait(); diff --git a/src/multio/action/statistics/CMakeLists.txt b/src/multio/action/statistics/CMakeLists.txt index 805be209c..68a99c870 100644 --- a/src/multio/action/statistics/CMakeLists.txt +++ b/src/multio/action/statistics/CMakeLists.txt @@ -51,17 +51,15 @@ list( APPEND _statistics_libs multio ) -if(HAVE_ATLAS_IO) - -ecbuild_find_package( NAME atlas_io VERSION 0.33 REQUIRED ) +if(HAVE_ECKIT_CODEC) list( APPEND _statistics_sources - io/AtlasIO.cc - io/AtlasIO.h + io/EckitCodecIO.cc + io/EckitCodecIO.h ) list( APPEND _statistics_libs - atlas_io + eckit_codec ) endif() diff --git a/src/multio/action/statistics/io/AtlasIO.cc b/src/multio/action/statistics/io/EckitCodecIO.cc similarity index 67% rename from src/multio/action/statistics/io/AtlasIO.cc rename to src/multio/action/statistics/io/EckitCodecIO.cc index c7ece2650..00da1a281 100644 --- a/src/multio/action/statistics/io/AtlasIO.cc +++ b/src/multio/action/statistics/io/EckitCodecIO.cc @@ -1,12 +1,11 @@ -#include "AtlasIO.h" +#include "EckitCodecIO.h" #include #include #include -#include "atlas_io/atlas-io.h" - +#include "eckit/codec/codec.h" #include "eckit/exception/Exceptions.h" #include "eckit/filesystem/PathName.h" @@ -35,13 +34,13 @@ class SubVector { }; template -void interprete(const SubVector& in, atlas::io::ArrayReference& out) { - out = atlas::io::ArrayReference(in.data(), atlas::io::make_datatype(), atlas::io::ArrayShape{in.size()}); +void interprete(const SubVector& in, eckit::codec::ArrayReference& out) { + out = eckit::codec::ArrayReference(in.data(), eckit::codec::make_datatype(), eckit::codec::ArrayShape{in.size()}); } template -void decode(const atlas::io::Metadata& metadata, const atlas::io::Data& data, SubVector& out) { - atlas::io::ArrayMetadata array(metadata); +void decode(const eckit::codec::Metadata& metadata, const eckit::codec::Data& data, SubVector& out) { + eckit::codec::ArrayMetadata array(metadata); if (out.checkSize(array.shape(0))) { ::memcpy(out.data(), data, data.size()); } @@ -51,38 +50,38 @@ void decode(const atlas::io::Metadata& metadata, const atlas::io::Data& data, Su } } // namespace -AtlasIO::AtlasIO(const std::string& path, const std::string& prefix) : StatisticsIO{path, prefix, "atlasIO"} {}; +EckitCodecIO::EckitCodecIO(const std::string& path, const std::string& prefix) : StatisticsIO{path, prefix, "atlasIO"} {}; -void AtlasIO::write(const std::string& name, std::size_t fieldSize, std::size_t writeSize) { +void EckitCodecIO::write(const std::string& name, std::size_t fieldSize, std::size_t writeSize) { LOG_DEBUG_LIB(LibMultio) << " - The name of the window write file is :: " << generateCurrFileName(name) << std::endl; const std::string fname = generateCurrFileName(name); - atlas::io::RecordWriter record; + eckit::codec::RecordWriter record; SubVector dat{buffer_.data(), writeSize}; record.set("size", fieldSize, no_compression); - record.set(name, atlas::io::ref(dat), no_compression); + record.set(name, eckit::codec::ref(dat), no_compression); record.write(fname); return; }; -void AtlasIO::readSize(const std::string& name, std::size_t& readSize) { +void EckitCodecIO::readSize(const std::string& name, std::size_t& readSize) { LOG_DEBUG_LIB(LibMultio) << " - The name of the operation read file is :: " << generateCurrFileName(name) << std::endl; const std::string fname = generateCurrFileName(name); checkFileExist(fname); std::uint64_t sz; - atlas::io::RecordReader record(fname); + eckit::codec::RecordReader record(fname); record.read("size", sz).wait(); readSize = static_cast(sz); return; }; -void AtlasIO::read(const std::string& name, std::size_t readSize) { +void EckitCodecIO::read(const std::string& name, std::size_t readSize) { LOG_DEBUG_LIB(LibMultio) << " - The name of the operation read file is :: " << generateCurrFileName(name) << std::endl; const std::string fname = generateCurrFileName(name); checkFileExist(fname); - atlas::io::RecordReader record(fname); + eckit::codec::RecordReader record(fname); SubVector dat{buffer_.data(), readSize}; record.read(name, dat).wait(); if (!dat.good()) { @@ -93,13 +92,13 @@ void AtlasIO::read(const std::string& name, std::size_t readSize) { return; }; -void AtlasIO::flush() { +void EckitCodecIO::flush() { // TODO: Decide what to do when flush is called. Flush partial statistics when the Tag::Flush is received is // probably okay return; }; -void AtlasIO::checkFileExist(const std::string& name) const { +void EckitCodecIO::checkFileExist(const std::string& name) const { eckit::PathName file{name}; if (!file.exists()) { std::ostringstream os; @@ -109,6 +108,7 @@ void AtlasIO::checkFileExist(const std::string& name) const { return; }; -StatisticsIOBuilder AtalsIOBuilder("atlas_io"); +StatisticsIOBuilder EckitCodecIOBuilder("eckit_codec"); +StatisticsIOBuilder AtlasIOBuilder("atlas_io"); // Legacy name } // namespace multio::action diff --git a/src/multio/action/statistics/io/AtlasIO.h b/src/multio/action/statistics/io/EckitCodecIO.h similarity index 86% rename from src/multio/action/statistics/io/AtlasIO.h rename to src/multio/action/statistics/io/EckitCodecIO.h index bb4a23f18..5747908f9 100644 --- a/src/multio/action/statistics/io/AtlasIO.h +++ b/src/multio/action/statistics/io/EckitCodecIO.h @@ -17,9 +17,9 @@ static eckit::LocalConfiguration no_compression = [] { }(); -class AtlasIO final : public StatisticsIO { +class EckitCodecIO final : public StatisticsIO { public: - AtlasIO(const std::string& path, const std::string& prefix); + EckitCodecIO(const std::string& path, const std::string& prefix); void write(const std::string& name, std::size_t fieldSize, std::size_t writeSize) override; void readSize(const std::string& name, std::size_t& writeSize) override; void read(const std::string& name, std::size_t writeSize) override; diff --git a/src/multio/api/c/CMakeLists.txt b/src/multio/api/c/CMakeLists.txt index 16ef0dd64..d1d83379c 100644 --- a/src/multio/api/c/CMakeLists.txt +++ b/src/multio/api/c/CMakeLists.txt @@ -29,7 +29,7 @@ if( HAVE_MIR ) ) endif() -if ( HAVE_ATLAS_IO ) +if ( HAVE_ECKIT_CODEC ) list( APPEND multio_action_plugins multio-action-interpolate-fesom ) diff --git a/tests/multio/action/statistics/moving-mask/test-avg-moving-mask.yaml b/tests/multio/action/statistics/moving-mask/test-avg-moving-mask.yaml index 6d8d8492e..bc48f9c56 100644 --- a/tests/multio/action/statistics/moving-mask/test-avg-moving-mask.yaml +++ b/tests/multio/action/statistics/moving-mask/test-avg-moving-mask.yaml @@ -12,7 +12,7 @@ plans: read-restart: '{READ_RESTART}' write-restart: '{WRITE_RESTART}' debug-restart: true - restart-lib: "atlas_io" + restart-lib: "eckit_codec" restart-time: '{RESTART_ID}' step-frequency: 24 time-step: 3600 diff --git a/tests/multio/action/statistics/moving-mask/test-max-moving-mask.yaml b/tests/multio/action/statistics/moving-mask/test-max-moving-mask.yaml index fc3d2a4d8..91800681d 100644 --- a/tests/multio/action/statistics/moving-mask/test-max-moving-mask.yaml +++ b/tests/multio/action/statistics/moving-mask/test-max-moving-mask.yaml @@ -12,7 +12,7 @@ plans: read-restart: '{READ_RESTART}' write-restart: '{WRITE_RESTART}' debug-restart: true - restart-lib: "atlas_io" + restart-lib: "eckit_codec" restart-time: '{RESTART_ID}' step-frequency: 24 time-step: 3600 diff --git a/tests/multio/action/statistics/moving-mask/test-min-moving-mask.yaml b/tests/multio/action/statistics/moving-mask/test-min-moving-mask.yaml index a94af0b84..36c66c529 100644 --- a/tests/multio/action/statistics/moving-mask/test-min-moving-mask.yaml +++ b/tests/multio/action/statistics/moving-mask/test-min-moving-mask.yaml @@ -12,7 +12,7 @@ plans: read-restart: '{READ_RESTART}' write-restart: '{WRITE_RESTART}' debug-restart: true - restart-lib: "atlas_io" + restart-lib: "eckit_codec" restart-time: '{RESTART_ID}' step-frequency: 24 time-step: 3600 diff --git a/tests/multio/action/statistics/restart/CMakeLists.txt b/tests/multio/action/statistics/restart/CMakeLists.txt index 0d6223dd3..f38f1acf5 100644 --- a/tests/multio/action/statistics/restart/CMakeLists.txt +++ b/tests/multio/action/statistics/restart/CMakeLists.txt @@ -49,7 +49,7 @@ endif() ecbuild_add_test( TARGET ${PREFIX}_run_0-24_new_restart - CONDITION HAVE_ATLAS_IO + CONDITION HAVE_ECKIT_CODEC TEST_DEPENDS ${PREFIX}_get_data ENVIRONMENT READ_RESTART=${READ_RESTART} WRITE_RESTART=${WRITE_RESTART} RESTART_MAIN_FOLDER=${RESTART_MAIN_FOLDER} RESULT_FILE=${RESULT_FILE} RESTART_ID=${RESTART_ID} COMMAND multio-feed @@ -86,7 +86,7 @@ foreach(i RANGE 1 ${NUM_STEPS}) # Run the multio-feed command ecbuild_add_test( TARGET ${PREFIX}_run_${stepRange}_new_restart - CONDITION HAVE_ATLAS_IO + CONDITION HAVE_ECKIT_CODEC ENVIRONMENT READ_RESTART=${READ_RESTART} WRITE_RESTART=${WRITE_RESTART} RESTART_MAIN_FOLDER=${RESTART_MAIN_FOLDER} RESULT_FILE=${RESULT_FILE} RESTART_ID=${RESTART_ID} COMMAND multio-feed ARGS --stepRange=${stepRange} --decode --plans=${CMAKE_CURRENT_SOURCE_DIR}/test-restart.yaml standard_0_statistics_test_data.grib @@ -112,7 +112,7 @@ set(RESTART_ID "none") ecbuild_add_test( TARGET ${PREFIX}_run_single_chunk_new_restart - CONDITION HAVE_ATLAS_IO + CONDITION HAVE_ECKIT_CODEC TEST_DEPENDS ${PREFIX}_get_data ENVIRONMENT READ_RESTART=${READ_RESTART} WRITE_RESTART=${WRITE_RESTART} RESTART_MAIN_FOLDER=${RESTART_MAIN_FOLDER} RESULT_FILE=${RESULT_FILE} RESTART_ID=${RESTART_ID} COMMAND multio-feed diff --git a/tests/multio/action/statistics/restart/test-restart.yaml b/tests/multio/action/statistics/restart/test-restart.yaml index 3a3841d6a..3ededca83 100644 --- a/tests/multio/action/statistics/restart/test-restart.yaml +++ b/tests/multio/action/statistics/restart/test-restart.yaml @@ -12,7 +12,7 @@ plans: read-restart: '{READ_RESTART}' write-restart: '{WRITE_RESTART}' debug-restart: true - restart-lib: "atlas_io" + restart-lib: "eckit_codec" restart-time: '{RESTART_ID}' step-frequency: 1 time-step: 3600