Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Remove dependency on atlasIO #64

Merged
merged 1 commit into from
Mar 24, 2025
Merged

Remove dependency on atlasIO #64

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 10 additions & 5 deletions CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -20,7 +20,7 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON)
### dependencies

ecbuild_find_package( NAME eccodes VERSION 2.31.2 REQUIRED )
ecbuild_find_package( NAME eckit VERSION 1.24 REQUIRED )
ecbuild_find_package( NAME eckit VERSION 1.25 REQUIRED )
ecbuild_find_package( NAME metkit VERSION 1.10 REQUIRED )
ecbuild_find_package( NAME atlas VERSION 0.35 REQUIRED )

Expand All @@ -35,12 +35,17 @@ ecbuild_add_option( FEATURE FDB5
DESCRIPTION "Send data to FDB5"
REQUIRED_PACKAGES "NAME fdb5 VERSION 5.11.20" )

### atlas-io plugin
### eckit-codec plugin

ecbuild_add_option( FEATURE ATLAS_IO
set( eckit_HAVE_ECKIT_CODEC 0 )
if( TARGET eckit_codec )
set( eckit_HAVE_ECKIT_CODEC 1 )
endif()

ecbuild_add_option( FEATURE ECKIT_CODEC
DEFAULT ON
DESCRIPTION "Use atlasIO for the restart"
REQUIRED_PACKAGES "NAME atlas_io VERSION 0.33" )
DESCRIPTION "Use eckit codec for IO"
CONDITION eckit_HAVE_ECKIT_CODEC )

### interpolate action

Expand Down
17 changes: 9 additions & 8 deletions src/multio/action/interpolate-fesom/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,12 @@ ecbuild_add_library(
${ECKIT_INCLUDE_DIRS}

CONDITION
HAVE_ATLAS_IO
HAVE_ECKIT_CODEC

PUBLIC_LIBS
multio
atlas_io
eckit
eckit_codec
)


Expand All @@ -38,12 +39,12 @@ ecbuild_add_executable(
InterpolateFesom.h

CONDITION
HAVE_ATLAS_IO AND HAVE_MULTIO_BUILD_TOOLS
HAVE_ECKIT_CODEC AND HAVE_MULTIO_BUILD_TOOLS

LIBS
multio
atlas_io
eckit
eckit_codec
)

ecbuild_add_executable(
Expand All @@ -59,12 +60,12 @@ ecbuild_add_executable(
InterpolateFesom.h

CONDITION
HAVE_ATLAS_IO AND HAVE_MULTIO_BUILD_TOOLS
HAVE_ECKIT_CODEC AND HAVE_MULTIO_BUILD_TOOLS

LIBS
multio
atlas_io
eckit
eckit_codec
)

ecbuild_add_executable(
Expand All @@ -80,10 +81,10 @@ ecbuild_add_executable(
InterpolateFesom.h

CONDITION
HAVE_ATLAS_IO AND HAVE_MULTIO_BUILD_TOOLS
HAVE_ECKIT_CODEC AND HAVE_MULTIO_BUILD_TOOLS

LIBS
multio
atlas_io
eckit
eckit_codec
)
12 changes: 6 additions & 6 deletions src/multio/action/interpolate-fesom/FesomInterpolationWeights.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
#include <vector>

#include "InterpolateFesom_debug.h"
#include "atlas_io/atlas-io.h"
#include "eckit/codec/codec.h"
#include "eckit/exception/Exceptions.h"
#include "multio/LibMultio.h"

Expand Down Expand Up @@ -143,7 +143,7 @@ class FesomInterpolationWeights {
level)
<< ".atlas";

atlas::io::RecordWriter record;
eckit::codec::RecordWriter record;
record.compression("none");
record.set("version", static_cast<size_t>(0));
record.set("nside", static_cast<size_t>(NSide));
Expand All @@ -154,10 +154,10 @@ class FesomInterpolationWeights {
record.set("nOutRows", static_cast<size_t>(nOutRows));

record.set("landSeaMask",
atlas::io::ArrayReference(landSeaMask.data(), std::vector<size_t>{landSeaMask.size()}));
record.set("rowPtr", atlas::io::ArrayReference(rowStart.data(), std::vector<size_t>{rowStart.size()}));
record.set("colIdx", atlas::io::ArrayReference(colIdx.data(), std::vector<size_t>{colIdx.size()}));
record.set("weights", atlas::io::ArrayReference(values.data(), std::vector<size_t>{values.size()}));
eckit::codec::ArrayReference(landSeaMask.data(), std::vector<size_t>{landSeaMask.size()}));
record.set("rowPtr", eckit::codec::ArrayReference(rowStart.data(), std::vector<size_t>{rowStart.size()}));
record.set("colIdx", eckit::codec::ArrayReference(colIdx.data(), std::vector<size_t>{colIdx.size()}));
record.set("weights", eckit::codec::ArrayReference(values.data(), std::vector<size_t>{values.size()}));
record.write(os.str());

INTERPOLATE_FESOM_OUT_STREAM << " - FesomIntermopationWeights: exit dumpCache"
Expand Down
4 changes: 2 additions & 2 deletions src/multio/action/interpolate-fesom/InterpolateFesom.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@

#include "FesomInterpolationWeights.h"
#include "InterpolateFesom_debug.h"
#include "atlas_io/atlas-io.h"
#include "eckit/codec/codec.h"
#include "eckit/exception/Exceptions.h"
#include "eckit/filesystem/PathName.h"
#include "multio/LibMultio.h"
Expand Down Expand Up @@ -68,7 +68,7 @@ class Fesom2HEALPix {
size_t version;
size_t NSideR;
size_t levelR;
atlas::io::RecordReader reader(file);
eckit::codec::RecordReader reader(file);
// Read the objects needed for the interpolation
reader.read("version", version);
reader.wait();
Expand Down
4 changes: 2 additions & 2 deletions src/multio/action/interpolate/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,11 +32,11 @@ ecbuild_add_executable(
../../tools/MultioTool.cc

CONDITION
HAVE_MIR AND HAVE_ATLAS_IO AND HAVE_MULTIO_BUILD_TOOLS
HAVE_MIR AND HAVE_ECKIT_CODEC AND HAVE_MULTIO_BUILD_TOOLS

LIBS
multio
atlas_io
eckit
eckit_codec
mir
)
6 changes: 3 additions & 3 deletions src/multio/action/renumber-healpix/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,13 +12,13 @@ ecbuild_add_library(

PUBLIC_LIBS
multio
atlas_io
eckit_codec
)

# TODO: move this to the tools folder
ecbuild_add_executable(
TARGET multio-generate-healpix-cache
SOURCES GenerateCache.cc ${PROJECT_SOURCE_DIR}/src/multio/tools/MultioTool.cc HEALPix.cc HEALPix.h
CONDITION HAVE_ATLAS_IO AND HAVE_MULTIO_BUILD_TOOLS
LIBS multio atlas_io eckit
CONDITION HAVE_ECKIT_CODEC AND HAVE_MULTIO_BUILD_TOOLS
LIBS multio eckit eckit_codec
)
4 changes: 2 additions & 2 deletions src/multio/action/renumber-healpix/GenerateCache.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@
#include "multio/ifsio/ifsio.h"
#include "multio/tools/MultioTool.h"

#include "atlas_io/atlas-io.h"
#include "eckit/codec/codec.h"

namespace multio::action {

Expand Down Expand Up @@ -142,7 +142,7 @@ void CacheGenerator::init(const eckit::option::CmdArgs& args) {

void CacheGenerator::execute(const eckit::option::CmdArgs& args) {
size_t ref = 1;
atlas::io::RecordWriter record;
eckit::codec::RecordWriter record;
record.compression("none");
// for ( size_t i=from_; i<=to_; ++i ){
for (size_t i = 0; i < list_.size(); ++i) {
Expand Down
4 changes: 2 additions & 2 deletions src/multio/action/renumber-healpix/HEALPix_ring2nest.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@
#include "eckit/filesystem/PathName.h"
#include "eckit/log/Log.h"

#include "atlas_io/atlas-io.h"
#include "eckit/codec/codec.h"

#include "multio/LibMultio.h"
#include "multio/util/PrecisionTag.h"
Expand Down Expand Up @@ -95,7 +95,7 @@ void checkMetadata(const message::Metadata& md) {

std::vector<size_t> makeMapping(size_t Nside, const std::string& cacheFileName) {
std::vector<size_t> map;
atlas::io::RecordReader reader(cacheFileName);
eckit::codec::RecordReader reader(cacheFileName);
std::ostringstream os;
os << "H" << std::setfill('0') << std::setw(8) << Nside << "_ring2nest";
reader.read(os.str(), map).wait();
Expand Down
10 changes: 4 additions & 6 deletions src/multio/action/statistics/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -51,17 +51,15 @@ list( APPEND _statistics_libs
multio
)

if(HAVE_ATLAS_IO)

ecbuild_find_package( NAME atlas_io VERSION 0.33 REQUIRED )
if(HAVE_ECKIT_CODEC)

list( APPEND _statistics_sources
io/AtlasIO.cc
io/AtlasIO.h
io/EckitCodecIO.cc
io/EckitCodecIO.h
)

list( APPEND _statistics_libs
atlas_io
eckit_codec
)

endif()
Expand Down
36 changes: 18 additions & 18 deletions src/multio/action/statistics/io/AtlasIO.cc → ...ltio/action/statistics/io/EckitCodecIO.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,12 +1,11 @@

#include "AtlasIO.h"
#include "EckitCodecIO.h"

#include <cstdio>
#include <cstring>
#include <iomanip>

#include "atlas_io/atlas-io.h"

#include "eckit/codec/codec.h"
#include "eckit/exception/Exceptions.h"
#include "eckit/filesystem/PathName.h"

Expand Down Expand Up @@ -35,13 +34,13 @@ class SubVector {
};

template <typename T>
void interprete(const SubVector<T>& in, atlas::io::ArrayReference& out) {
out = atlas::io::ArrayReference(in.data(), atlas::io::make_datatype<T>(), atlas::io::ArrayShape{in.size()});
void interprete(const SubVector<T>& in, eckit::codec::ArrayReference& out) {
out = eckit::codec::ArrayReference(in.data(), eckit::codec::make_datatype<T>(), eckit::codec::ArrayShape{in.size()});
}

template <typename T>
void decode(const atlas::io::Metadata& metadata, const atlas::io::Data& data, SubVector<T>& out) {
atlas::io::ArrayMetadata array(metadata);
void decode(const eckit::codec::Metadata& metadata, const eckit::codec::Data& data, SubVector<T>& out) {
eckit::codec::ArrayMetadata array(metadata);
if (out.checkSize(array.shape(0))) {
::memcpy(out.data(), data, data.size());
}
Expand All @@ -51,38 +50,38 @@ void decode(const atlas::io::Metadata& metadata, const atlas::io::Data& data, Su
}
} // namespace

AtlasIO::AtlasIO(const std::string& path, const std::string& prefix) : StatisticsIO{path, prefix, "atlasIO"} {};
EckitCodecIO::EckitCodecIO(const std::string& path, const std::string& prefix) : StatisticsIO{path, prefix, "atlasIO"} {};

void AtlasIO::write(const std::string& name, std::size_t fieldSize, std::size_t writeSize) {
void EckitCodecIO::write(const std::string& name, std::size_t fieldSize, std::size_t writeSize) {
LOG_DEBUG_LIB(LibMultio) << " - The name of the window write file is :: " << generateCurrFileName(name)
<< std::endl;
const std::string fname = generateCurrFileName(name);
atlas::io::RecordWriter record;
eckit::codec::RecordWriter record;
SubVector<std::uint64_t> dat{buffer_.data(), writeSize};
record.set("size", fieldSize, no_compression);
record.set(name, atlas::io::ref(dat), no_compression);
record.set(name, eckit::codec::ref(dat), no_compression);
record.write(fname);
return;
};

void AtlasIO::readSize(const std::string& name, std::size_t& readSize) {
void EckitCodecIO::readSize(const std::string& name, std::size_t& readSize) {
LOG_DEBUG_LIB(LibMultio) << " - The name of the operation read file is :: " << generateCurrFileName(name)
<< std::endl;
const std::string fname = generateCurrFileName(name);
checkFileExist(fname);
std::uint64_t sz;
atlas::io::RecordReader record(fname);
eckit::codec::RecordReader record(fname);
record.read("size", sz).wait();
readSize = static_cast<std::size_t>(sz);
return;
};

void AtlasIO::read(const std::string& name, std::size_t readSize) {
void EckitCodecIO::read(const std::string& name, std::size_t readSize) {
LOG_DEBUG_LIB(LibMultio) << " - The name of the operation read file is :: " << generateCurrFileName(name)
<< std::endl;
const std::string fname = generateCurrFileName(name);
checkFileExist(fname);
atlas::io::RecordReader record(fname);
eckit::codec::RecordReader record(fname);
SubVector<std::uint64_t> dat{buffer_.data(), readSize};
record.read(name, dat).wait();
if (!dat.good()) {
Expand All @@ -93,13 +92,13 @@ void AtlasIO::read(const std::string& name, std::size_t readSize) {
return;
};

void AtlasIO::flush() {
void EckitCodecIO::flush() {
// TODO: Decide what to do when flush is called. Flush partial statistics when the Tag::Flush is received is
// probably okay
return;
};

void AtlasIO::checkFileExist(const std::string& name) const {
void EckitCodecIO::checkFileExist(const std::string& name) const {
eckit::PathName file{name};
if (!file.exists()) {
std::ostringstream os;
Expand All @@ -109,6 +108,7 @@ void AtlasIO::checkFileExist(const std::string& name) const {
return;
};

StatisticsIOBuilder<AtlasIO> AtalsIOBuilder("atlas_io");
StatisticsIOBuilder<EckitCodecIO> EckitCodecIOBuilder("eckit_codec");
StatisticsIOBuilder<EckitCodecIO> AtlasIOBuilder("atlas_io"); // Legacy name
Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This is a fix to make sure plans with restart-lib: "atlas_io" do not break for now.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we really want to have this legacy feature??

We tag the branches anyway. So for me it would be fine that projects that upgrade will have to update their configuration properly. Someday this has to be done anyway.

Copy link
Collaborator

@dsarmany dsarmany Mar 21, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@tweska -- Check with Steffi if there are any multio plans currently that use atlas_io. In general, I'm also in favour of getting rid of this.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

already talked to Kevin. Yes currently plans use atlas_io. But we are rewriting them anyway so we might as well get rid of it at that point.


} // namespace multio::action
4 changes: 2 additions & 2 deletions src/multio/action/statistics/io/AtlasIO.h → ...ultio/action/statistics/io/EckitCodecIO.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,9 +17,9 @@ static eckit::LocalConfiguration no_compression = [] {
}();


class AtlasIO final : public StatisticsIO {
class EckitCodecIO final : public StatisticsIO {
public:
AtlasIO(const std::string& path, const std::string& prefix);
EckitCodecIO(const std::string& path, const std::string& prefix);
void write(const std::string& name, std::size_t fieldSize, std::size_t writeSize) override;
void readSize(const std::string& name, std::size_t& writeSize) override;
void read(const std::string& name, std::size_t writeSize) override;
Expand Down
2 changes: 1 addition & 1 deletion src/multio/api/c/CMakeLists.txt
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ if( HAVE_MIR )
)
endif()

if ( HAVE_ATLAS_IO )
if ( HAVE_ECKIT_CODEC )
list( APPEND multio_action_plugins
multio-action-interpolate-fesom
)
Expand Down
2 changes: 1 addition & 1 deletion tests/multio/action/statistics/moving-mask/test-avg-moving-mask.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ plans:
read-restart: '{READ_RESTART}'
write-restart: '{WRITE_RESTART}'
debug-restart: true
restart-lib: "atlas_io"
restart-lib: "eckit_codec"
restart-time: '{RESTART_ID}'
step-frequency: 24
time-step: 3600
Expand Down
2 changes: 1 addition & 1 deletion tests/multio/action/statistics/moving-mask/test-max-moving-mask.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ plans:
read-restart: '{READ_RESTART}'
write-restart: '{WRITE_RESTART}'
debug-restart: true
restart-lib: "atlas_io"
restart-lib: "eckit_codec"
restart-time: '{RESTART_ID}'
step-frequency: 24
time-step: 3600
Expand Down
2 changes: 1 addition & 1 deletion tests/multio/action/statistics/moving-mask/test-min-moving-mask.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ plans:
read-restart: '{READ_RESTART}'
write-restart: '{WRITE_RESTART}'
debug-restart: true
restart-lib: "atlas_io"
restart-lib: "eckit_codec"
restart-time: '{RESTART_ID}'
step-frequency: 24
time-step: 3600
Expand Down
Loading
Loading