Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/short lived fdb #43

Merged
merged 2 commits into from
Mar 17, 2025
Merged

Feature/short lived fdb #43

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
8650bcb
Switch to short lived fdb objects
ChrisspyB Mar 4, 2025
57d41a6
Remove FDBLister mutex
ChrisspyB Mar 4, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 9 additions & 47 deletions src/gribjump/Lister.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,9 +12,7 @@

#include "eckit/config/Resource.h"
#include "eckit/log/Log.h"
#include "eckit/thread/AutoLock.h"

#include "gribjump/GribJump.h"
#include "gribjump/Lister.h"
#include "gribjump/GribJumpException.h"
#include "gribjump/URIHelper.h"
Expand Down Expand Up @@ -45,13 +43,12 @@ FDBLister::~FDBLister() {

std::vector<eckit::URI> FDBLister::list(const std::vector<metkit::mars::MarsRequest> requests) {

eckit::AutoLock<FDBLister> lock(this);

std::vector<eckit::URI> uris;
fdb5::FDB fdb;
for (auto& request : requests) {

fdb5::FDBToolRequest fdbreq(request);
auto listIter = fdb_.list(fdbreq, true);
auto listIter = fdb.list(fdbreq, true);

fdb5::ListElement elem;
while (listIter.next(elem)) {
Expand Down Expand Up @@ -84,11 +81,12 @@ std::string fdbkeyToStr(const fdb5::Key& key) {

// i.e. do all of the listing work I want...
filemap_t FDBLister::fileMap(const metkit::mars::MarsRequest& unionRequest, const ExItemMap& reqToExtractionItem) {
eckit::AutoLock<FDBLister> lock(this);
filemap_t filemap;

fdb5::FDBToolRequest fdbreq(unionRequest);
auto listIter = fdb_.list(fdbreq, true);

fdb5::FDB fdb;
auto listIter = fdb.list(fdbreq, true);

size_t fdb_count = 0;
size_t count = 0;
Expand Down Expand Up @@ -169,11 +167,11 @@ std::map< eckit::PathName, eckit::OffsetList > FDBLister::filesOffsets(const std
}

std::vector<eckit::URI> FDBLister::URIs(const std::vector<metkit::mars::MarsRequest>& requests) {
eckit::AutoLock<FDBLister> lock(this);
std::vector<eckit::URI> uris;
fdb5::FDB fdb;
for (auto& request : requests) {
fdb5::FDBToolRequest fdbreq(request);
auto listIter = fdb_.list(fdbreq, true);
auto listIter = fdb.list(fdbreq, true);
fdb5::ListElement elem;
while (listIter.next(elem)) {
uris.push_back(elem.location().fullUri());
Expand All @@ -190,12 +188,12 @@ std::map<std::string, std::unordered_set<std::string> > FDBLister::axes(const st
}

std::map<std::string, std::unordered_set<std::string> > FDBLister::axes(const fdb5::FDBToolRequest& request, int level) {
eckit::AutoLock<FDBLister> lock(this);
std::map<std::string, std::unordered_set<std::string>> values;

LOG_DEBUG_LIB(LibGribJump) << "Using FDB's (new) axes impl" << std::endl;

fdb5::IndexAxis ax = fdb_.axes(request, level);
fdb5::FDB fdb;
fdb5::IndexAxis ax = fdb.axes(request, level);
ax.sort();
std::map<std::string, eckit::DenseSet<std::string>> fdbValues = ax.map();

Expand All @@ -217,41 +215,5 @@ std::map<std::string, std::unordered_set<std::string> > FDBLister::axes(const fd

// ------------------------------------------------------------------

#if 0
// map (file : offsets)
std::map< eckit::PathName, eckit::OffsetList > FDBLister::list(const std::vector<metkit::mars::MarsRequest> requests) {

eckit::AutoLock<FDBLister> lock(this);

std::map< eckit::PathName, eckit::OffsetList > files;

for (auto& request : requests) {

size_t count = request.count(); // worse case scenario all fields in one file

fdb5::FDBToolRequest fdbreq(request);
auto listIter = fdb_.list(fdbreq, true);

fdb5::ListElement elem;
while (listIter.next(elem)) {

const fdb5::FieldLocation& loc = elem.location();

eckit::PathName path = loc.uri().path();

auto it = files.find(path);
if(it == files.end()) {
eckit::OffsetList offsets;
offsets.reserve(count);
offsets.push_back(loc.offset());
files.emplace(path, offsets);
}
else {
it->second.push_back(loc.offset());
}
}
}
}
#endif

} // namespace gribjump
7 changes: 0 additions & 7 deletions src/gribjump/Lister.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -34,17 +34,11 @@ class Lister {
virtual std::vector<eckit::URI> list(const std::vector<metkit::mars::MarsRequest> requests) = 0; // <-- May not want to use mars request?
virtual std::map<std::string, std::unordered_set<std::string> > axes(const std::string& request, int level) = 0 ;

void lock() { mutex_.lock(); locked_ = true; }
void unlock() { mutex_.unlock(); locked_ = false; }

protected:

Lister();
~Lister();

private:
std::mutex mutex_;
bool locked_ = false;
};

// ------------------------------------------------------------------
Expand All @@ -70,7 +64,6 @@ class FDBLister : public Lister {
~FDBLister();

private:
fdb5::FDB fdb_;
bool allowMissing_;
};

Expand Down
8 changes: 6 additions & 2 deletions src/gribjump/remote/GribJumpUser.cc
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,15 @@ void GribJumpUser::serve(eckit::Stream& s, std::istream& in, std::ostream& out)
try {
s << e;
}
catch (std::exception& a) {
eckit::Log::error() << "** " << a.what() << " Caught in " << Here() << std::endl;
catch (...) {
eckit::Log::error() << "** Exception is ignored" << std::endl;
}
}
catch (...) {
eckit::Log::error() << "** Unknown exception caught in " << Here() << std::endl;
eckit::Log::error() << "** Exception is ignored" << std::endl;
MetricsManager::instance().set("error", "Uncaught exception");
}

LOG_DEBUG_LIB(LibGribJump) << eckit::system::ResourceUsage() << std::endl;

Expand Down
Loading