Add meta information object

Author: franzpoeschel

include/openPMD/toolkit/Aws.hpp

@@ -11,11 +11,17 @@ struct ExternalBlockStorageAws : ExternalBlockStorageBackend
 private:
     Aws::S3::S3Client m_client;
     std::string m_bucketName;
+    std::optional<std::string> m_endpoint;
 
 public:
-    ExternalBlockStorageAws(Aws::S3::S3Client, std::string bucketName);
+    ExternalBlockStorageAws(
+        Aws::S3::S3Client,
+        std::string bucketName,
+        std::optional<std::string> endpoint);
     auto put(std::string const &identifier, void const *data, size_t len)
         -> std::string override;
+    [[nodiscard]] auto externalStorageLocation() const
+        -> nlohmann::json override;
     ~ExternalBlockStorageAws() override;
 };
 } // namespace openPMD::internal

include/openPMD/toolkit/ExternalBlockStorage.hpp

@@ -24,6 +24,9 @@ struct ExternalBlockStorageBackend
     virtual auto
     put(std::string const &identifier, void const *data, size_t len)
         -> std::string = 0;
+    [[nodiscard]] virtual auto externalStorageLocation() const
+        -> nlohmann::json = 0;
+
     virtual ~ExternalBlockStorageBackend();
 };
 } // namespace openPMD::internal
@@ -71,10 +74,7 @@ class ExternalBlockStorage
         std::optional<std::string> infix, // e.g. for distinguishing MPI ranks
         T const *data) -> std::string;
 
-    auto externalStorageLocation() const -> nlohmann::json
-    {
-        return "implement me";
-    }
+    [[nodiscard]] auto externalStorageLocation() const -> nlohmann::json;
 
     static void sanitizeString(std::string &s);
 };

include/openPMD/toolkit/Stdio.hpp

@@ -14,6 +14,8 @@ struct ExternalBlockStorageStdio : ExternalBlockStorageBackend
     ExternalBlockStorageStdio(std::string directory, std::string openMode);
     auto put(std::string const &identifier, void const *data, size_t len)
         -> std::string override;
+    [[nodiscard]] auto externalStorageLocation() const
+        -> nlohmann::json override;
     ~ExternalBlockStorageStdio() override;
 };
 } // namespace openPMD::internal

src/toolkit/Aws.cpp

@@ -30,8 +30,12 @@ struct imemstream : std::iostream
 namespace openPMD::internal
 {
 ExternalBlockStorageAws::ExternalBlockStorageAws(
-    Aws::S3::S3Client client, std::string bucketName)
-    : m_client{std::move(client)}, m_bucketName(std::move(bucketName))
+    Aws::S3::S3Client client,
+    std::string bucketName,
+    std::optional<std::string> endpoint)
+    : m_client{std::move(client)}
+    , m_bucketName(std::move(bucketName))
+    , m_endpoint(std::move(endpoint))
 {
     Aws::S3::Model::CreateBucketRequest create_request;
     create_request.SetBucket(m_bucketName);
@@ -77,4 +81,17 @@ auto ExternalBlockStorageAws::put(
     return sanitized;
 }
 
+[[nodiscard]] auto ExternalBlockStorageAws::externalStorageLocation() const
+    -> nlohmann::json
+{
+    nlohmann::json j;
+    j["provider"] = "s3";
+    if (m_endpoint.has_value())
+    {
+        j["endpoint"] = *m_endpoint;
+    }
+    j["bucket"] = m_bucketName;
+    return j;
+}
+
 } // namespace openPMD::internal

src/toolkit/AwsBuilder.cpp

@@ -117,7 +117,9 @@ AwsBuilder::operator ExternalBlockStorage()
         false);
 
     return ExternalBlockStorage{std::make_unique<ExternalBlockStorageAws>(
-        std::move(s3_client), std::move(m_bucketName))};
+        std::move(s3_client),
+        std::move(m_bucketName),
+        std::move(m_endpointOverride))};
 }
 
 auto AwsBuilder::build() -> ExternalBlockStorage

src/toolkit/ExternalBlockStorage.cpp

@@ -134,6 +134,12 @@ auto ExternalBlockStorage::store(
     return index_as_str;
 }
 
+[[nodiscard]] auto ExternalBlockStorage::externalStorageLocation() const
+    -> nlohmann::json
+{
+    return m_worker->externalStorageLocation();
+}
+
 void ExternalBlockStorage::sanitizeString(std::string &s)
 {
     for (char &c : s)

src/toolkit/Stdio.cpp

@@ -5,6 +5,39 @@
 #include <cstdio>
 #include <stdexcept>
 
+namespace
+{
+auto concat_filepath(std::string const &s1, std::string const &s2)
+    -> std::string
+{
+    if (s1.empty())
+    {
+        return s2;
+    }
+    if (s2.empty())
+    {
+        return s1;
+    }
+    bool ends_with_slash =
+        *s1.crbegin() == openPMD::auxiliary::directory_separator;
+    bool starts_with_slash =
+        *s2.cbegin() == openPMD::auxiliary::directory_separator;
+
+    if (ends_with_slash ^ starts_with_slash)
+    {
+        return s1 + s2;
+    }
+    else if (ends_with_slash && starts_with_slash)
+    {
+        return s1 + (s2.c_str() + 1);
+    }
+    else
+    {
+        return s1 + openPMD::auxiliary::directory_separator + s2;
+    }
+}
+} // namespace
+
 namespace openPMD::internal
 {
 ExternalBlockStorageStdio::ExternalBlockStorageStdio(
@@ -33,7 +66,7 @@ auto ExternalBlockStorageStdio::put(
 {
     auto sanitized = identifier + ".dat";
     ExternalBlockStorage::sanitizeString(sanitized);
-    std::string filepath = m_directory + "/" + sanitized;
+    std::string filepath = concat_filepath(m_directory, sanitized);
 
     if (len == 0)
     {
@@ -63,6 +96,16 @@ auto ExternalBlockStorageStdio::put(
             filepath);
     }
 
-    return filepath;
+    return sanitized;
+}
+
+[[nodiscard]] auto ExternalBlockStorageStdio::externalStorageLocation() const
+    -> nlohmann::json
+{
+    nlohmann::json j;
+    j["provider"] = "stdio";
+    j["directory"] = m_directory;
+    j["open_mode"] = m_openMode;
+    return j;
 }
 } // namespace openPMD::internal