Merge pull request #22 from rok4/develop

Release 1.1.2

Author: Dolite

.gitignore

@@ -1,2 +1,3 @@
 build
-dist
+dist
+.vscode

CHANGELOG.md

@@ -1,3 +1,17 @@
+## 1.1.2
+
+### [Changed]
+
+* Le test d'existence d'un objet ou d'un fichier n'est plus une lecture de 1 octet mais une implémentation spécifique à chaque type
+* Les TMS et les styles sont cherchés sur le stockage avec et sans extension JSON
+
+### [Fixed]
+
+* Table
+    * Correction d'une typo dans l'écriture du metadata.json : filedsCount -> fieldsCount
+* Style
+    * La valeur de nodata en sortie d'un style est la première valeur de la palette (et non la couleur pour la valeur 0)
+
 ## 1.1.1
 
 ### [Fixed]

include/rok4/storage/Context.h

@@ -170,12 +170,9 @@ class Context {
      * \~french \brief Précise si l'objet demandé existe dans ce contexte
      * \param[in] name Nom de l'objet dont on veut savoir l'existence 
      * \~english \brief Precise if provided object exists in this context
-     * \param[in] name Object's name whose existency is asked
+     * \param[in] name Object's name whose existence is asked
      */
-    bool exists(std::string name) {
-        uint8_t test;
-        return (read(&test, 0, 1, name) == 1);
-    }
+    virtual bool exists(std::string name) = 0;
 
     /**
      * \~french \brief Précise si le contexte est connecté

include/rok4/style/Style.h

@@ -285,7 +285,7 @@ private :
      */
     int getNodata (int** nodata) {
         if (palette && palette->getColoursMap() && ! palette->getColoursMap()->empty()) {
-            Colour c = palette->getColour(0);
+            Colour c = palette->getColoursMap()->begin()->second;
             if (palette->isNoAlpha()) {
                 (*nodata) = new int[3];
                 (*nodata)[0] = c.r;

include/rok4/utils/Cache.h

@@ -727,7 +727,7 @@ class TmsBook {
             d = directory;
         }
 
-        std::string tms_path = d + "/" + id + ".json";
+        std::string tms_path = d + "/" + id;
 
         TileMatrixSet* tms = new TileMatrixSet(tms_path);
         if ( ! tms->isOk() ) {
@@ -894,7 +894,7 @@ class StyleBook {
             d = directory;
         }
 
-        std::string style_path = d + "/" + id + ".json";
+        std::string style_path = d + "/" + id;
 
         Style* style = new Style(style_path, inspire);
         if ( ! style->isOk() ) {

include/rok4/utils/Configuration.h

@@ -110,11 +110,11 @@ class Configuration
 
             std::string id;
 
-            unsigned int idBegin=file.rfind ( "/" );
+            size_t idBegin=file.rfind ( "/" );
             if ( idBegin == std::string::npos ) {
                 idBegin=0;
             }
-            unsigned int idEnd=file.rfind ( extension );
+            size_t idEnd=file.rfind ( extension );
             if ( idEnd == std::string::npos ) {
                 idEnd=file.size();
             }

include/rok4/utils/Table.h

@@ -79,7 +79,7 @@ class Table
             res << ",\"geometry\":\"" << geometry << "\"";
             res << ",\"maxzoom\":\"" << max << "\"";
             res << ",\"minzoom\":\"" << min << "\"";
-            res << ",\"filedsCount\":\"" << attributes.size() << "\"";
+            res << ",\"fieldsCount\":\"" << attributes.size() << "\"";
             res << ",\"fields\":{";
             for (int i = 0; i < attributes.size(); i++) {
                 if (i != 0) {

src/storage/Context.cpp

@@ -80,39 +80,31 @@ eContextType fromString ( std::string strct ) {
 
 void split_path(std::string path, ContextType::eContextType& type, std::string& fo, std::string& tray) {
 
-    std::stringstream ss(path);
-    std::string token;
-    char delim = ':';
-    std::getline(ss, token, delim);
-    std::string storage_type = token;
-
-    
-    if (storage_type != path) {
-        // Un type de stockage a été précisé, on l'enlève pour passer à la suite
+    size_t pos = path.find ( "://" );
+    if ( pos == std::string::npos ) {
+        // Il n'y a pas de préfixe de stockage, on considère le mode fichier
+        type = ContextType::FILECONTEXT;
+    } else {
+        std::string storage_type = path.substr ( 0, pos );
         type = ContextType::fromString(storage_type);
         if (type == ContextType::UNKNOWN) {
             type = ContextType::FILECONTEXT;
         }
-        path.erase(0, storage_type.length() + 3);
-    } else {
-        type = ContextType::FILECONTEXT;
+        path = path.erase ( 0, storage_type.length() + 3);
     }
 
-
     if (type == ContextType::FILECONTEXT) {
         // Dans le cas d'un stockage fichier, le nom du fichier est l'ensemble et on ne définit pas de contenant
         fo = path;
+        tray = "";
         return;
     }
 
     // Dans le cas d'un stockage objet, on sépare le contenant du nom de l'objet
-    ss = std::stringstream(path);
-    delim = '/';
-    std::getline(ss, token, delim);
-    tray = token;
+    pos = path.find ( "/" );
+    tray = path.substr ( 0, pos );
+    fo = path.substr ( pos + 1, std::string::npos );
 
-    path.erase(0, tray.length() + 1);
-    fo = path;
     return;
 }
 

src/storage/FileContext.cpp

@@ -200,3 +200,8 @@ std::string FileContext::getPath(std::string name) {
     return root_dir + name;
 }
 
+bool FileContext::exists(std::string name) {
+    BOOST_LOG_TRIVIAL(debug) << "Exists (FILE) ? " << getPath(name);
+    struct stat buffer;   
+    return (stat (getPath(name).c_str(), &buffer) == 0);
+}

src/storage/FileContext.h

@@ -150,6 +150,8 @@ class FileContext : public Context {
 
     bool connection();
 
+    bool exists(std::string name);
+
     void closeConnection() {
         connected = false;
     }

src/storage/S3Context.cpp

@@ -438,7 +438,6 @@ bool S3Context::openToWrite(std::string name) {
 
 bool S3Context::closeToWrite(std::string name) {
 
-
     std::map<std::string, std::vector<char>*>::iterator it1 = writingBuffers.find ( name );
     if ( it1 == writingBuffers.end() ) {
         BOOST_LOG_TRIVIAL(error) << "The S3 writing buffer with name " << name << "does not exist, cannot flush it";
@@ -528,4 +527,78 @@ bool S3Context::closeToWrite(std::string name) {
     writingBuffers.erase(it1);
 
     return true;
+}
+
+bool S3Context::exists(std::string name) {
+
+    BOOST_LOG_TRIVIAL(debug) << "Exists (S3) ? " << getPath(name);
+
+    CURLcode res;
+    struct curl_slist *list = NULL;
+
+    CURL* curl = CurlPool::getCurlEnv();
+
+    std::string fullUrl = url + "/" + bucket_name + "/" + name;
+
+    time_t current;
+
+    time(&current);
+    struct tm * ptm = gmtime ( &current );
+
+    static char gmt_time[40];
+    sprintf(
+        gmt_time, "%s, %.2d %s %d %.2d:%.2d:%.2d GMT",
+        wday_name[ptm->tm_wday], ptm->tm_mday, mon_name[ptm->tm_mon], 1900 + ptm->tm_year,
+        ptm->tm_hour, ptm->tm_min, ptm->tm_sec
+    );
+
+    std::string content_type = "application/octet-stream";
+    std::string resource = "/" + bucket_name + "/" + name;
+    std::string stringToSign = "HEAD\n\n" + content_type + "\n" + std::string(gmt_time) + "\n" + resource;
+    std::string signature = getAuthorizationHeader(stringToSign);
+
+    // Constitution du header
+
+    char hd_host[256];
+    sprintf(hd_host, "Host: %s", host.c_str());
+    list = curl_slist_append(list, hd_host);
+
+    char d[100];
+    sprintf(d, "Date: %s", gmt_time);
+    list = curl_slist_append(list, d);
+
+    char ct[50];
+    sprintf(ct, "Content-Type: %s", content_type.c_str());
+    list = curl_slist_append(list, ct);
+
+    char auth[512];
+    sprintf(auth, "Authorization: AWS %s:%s", key.c_str(), signature.c_str());
+
+    list = curl_slist_append(list, auth);
+
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
+    curl_easy_setopt(curl, CURLOPT_URL, fullUrl.c_str());
+    curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "HEAD");
+    curl_easy_setopt(curl, CURLOPT_NOBODY, 1L);
+    if(ssl_no_verify){
+      curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
+    }
+
+    res = curl_easy_perform(curl);
+    
+    curl_slist_free_all(list);
+
+    if( CURLE_OK != res) {
+        BOOST_LOG_TRIVIAL(error) << "Cannot test object existence from S3 : " << bucket_name + "/" + name;
+        BOOST_LOG_TRIVIAL(error) << curl_easy_strerror(res);
+        return false;
+    }
+
+    long http_code = 0;
+    curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code);
+    if (http_code >= 200 && http_code <= 299) {
+        return true;
+    } else {
+        return false;
+    }
 }

src/storage/S3Context.h

@@ -185,6 +185,8 @@ class S3Context : public Context {
      * \~english \brief Get public URL #public_url and constitute the HTTP header #authHdr
      */
     bool connection();
+    
+    bool exists(std::string name);
 
     void closeConnection() {
         connected = false;

src/storage/SwiftContext.cpp

@@ -617,3 +617,49 @@ std::string SwiftContext::getPath(std::string racine,int x,int y,int pathDepth){
 std::string SwiftContext::getPath(std::string name) {  
     return container_name + "/" + name;
 }
+
+
+bool SwiftContext::exists(std::string name) {
+
+    BOOST_LOG_TRIVIAL(debug) << "Exists (SWIFT) ? " << getPath(name);
+
+    if (! connected) {
+        BOOST_LOG_TRIVIAL(error) << "Try to test object existence using the unconnected swift context " << container_name;
+        return false;
+    }
+        
+    CURLcode res;
+    struct curl_slist *list = NULL;
+    CURL* curl = CurlPool::getCurlEnv();
+
+    std::string fullUrl;
+    fullUrl = public_url + "/" + container_name + "/" + name;
+
+    list = curl_slist_append(list, token.c_str());
+
+    curl_easy_setopt(curl, CURLOPT_HTTPHEADER, list);
+    curl_easy_setopt(curl, CURLOPT_URL, fullUrl.c_str());
+    curl_easy_setopt(curl, CURLOPT_CUSTOMREQUEST, "HEAD");
+    if(ssl_no_verify){
+        curl_easy_setopt(curl, CURLOPT_SSL_VERIFYPEER, 0L);
+    }
+
+    res = curl_easy_perform(curl);
+    
+    curl_slist_free_all(list);
+
+    if( CURLE_OK != res) {
+        BOOST_LOG_TRIVIAL(error) << "Cannot test object existence from SWIFT : " << container_name + "/" + name;
+        BOOST_LOG_TRIVIAL(error) << curl_easy_strerror(res);
+        return false;
+    }
+
+    long http_code = 0;
+    curl_easy_getinfo (curl, CURLINFO_RESPONSE_CODE, &http_code);
+    if (http_code >= 200 && http_code <= 299) {
+        return true;
+    } else {
+        return false;
+    }
+
+}

src/storage/SwiftContext.h

@@ -244,6 +244,8 @@ class SwiftContext : public Context{
      */
     bool connection();
 
+    bool exists(std::string name);
+
     /**
      * \~french \brief Returne le jeton d'authentification #token
      * \~english \brief Returns authentification token #token

src/storage/ceph/CephPoolContext.cpp

@@ -329,4 +329,23 @@ std::string CephPoolContext::getPath(std::string racine,int x,int y,int pathDept
 
 std::string CephPoolContext::getPath(std::string name) {  
     return pool_name + "/" + name;
+}
+
+bool CephPoolContext::exists(std::string name) {
+
+    BOOST_LOG_TRIVIAL(debug) << "Exists (CEPH) ? " << getPath(name);
+
+    if (! connected) {
+        BOOST_LOG_TRIVIAL(error) << "Try to test object existence using the unconnected ceph pool context " << pool_name;
+        return false;
+    }
+
+    uint64_t fullSize;
+    time_t time;
+    int ret = rados_stat(io_ctx, name.c_str(), &fullSize, &time);
+    if (ret < 0) {
+        return false;
+    } else {
+        return true;
+    }
 }

src/storage/ceph/CephPoolContext.h

@@ -139,6 +139,8 @@ class CephPoolContext : public Context {
      */
     bool connection();
 
+    bool exists(std::string name);
+
     /**
      * \~french \brief Nettoie les objets librados
      * \~english \brief Clean librados objects

src/style/Palette.cpp

@@ -318,3 +318,4 @@ Colour Palette::getColour ( double index ) {
     return tmp;
 }
 
+