[IO] Add Redis Connector

CMakeLists.txt

@@ -65,13 +65,27 @@ if(MONGOCLIENT_FOUND)
     list(APPEND HUSKY_EXTERNAL_DEFINITION ${MONGOCLIENT_DEFINITION})
 endif(MONGOCLIENT_FOUND)
 
+# Redis 
+if(REDISCLIENT_FOUND)
+    list(APPEND HUSKY_EXTERNAL_INCLUDE ${REDISCLIENT_INCLUDE_DIR})
+    list(APPEND HUSKY_EXTERNAL_LIB ${REDISCLIENT_LIBRARY})
+    list(APPEND HUSKY_EXTERNAL_DEFINITION ${REDISCLIENT_DEFINITION})
+endif(REDISCLIENT_FOUND)
+
 # Thrift
 if(THRIFT_FOUND)
     list(APPEND HUSKY_EXTERNAL_INCLUDE ${THRIFT_INCLUDE_DIR})
     list(APPEND HUSKY_EXTERNAL_LIB ${THRIFT_LIBRARY})
     list(APPEND HUSKY_EXTERNAL_DEFINITION ${THRIFT_DEFINITION})
 endif(THRIFT_FOUND)
 
+# ORC
+if(ORC_FOUND)
+    list(APPEND HUSKY_EXTERNAL_INCLUDE ${ORC_INCLUDE_DIR})
+    list(APPEND HUSKY_EXTERNAL_LIB ${ORC_LIBRARY})
+    list(APPEND HUSKY_EXTERNAL_DEFINITION ${ORC_DEFINITION})
+endif(ORC_FOUND)
+
 if(WIN32)
     list(APPEND HUSKY_EXTERNAL_LIB wsock32 ws2_32)
 endif()

cmake/dep.cmake

@@ -99,6 +99,65 @@ if(WITHOUT_MONGODB)
     unset(MONGOCLIENT_FOUND)
     message(STATUS "Not using MongoClient due to WITHOUT_MONGODB option")
 endif(WITHOUT_MONGODB)
+
+### Redis ###
+
+if(REDISCLIENT_SEARCH_PATH)
+    find_path(REDISCLIENT_INCLUDE_DIR NAMES hiredis PATHS ${REDISCLIENT_SEARCH_PATH})
+    find_library(REDISCLIENT_LIBRARY NAMES hiredis PATHS ${REDISCLIENT_SEARCH_PATH})
+else(REDISCLIENT_SEARCH_PATH)
+    find_path(REDISCLIENT_INCLUDE_DIR NAMES hiredis)
+    find_library(REDISCLIENT_LIBRARY NAMES hiredis)
+endif(REDISCLIENT_SEARCH_PATH)
+if(REDISCLIENT_INCLUDE_DIR AND REDISCLIENT_LIBRARY)
+    set(REDISCLIENT_FOUND true)
+endif(REDISCLIENT_INCLUDE_DIR AND REDISCLIENT_LIBRARY)
+if(REDISCLIENT_FOUND)
+    set(REDISCLIENT_DEFINITION "-DWITH_REDIS")
+    message (STATUS "Found Hiredis:")
+    message (STATUS "  (Headers)       ${REDISCLIENT_INCLUDE_DIR}")
+    message (STATUS "  (Library)       ${REDISCLIENT_LIBRARY}")
+    message (STATUS "  (Definition)    ${REDISCLIENT_DEFINITION}")
+else(REDISCLIENT_FOUND)
+    if(WIN32)
+        message (STATUS "Redis and hiredis are currently not available on win32")
+    else(WIN32)
+        message (STATUS "hiredis will be included as a third party:")
+        include(ExternalProject)
+        set(THIRDPARTY_DIR ${PROJECT_SOURCE_DIR}/third_party)
+        if(NOT REDISCLIENT_INCLUDE_DIR OR NOT REDISCLIENT_LIBRARY)
+            set(REDIS_INSTALL "cp")
+            ExternalProject_Add(
+                hiredis
+                GIT_REPOSITORY "https://github.com/redis/hiredis"
+                GIT_TAG "v0.13.3"
+                PREFIX ${THIRDPARTY_DIR}
+                UPDATE_COMMAND ""
+                CONFIGURE_COMMAND ""
+                # TODO: if remove "-pedantic" strict Warnings.
+                BUILD_COMMAND sed -i "s/ -pedantic//g" ${THIRDPARTY_DIR}/src/hiredis/Makefile COMMAND make dynamic COMMAND make static
+                BUILD_IN_SOURCE 1
+                INSTALL_COMMAND mkdir -p ${PROJECT_BINARY_DIR}/include/hiredis/adapters ${PROJECT_BINARY_DIR}/lib COMMAND ${REDIS_INSTALL} ${THIRDPARTY_DIR}/src/hiredis/hiredis.h ${PROJECT_BINARY_DIR}/include/hiredis/hiredis.h COMMAND ${REDIS_INSTALL} ${THIRDPARTY_DIR}/src/hiredis/read.h ${PROJECT_BINARY_DIR}/include/hiredis/read.h COMMAND ${REDIS_INSTALL} ${THIRDPARTY_DIR}/src/hiredis/sds.h ${PROJECT_BINARY_DIR}/include/hiredis/sds.h COMMAND ${REDIS_INSTALL} ${THIRDPARTY_DIR}/src/hiredis/async.h ${PROJECT_BINARY_DIR}/include/hiredis/async.h COMMAND ${REDIS_INSTALL} ${THIRDPARTY_DIR}/src/hiredis/adapters/ae.h ${PROJECT_BINARY_DIR}/include/hiredis/adapters/ae.h COMMAND ${REDIS_INSTALL} ${THIRDPARTY_DIR}/src/hiredis/adapters/libev.h ${PROJECT_BINARY_DIR}/include/hiredis/adapters/libev.h COMMAND ${REDIS_INSTALL} ${THIRDPARTY_DIR}/src/hiredis/adapters/libevent.h ${PROJECT_BINARY_DIR}/include/hiredis/adapters/libevent.h COMMAND ${REDIS_INSTALL} ${THIRDPARTY_DIR}/src/hiredis/libhiredis.so ${PROJECT_BINARY_DIR}/lib/libhiredis.so.0.13 COMMAND ln -sf ${PROJECT_BINARY_DIR}/lib/libhiredis.so.0.13 ${PROJECT_BINARY_DIR}/lib/libhiredis.so.0 COMMAND ln -sf ${PROJECT_BINARY_DIR}/lib/libhiredis.so.0 ${PROJECT_BINARY_DIR}/lib/libhiredis.so COMMAND ${REDIS_INSTALL} ${THIRDPARTY_DIR}/src/hiredis/libhiredis.a ${PROJECT_BINARY_DIR}/lib/libhiredis.a
+            )
+            list(APPEND external_project_dependencies hiredis)
+        endif(NOT REDISCLIENT_INCLUDE_DIR OR NOT REDISCLIENT_LIBRARY)
+        set(REDISCLIENT_INCLUDE_DIR "${PROJECT_BINARY_DIR}/include/hiredis")
+        if(BUILD_SHARED_LIBRARY)
+            set(REDISCLIENT_LIBRARY "${PROJECT_BINARY_DIR}/lib/libhiredis.so")
+        else(BUILD_SHARED_LIBRARY)
+            set(REDISCLIENT_LIBRARY "${PROJECT_BINARY_DIR}/lib/libhiredis.a")
+        endif(BUILD_SHARED_LIBRARY)
+        message (STATUS "  (Headers should be)       ${REDISCLIENT_INCLUDE_DIR}")
+        message (STATUS "  (Library should be)       ${REDISCLIENT_LIBRARY}")
+        set(REDISCLIENT_FOUND true)
+        set(REDISCLIENT_DEFINITION "-DWITH_REDIS")
+    endif(WIN32)
+endif(REDISCLIENT_FOUND)
+if(WITHOUT_REDIS)
+    unset(REDISCLIENT_FOUND)
+    unset(REDISCLIENT_DEFINITION)
+    message(STATUS "Not using Hiredis due to WITHOUT_REDIS option")
+endif(WITHOUT_REDIS)
 
 ### RT ###
 
@@ -135,3 +194,36 @@ if(WITHOUT_THRIFT)
     unset(THRIFT_FOUND)
     message(STATUS "Not using Thrift due to WITHOUT_THRIFT option")
 endif(WITHOUT_THRIFT)
+
+### ORC ###
+
+#NAMES liblz4.a liborc.a libprotobuf.a libsnappy.a libz.a 
+#NAMES ColumnPrinter.hh Int128.hh MemoryPool.hh orc-config.hh OrcFile.hh Reader.hh Type.hh  Vector.hh
+find_path(ORC_INCLUDE_DIR NAMES orc/OrcFile.hh)
+find_library(ORC_L0 NAMES protobuf NO_CMAKE_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH NO_SYSTEM_ENVIRONMENT_PATH)
+find_library(ORC_L1 NAMES z NO_CMAKE_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH NO_SYSTEM_ENVIRONMENT_PATH)
+find_library(ORC_L2 NAMES lz4 NO_CMAKE_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH NO_SYSTEM_ENVIRONMENT_PATH)
+find_library(ORC_L3 NAMES snappy NO_CMAKE_ENVIRONMENT_PATH NO_CMAKE_SYSTEM_PATH NO_SYSTEM_ENVIRONMENT_PATH)
+find_library(ORC_L4 NAMES orc)
+
+if (ORC_INCLUDE_DIR AND ORC_L1 AND ORC_L0 AND ORC_L2 AND ORC_L3 AND ORC_L4)
+    set(ORC_FOUND true)
+endif (ORC_INCLUDE_DIR AND ORC_L1 AND ORC_L0 AND ORC_L2 AND ORC_L3 AND ORC_L4)
+if (ORC_FOUND)
+    set(ORC_DEFINITION "-DWITH_ORC")
+    # The order is important for dependencies.
+    set(ORC_LIBRARY ${ORC_L4} ${ORC_L3} ${ORC_L2} ${ORC_L1} ${ORC_L0})
+    message (STATUS "Found ORC:")
+    message (STATUS "  (Headers)       ${ORC_INCLUDE_DIR}")
+    message (STATUS "  (Library)       ${ORC_L0}")
+    message (STATUS "  (Library)       ${ORC_L1}")
+    message (STATUS "  (Library)       ${ORC_L2}")
+    message (STATUS "  (Library)       ${ORC_L3}")
+    message (STATUS "  (Library)       ${ORC_L4}")
+else(ORC_FOUND)
+    message (STATUS "Could NOT find ORC")
+endif(ORC_FOUND)
+if(WITHOUT_ORC)
+    unset(ORC_FOUND)
+    message(STATUS "Not using ORC due to WITHOUT_ORC option")
+endif(WITHOUT_ORC)

core/constants.hpp

@@ -72,7 +72,11 @@ const uint32_t TYPE_KAFKA_REQ = 0xfa091343;
 const uint32_t TYPE_KAFKA_END_REQ = 0xfa091344;
 const uint32_t TYPE_MONGODB_REQ = 0xfa091388;
 const uint32_t TYPE_MONGODB_END_REQ = 0xfa091389;
+const uint32_t TYPE_REDIS_REQ = 0xfa0913c8;
+const uint32_t TYPE_REDIS_QRY_REQ = 0xfa0913c9;
+const uint32_t TYPE_REDIS_END_REQ = 0xfa0913ca;
 const uint32_t TYPE_LOCAL_BLK_REQ = 0xfa0e12a2;
+const uint32_t TYPE_ORC_BLK_REQ = 0xfa2e32a1;
 const uint32_t TYPE_STOP_ASYNC_REQ = 0xf89d74b4;
 const uint32_t TYPE_STOP_ASYNC_YES = 0x09b8ab2b;
 const uint32_t TYPE_STOP_ASYNC_NO = 0x192a241a;
@@ -87,5 +91,6 @@ const uint32_t TYPE_EXIT = 0x47d79fd5;
 const uint32_t TYPE_GET_HASH_RING = 0x48d693d5;
 const uint32_t TYPE_NFS_FILE_REQ = 0x4E465251;
 const uint32_t TYPE_HDFS_FILE_REQ = 0x48465251;
+const uint32_t TYPE_ELASTICSEARCH_REQ = 0xfa081098;
 
 }  // namespace husky

examples/CMakeLists.txt

@@ -49,6 +49,12 @@ target_link_libraries(WordCountMR ${husky})
 target_link_libraries(WordCountMR ${HUSKY_EXTERNAL_LIB})
 husky_default_properties(WordCountMR)
 
+# WordCountES
+add_executable(WordCountES wc_mr_elasticsearch.cpp)
+target_link_libraries(WordCountES ${husky})
+target_link_libraries(WordCountES ${HUSKY_EXTERNAL_LIB})
+husky_default_properties(WordCountES)
+
 # WordCountMongo
 add_executable(WordCountMongo wc_mr_mongo.cpp)
 target_link_libraries(WordCountMongo ${husky})
@@ -61,6 +67,18 @@ target_link_libraries(WordCountFlume ${husky})
 target_link_libraries(WordCountFlume ${HUSKY_EXTERNAL_LIB})
 husky_default_properties(WordCountFlume)
 
+# WordCountRedis
+add_executable(WordCountRedis wc_mr_redis.cpp)
+target_link_libraries(WordCountRedis ${husky})
+target_link_libraries(WordCountRedis ${HUSKY_EXTERNAL_LIB})
+husky_default_properties(WordCountRedis)
+
+# WordCountORC
+add_executable(WordCountORC wc_mr_orc.cpp)
+target_link_libraries(WordCountORC ${husky})
+target_link_libraries(WordCountORC ${HUSKY_EXTERNAL_LIB})
+husky_default_properties(WordCountORC)
+
 # PI
 add_executable(PI pi.cpp)
 target_link_libraries(PI ${husky})
@@ -132,3 +150,9 @@ add_executable(NN_MNIST nn_mnist.cpp)
 target_link_libraries(NN_MNIST ${husky})
 target_link_libraries(NN_MNIST ${HUSKY_EXTERNAL_LIB})
 husky_default_properties(NN_MNIST)
+
+# Cube
+add_executable(Cube cube.cpp)
+target_link_libraries(Cube ${husky})
+target_link_libraries(Cube ${HUSKY_EXTERNAL_LIB})
+husky_default_properties(Cube)