Merge pull request #169 from cwlacewe/master

v2.8.0 Release

Author: michaelbeale-IL

.gitignore

@@ -9,4 +9,7 @@ tmp
 # VS Code Specific
 .devcontainer
 .vscode
-.coverage
+.coverage
+
+# Jetbrains Specific
+.idea

CMakeLists.txt

@@ -56,24 +56,28 @@ else()
   add_subdirectory(tests)
   add_subdirectory(src/vcl)
   add_subdirectory(client/cpp)
-  add_subdirectory(ext/custom_vcl)
   add_subdirectory(distributed)
 
   link_directories(/usr/local/lib /usr/lib/x86_64-linux-gnu/)
   include_directories(/usr/include/jsoncpp utils/include/ src/pmgd/include src/pmgd/util include/ src/vcl /usr/include ${CMAKE_CURRENT_BINARY_DIR}/utils/src/protobuf)
   add_library(dms SHARED
+    src/BackendNeo4j.cc
     src/BoundingBoxCommand.cc
     src/BlobCommand.cc
     src/CommunicationManager.cc
     src/DescriptorsCommand.cc
     src/DescriptorsManager.cc
     src/ExceptionsCommand.cc
     src/ImageCommand.cc
+    src/Neo4jBaseCommands.cc
+    src/Neo4JHandlerCommands.cc
+    src/OpsIOCoordinator.cc
     src/PMGDIterators.cc
     src/PMGDQuery.cc
     src/PMGDQueryHandler.cc
     src/QueryHandlerExample.cc
     src/QueryHandlerBase.cc
+    src/QueryHandlerNeo4j.cc
     src/QueryHandlerPMGD.cc
     src/QueryMessage.cc
     src/RSCommand.cc
@@ -85,7 +89,7 @@ else()
     src/ImageLoop.cc
     src/VideoLoop.cc
   )
-  target_link_libraries(dms vcl pmgd pmgd-util protobuf tbb tiledb vdms-utils pthread -lcurl -lzmq ${AWSSDK_LINK_LIBRARIES})
+  target_link_libraries(dms vcl pmgd pmgd-util protobuf tbb tiledb vdms-utils pthread -lcurl -lzmq ${AWSSDK_LINK_LIBRARIES} neo4j-client)
   add_executable(vdms src/vdms.cc)
   target_link_libraries(vdms dms vdms_protobuf vcl tiledb faiss flinng jsoncpp ${OpenCV_LIBS} ${AWSSDK_LINK_LIBRARIES})
 endif ()

INSTALL.md

@@ -10,10 +10,10 @@ Here we will install the Debian/Ubuntu packages.
 sudo apt-get update -y  --fix-missing
 sudo apt-get upgrade -y
 sudo apt-get install -y --no-install-suggests --no-install-recommends \
-    apt-transport-https autoconf automake bison build-essential bzip2 ca-certificates \
+    apt-transport-https automake bison build-essential bzip2 ca-certificates \
     curl ed flex g++-9 gcc-9 git gnupg-agent javacc libarchive-tools libatlas-base-dev \
     libavcodec-dev libavformat-dev libboost-all-dev libbz2-dev libc-ares-dev libcurl4-openssl-dev \
-    libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtk-3-dev libgtk2.0-dev \
+    libncurses5-dev libdc1394-22-dev libgflags-dev libgoogle-glog-dev libgtk-3-dev libgtk2.0-dev \
     libhdf5-dev libjpeg-dev libjsoncpp-dev libleveldb-dev liblmdb-dev \
     liblz4-dev libopenblas-dev libopenmpi-dev libpng-dev librdkafka-dev libsnappy-dev libssl-dev \
     libswscale-dev libtbb-dev libtbb2 libtiff-dev libtiff5-dev libtool libzmq3-dev linux-libc-dev mpich \
@@ -80,6 +80,20 @@ cd $VDMS_DEP_DIR/CMake
 make ${BUILD_THREADS}
 sudo make install
 ```
+***NOTE:*** If multiple versions of Python 3 are present on your system, verify you are using Python3.9 or higher. You can specify the specific verison in above command and also set the following with your specific version: `alias python3=/usr/bin/python3.x`.
+
+
+#### **Autoconf v2.71**
+```bash
+AUTOCONF_VERSION="2.71"
+curl -L -o $VDMS_DEP_DIR/autoconf-${AUTOCONF_VERSION}.tar.xz https://ftp.gnu.org/gnu/autoconf/autoconf-${AUTOCONF_VERSION}.tar.xz
+cd $VDMS_DEP_DIR
+tar -xf autoconf-${AUTOCONF_VERSION}.tar.xz
+cd autoconf-${AUTOCONF_VERSION}
+./configure
+make ${BUILD_THREADS}
+sudo make install
+```
 
 
 #### **Protobuf v24.2 (4.24.2)**
@@ -90,38 +104,44 @@ git clone -b v${PROTOBUF_VERSION} --recurse-submodules https://github.com/protoc
 
 cd $VDMS_DEP_DIR/protobuf/third_party/googletest
 mkdir build && cd build
-cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=/usr/local \
+cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_BUILD_TYPE=Release \
+    -DBUILD_SHARED_LIBS=ON -DCMAKE_INSTALL_PREFIX=/usr/local \
     -DBUILD_GMOCK=ON -DCMAKE_CXX_STANDARD=17 ..
 make ${BUILD_THREADS}
 sudo make install
-sudo ldconfig
 
 cd $VDMS_DEP_DIR/protobuf/third_party/abseil-cpp
 mkdir build && cd build
-cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_PREFIX_PATH=/usr/local/ -DCMAKE_INSTALL_PREFIX=/usr/local/ \
-    -DABSL_BUILD_TESTING=ON -DABSL_ENABLE_INSTALL=ON -DABSL_USE_EXTERNAL_GOOGLETEST=ON \
+cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DBUILD_SHARED_LIBS=ON \
+    -DCMAKE_INSTALL_PREFIX=/usr/local -DABSL_BUILD_TESTING=ON \
+    -DABSL_USE_EXTERNAL_GOOGLETEST=ON \
     -DABSL_FIND_GOOGLETEST=ON -DCMAKE_CXX_STANDARD=17 ..
 make ${BUILD_THREADS}
 sudo make install
+sudo ldconfig /usr/local/lib
 
 cd $VDMS_DEP_DIR/protobuf
-cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_INSTALL_PREFIX=/usr/local -DCMAKE_CXX_STANDARD=17 \
-    -Dprotobuf_ABSL_PROVIDER=package -DCMAKE_PREFIX_PATH=/usr/local .
+cmake -DCMAKE_POSITION_INDEPENDENT_CODE=ON -DCMAKE_INSTALL_PREFIX=/usr/local \
+    -DCMAKE_CXX_STANDARD=17 -Dprotobuf_BUILD_SHARED_LIBS=ON \
+    -Dprotobuf_ABSL_PROVIDER=package \
+    -Dprotobuf_BUILD_TESTS=ON \
+    -Dabsl_DIR=/usr/local/lib/cmake/absl .
 make ${BUILD_THREADS}
 sudo make install
 
 python3 -m pip install --no-cache-dir "protobuf==4.${PROTOBUF_VERSION}"
 ```
 
 
-#### **Faiss v1.7.3**
+#### **Faiss v1.7.4**
 Install the Faiss library for similarity search.
 ```bash
-FAISS_VERSION="v1.7.3"
+FAISS_VERSION="v1.7.4"
 git clone --branch ${FAISS_VERSION} https://github.com/facebookresearch/faiss.git $VDMS_DEP_DIR/faiss
 cd $VDMS_DEP_DIR/faiss
 mkdir build && cd build
-cmake -DFAISS_ENABLE_GPU=OFF -DPython_EXECUTABLE=/usr/bin/python3 ..
+cmake -DFAISS_ENABLE_GPU=OFF -DPython_EXECUTABLE=/usr/bin/python3 \
+    -DBUILD_SHARED_LIBS=ON -DCMAKE_BUILD_TYPE=Release ..
 make ${BUILD_THREADS}
 sudo make install
 ```
@@ -192,6 +212,40 @@ cmake -D BUILD_PERF_TESTS=OFF -D BUILD_TESTS=OFF -D CMAKE_BUILD_TYPE=RELEASE -D
 make ${BUILD_THREADS}
 sudo make install
 ```
+
+
+#### **Neo4j Client**
+Below are instructions for installing ***libneo4j-omni*** which requires Peg, libcypher-parser and libedit as dependencies.
+```bash
+PEG_VERSION="0.1.19"
+curl -L -o $VDMS_DEP_DIR/peg-${PEG_VERSION}.tar.gz https://github.com/gpakosz/peg/releases/download/${PEG_VERSION}/peg-${PEG_VERSION}.tar.gz
+cd $VDMS_DEP_DIR/
+tar -xf peg-${PEG_VERSION}.tar.gz
+cd peg-${PEG_VERSION}
+make ${BUILD_THREADS}
+sudo make install
+
+git clone https://github.com/cleishm/libcypher-parser.git $VDMS_DEP_DIR/libcypher
+cd $VDMS_DEP_DIR/libcypher
+./autogen.sh
+./configure
+sudo make install
+
+LIBEDIT_VERSION="20230828-3.1"
+curl -L -o $VDMS_DEP_DIR/libedit-${LIBEDIT_VERSION}.tar.gz https://thrysoee.dk/editline/libedit-${LIBEDIT_VERSION}.tar.gz
+cd $VDMS_DEP_DIR/
+tar -xzf libedit-${LIBEDIT_VERSION}.tar.gz
+cd libedit-${LIBEDIT_VERSION}
+./configure
+make ${BUILD_THREADS}
+sudo make install
+
+git clone https://github.com/majensen/libneo4j-omni.git $VDMS_DEP_DIR/libomni
+cd $VDMS_DEP_DIR/libomni
+./autogen.sh
+./configure --disable-werror --prefix=/usr
+sudo make install -w --debug
+```
 <br>
 
 ## Install VDMS
@@ -216,3 +270,10 @@ cmake -DCMAKE_CXX_FLAGS='-DPM' ..
 make ${BUILD_THREADS}
 ```
 
+***NOTE:*** If error similar to `cannot open shared object file: No such file or directory` obtained during loading shared libraries, such as `libpmgd.so` or `libvcl.so`, add the correct directories to `LD_LIBRARY_PATH`. This may occur for non-root users. To find the correct directory, run `find` command for missing object file. An example solution for missing `libpmgd.so` and `libvcl.so` is:
+```bash
+find / -name "libpmgd*so*" # <Path_to_VDMS_directory>/build/src/pmgd/src
+find / -name "libvcl*so*"  # <Path_to_VDMS_directory>/build/src/vcl
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:<Path_to_VDMS_directory>/build/src/pmgd/src:<Path_to_VDMS_directory>/build/src/vcl
+```
+

client/cpp/CMakeLists.txt

@@ -1,13 +1,11 @@
 cmake_minimum_required (VERSION 3.10)
 project(vdms_client)
 find_package( OpenCV REQUIRED )
+find_package(Protobuf CONFIG REQUIRED)
 find_package( Threads REQUIRED )
 include_directories(../../utils/include/ )
 
 
-add_library(  vdms-client SHARED 
-                VDMSClient.cc
-                CSVParserUtil.cpp
-)
+add_library(vdms-client SHARED VDMSClient.cc CSVParserUtil.cpp)
 target_link_libraries(vdms-client protobuf vdms_protobuf vdms-utils pthread ${CMAKE_THREAD_LIBS_INIT} )
 

client/cpp/CSVParser.h

@@ -14,41 +14,52 @@ class CSVParser {
             int port)
       : m_filename(filename), m_num_threads(num_threads), vdms_server(server),
         vdms_port(port) {}
+  ~CSVParser() {}
+
   std::vector<VDMS::Response>
   parse_csv_lines(const std::string &filename, int start_line, int end_line,
-                  std::mutex &mutex, std::vector<VDMS::Response> &all_results,
+                  std::vector<VDMS::Response> &local_results,
                   const size_t thread_id) {
-    rapidcsv::Document csv(filename);
 
-    size_t rowCount = csv.GetRowCount();
+    rapidcsv::Document csv(filename);
     std::vector<std::string> columnNames = csv.GetColumnNames();
     VDMS::CSVParserUtil csv_util(vdms_server, vdms_port, columnNames,
                                  static_cast<int>(thread_id));
-    for (int i = start_line; i < end_line; ++i) {
+    for (int i = start_line; i <= end_line - 1; ++i) {
       std::vector<std::string> row = csv.GetRow<std::string>(i);
       VDMS::Response result = csv_util.parse_row(row);
+      if (local_results.empty()) {
+        // If local_results is empty, resize it to have at least one element
+        local_results.resize(1);
+      } else if (i - start_line >= static_cast<int>(local_results.size())) {
+        // If the index is beyond the current size, resize to accommodate the
+        // index
+        local_results.resize(i - start_line + 1);
+      }
 
-      std::lock_guard<std::mutex> lock(mutex);
-      all_results.push_back(result);
+      // Replace the value at the specified index in local_results
+      local_results[i - start_line] = result;
     }
 
-    return all_results;
+    return local_results;
   }
-  std::vector<VDMS::Response> parse() {
-    auto start = std::chrono::high_resolution_clock::now();
 
+  std::vector<VDMS::Response> parse() {
     std::ifstream file(m_filename);
-    if (!file) {
-      std::cerr << "Error opening file\n";
+    if (!file.is_open()) {
+      std::cerr << "Error opening file: " << m_filename << std::endl;
     }
-
     int num_lines = std::count(std::istreambuf_iterator<char>(file),
                                std::istreambuf_iterator<char>(), '\n');
+    file.close();
+
     std::size_t lines_per_thread = num_lines / m_num_threads;
 
     std::mutex mutex;
     std::vector<std::thread> threads;
-    std::vector<VDMS::Response> all_results;
+    std::vector<std::vector<VDMS::Response>> all_local_results(m_num_threads);
+    std::vector<VDMS::Response> all_results; // Local vector for each thread
+    all_results.reserve(num_lines);
 
     for (size_t i = 0; i < m_num_threads; i++) {
       size_t start_line = i * lines_per_thread;
@@ -57,16 +68,24 @@ class CSVParser {
 
       threads.emplace_back(&CSVParser::parse_csv_lines, this,
                            std::ref(m_filename), start_line, end_line,
-                           std::ref(mutex), std::ref(all_results), i);
+                           std::ref(all_local_results[i]), i);
     }
 
     for (auto &thread : threads) {
       thread.join();
     }
+    size_t allResultsSizeBefore = all_results.size();
+    for (const auto &local_results : all_local_results) {
+
+      // Extend the size of all_results to accommodate local_results
+      all_results.resize(all_results.size() + local_results.size());
+
+      // Copy elements from local_results to the appropriate positions in
+      // all_results
+      std::copy(local_results.begin(), local_results.end(),
+                all_results.begin() + allResultsSizeBefore);
+    }
 
-    auto finish = std::chrono::high_resolution_clock::now();
-    std::chrono::duration<double> elapsed = finish - start;
-    // std::cout << "Elapsed time: " << elapsed.count() << " s\n";
     return all_results;
   }
 

client/cpp/CSVParserUtil.cpp

@@ -11,7 +11,7 @@
 #include <jsoncpp/json/json.h>
 #include <mutex>
 #include <sstream>
-static std::mutex barrier;
+
 std::mutex mtx;
 using namespace std;
 using namespace std::chrono;
@@ -46,7 +46,6 @@ void VDMS::CSVParserUtil::initCommandsMap() {
               {"RectangleUpdate", RectangleUpdate}};
 }
 string VDMS::CSVParserUtil::function_accessing_columnNames(int i) {
-  std::lock_guard<std::mutex> lock(mtx);
 
   return _columnNames[i];
 }
@@ -146,7 +145,6 @@ VDMS::CSVParserUtil::commandType
 VDMS::CSVParserUtil::get_query_type(const string &str) {
   CSVParserUtil::commandType querytype = commandType::UNKNOWN;
 
-  std::lock_guard<std::mutex> lock(CSVParserUtil::querytype_mutex);
   std::map<std::string, QueryType>::iterator iter;
   iter = commands.find(str);
   if (iter != commands.end()) {
@@ -173,8 +171,6 @@ VDMS::CSVParserUtil::get_query_type(const string &str) {
       querytype = commandType::AddBoundingBox;
       break;
     }
-    //   std::cout << " I executed queryType "<< querytype << std::endl;
-    // return querytype;
   }
 
   return querytype;
@@ -200,9 +196,7 @@ void VDMS::CSVParserUtil::parseProperty(const string &columnNames,
                                         const string &row,
                                         const string &queryType,
                                         Json::Value &aquery) {
-  std::lock_guard<std::mutex> lock(CSVParserUtil::aquery_mutex);
   string propname = columnNames.substr(5, string::npos);
-  // std::cout << "Inside parseProp " << propname <<std::endl;
   CSVParserUtil::DATATYPE dtype = getDataType(row, propname);
   if (dtype == DATATYPE::DATE) {
     Json::Value date;
@@ -224,7 +218,6 @@ void VDMS::CSVParserUtil::parseProperty(const string &columnNames,
 void VDMS::CSVParserUtil::parseConstraints(const string &columnNames,
                                            const string &row, string &queryType,
                                            Json::Value &aquery) {
-  std::lock_guard<std::mutex> lock(CSVParserUtil::cons_mutex);
   vector<string> consvals = spiltrow(row);
   string consname = consvals[0];
   if (consname.substr(0, 5) == "date:") {
@@ -284,7 +277,6 @@ void VDMS::CSVParserUtil::parseOperations(string columnNames, string row,
                                           string queryType,
                                           Json::Value &aquery) {
 
-  std::lock_guard<std::mutex> lock(CSVParserUtil::ops_mutex);
   string type = columnNames.substr(4, string::npos);
   Json::Value opsjson;
   vector<string> opsKeys;
@@ -424,13 +416,6 @@ VDMS::CSVParserUtil::DATATYPE VDMS::CSVParserUtil ::isValidDataType(string data,
                                                                     int type) {
   CSVParserUtil::DATATYPE actualtype = getDataType(data, "");
   return actualtype;
-
-  // if(type==2 && (actualt=3 || acype=tualtype==4))//2 is for num
-  //     return actualtype;
-  // else if(type==1 && (actualtype==1 || actualtype==2))//1 is for bool
-  //     return actualtype;
-  // else
-  //     return -1;
 }
 
 bool VDMS::CSVParserUtil::isValidOpsType(string &type) {
@@ -542,6 +527,5 @@ VDMS::Response
 VDMS::CSVParserUtil::send_to_vdms(const Json::Value &query,
                                   const std::vector<std::string *> blobs) {
   Json::StyledWriter _fastwriter;
-
   return vdms_client->query(_fastwriter.write(query), blobs);
 }

client/cpp/CSVParserUtil.h

@@ -97,10 +97,6 @@ class CSVParserUtil {
   std::string vdms_server;
   int vdms_port;
   std::vector<std::string> _columnNames;
-  std::mutex querytype_mutex;
-  std::mutex aquery_mutex;
-  std::mutex cons_mutex;
-  std::mutex ops_mutex;
   int id;
   std::unique_ptr<VDMS::VDMSClient> vdms_client;
 };