proper threading

KoslickiLab · Nov 15, 2024 · 8a7c132 · 8a7c132
1 parent 4b3c080
commit 8a7c132
Show file tree

Hide file tree

Showing 2 changed files with 28 additions and 20 deletions.
diff --git a/Makefile b/Makefile
@@ -16,9 +16,6 @@ OBJ_FILES = $(SRC_FILES:.cpp=.o)
 TARGET1 = $(BIN_DIR)/run_yacht_train_core
 TARGET2 = $(BIN_DIR)/run_compute_similarity
 
-debug: CXXFLAGS += -g
-debug: all
-
 # Build rules
 all: $(TARGET1) $(TARGET2)
 

diff --git a/src/cpp/utils.cpp b/src/cpp/utils.cpp
@@ -39,23 +39,21 @@ should be many times faster??
 
 
 void compute_index_from_sketches_one_chunk( int sketch_index_start, int sketch_index_end,
-                                            std::vector<std::vector<hash_t>>& sketches,
-                                            std::unordered_map<hash_t, std::vector<int>>& hash_index,
-                                            std::mutex * mutex_list, int num_mutexes = 1024) {
-
+                                        std::vector<std::vector<hash_t>>& sketches,
+                                        std::vector<std::unordered_map<hash_t, std::vector<int>>>& hash_index_chunks,
+                                        std::mutex * mutex_list, int num_mutexes = 1024) {
+
+
     for (int i = sketch_index_start; i < sketch_index_end; i++) {
         for (uint j = 0; j < sketches[i].size(); j++) {
             hash_t hash_value = sketches[i][j];
-            int mutex_index = num_mutexes * (long double)hash_value * 1000.0 / 0xFFFFFFFFFFFFFFFF;
-            if (mutex_index >= num_mutexes) {
-                mutex_index = num_mutexes - 1;
+            int working_index = hash_value % num_mutexes;
+            mutex_list[working_index].lock();
+            if (hash_index_chunks[working_index].find(hash_value) == hash_index_chunks[working_index].end()) {
+                hash_index_chunks[working_index][hash_value] = std::vector<int>();
             }
-            mutex_list[mutex_index].lock();
-            if (hash_index.find(hash_value) == hash_index.end()) {
-                hash_index[hash_value] = std::vector<int>();
-            }
-            hash_index[hash_value].push_back(i);
-            mutex_list[mutex_index].unlock();
+            hash_index_chunks[working_index][hash_value].push_back(i);
+            mutex_list[working_index].unlock();
         }
     }
 
@@ -67,8 +65,13 @@ void compute_index_from_sketches_one_chunk( int sketch_index_start, int sketch_i
 void compute_index_from_sketches(std::vector<std::vector<hash_t>>& sketches, std::unordered_map<hash_t, std::vector<int>>& hash_index, const int num_threads) {
 
     // create mutexes
-    int num_mutexes = 1024;
-    std::mutex * mutex_list = new std::mutex[num_mutexes];
+    int num_unordered_maps = 1024;
+    std::mutex * mutex_list = new std::mutex[num_unordered_maps];
+    std::vector<std::unordered_map<hash_t, std::vector<int>>> hash_index_chunks(num_unordered_maps);
+
+    for (int i = 0; i < num_unordered_maps; i++) {
+        hash_index_chunks[i] = std::unordered_map<hash_t, std::vector<int>>();
+    }
 
     // create threads
     int num_sketches = sketches.size();
@@ -79,15 +82,23 @@ void compute_index_from_sketches(std::vector<std::vector<hash_t>>& sketches, std
         int end_index = (i == num_threads - 1) ? num_sketches : (i + 1) * chunk_size;
         threads.push_back(std::thread(compute_index_from_sketches_one_chunk, 
                                         start_index, end_index, 
-                                        std::ref(sketches), std::ref(hash_index), 
-                                        mutex_list, num_mutexes));
+                                        std::ref(sketches), std::ref(hash_index_chunks), 
+                                        mutex_list, num_unordered_maps));
     }
 
     // join threads
     for (int i = 0; i < num_threads; i++) {
         threads[i].join();
     }
 
+    // merge the hash_index_chunks to hash_index
+    for (int i = 0; i < num_unordered_maps; i++) {
+        for (auto it = hash_index_chunks[i].begin(); it != hash_index_chunks[i].end(); it++) {
+            hash_t hash_value = it->first;
+            hash_index[hash_value] = it->second;
+        }
+    }
+
 }