didOpen: Sort index requests by approximity to working files

For a large project, it is preferable to prioritize indexing files
neighboring working files.

Author: MaskRay

src/messages/ccls_info.cc

@@ -17,14 +17,14 @@ struct Out_cclsInfo {
     int files, funcs, types, vars;
   } db;
   struct Pipeline {
-    int64_t lastIdle, completed, enqueued;
+    int64_t lastIdle, completed, enqueued, opened;
   } pipeline;
   struct Project {
     int entries;
   } project;
 };
 REFLECT_STRUCT(Out_cclsInfo::DB, files, funcs, types, vars);
-REFLECT_STRUCT(Out_cclsInfo::Pipeline, lastIdle, completed, enqueued);
+REFLECT_STRUCT(Out_cclsInfo::Pipeline, lastIdle, completed, enqueued, opened);
 REFLECT_STRUCT(Out_cclsInfo::Project, entries);
 REFLECT_STRUCT(Out_cclsInfo, db, pipeline, project);
 } // namespace
@@ -38,6 +38,7 @@ void MessageHandler::ccls_info(EmptyParam &, ReplyOnce &reply) {
   result.pipeline.lastIdle = pipeline::stats.last_idle;
   result.pipeline.completed = pipeline::stats.completed;
   result.pipeline.enqueued = pipeline::stats.enqueued;
+  result.pipeline.opened = pipeline::stats.opened;
   result.project.entries = 0;
   for (auto &[_, folder] : project->root2folder)
     result.project.entries += folder.entries.size();

src/messages/textDocument_did.cc

@@ -50,6 +50,20 @@ void MessageHandler::textDocument_didOpen(DidOpenTextDocumentParam &param) {
     project->indexRelated(path);
 
   manager->onView(path);
+
+  // For the first few didOpen, sort indexer requests based on path similarity.
+  if (++pipeline::stats.opened >= 5)
+    return;
+  std::unordered_map<std::string, int> dir2prio;
+  {
+    std::lock_guard lock(wfiles->mutex);
+    for (auto &[f, wf] : wfiles->files) {
+      std::string cur = lowerPathIfInsensitive(f);
+      for (int pri = 1 << 20; !(cur = llvm::sys::path::parent_path(cur)).empty(); pri /= 2)
+        dir2prio[cur] += pri;
+    }
+  }
+  pipeline::indexerSort(dir2prio);
 }
 
 void MessageHandler::textDocument_didSave(TextDocumentParam &param) {

src/pipeline.cc

@@ -87,6 +87,7 @@ struct IndexRequest {
   bool must_exist = false;
   RequestId id;
   int64_t ts = tick++;
+  int prio = 0; // For didOpen sorting
 };
 
 std::mutex thread_mtx;
@@ -488,6 +489,23 @@ void indexer_Main(SemaManager *manager, VFS *vfs, Project *project,
         break;
 }
 
+void indexerSort(const std::unordered_map<std::string, int> &dir2prio) {
+  index_request->apply([&](std::deque<IndexRequest> &q) {
+    for (IndexRequest &request : q) {
+      std::string cur = lowerPathIfInsensitive(request.path);
+      while (!(cur = llvm::sys::path::parent_path(cur)).empty()) {
+        auto it = dir2prio.find(cur);
+        if (it != dir2prio.end()) {
+          request.prio = it->second;
+          LOG_V(3) << "set priority " << request.prio << " to " << request.path;
+          break;
+        }
+      }
+    }
+    std::stable_sort(q.begin(), q.end(), [](auto &l, auto &r) { return l.prio > r.prio; });
+  });
+}
+
 void main_OnIndexed(DB *db, WorkingFiles *wfiles, IndexUpdate *update) {
   if (update->refresh) {
     LOG_S(INFO)

src/pipeline.hh

@@ -40,7 +40,7 @@ enum class IndexMode {
 };
 
 struct IndexStats {
-  std::atomic<int64_t> last_idle, completed, enqueued;
+  std::atomic<int64_t> last_idle, completed, enqueued, opened;
 };
 
 namespace pipeline {
@@ -56,6 +56,7 @@ void launchStdin();
 void launchStdout();
 void indexer_Main(SemaManager *manager, VFS *vfs, Project *project,
                   WorkingFiles *wfiles);
+void indexerSort(const std::unordered_map<std::string, int> &dir2prio);
 void mainLoop();
 void standalone(const std::string &root);
 

src/project.hh

@@ -28,6 +28,7 @@ struct Project {
     // 0 unless coming from a compile_commands.json entry.
     int compdb_size = 0;
     int id = -1;
+    int prio = 0;
   };
 
   struct Folder {

src/threaded_queue.hh

@@ -161,12 +161,9 @@ public:
     return std::nullopt;
   }
 
-  template <typename Fn> void iterate(Fn fn) {
+  template <typename Fn> void apply(Fn fn) {
     std::lock_guard<std::mutex> lock(mutex_);
-    for (auto &entry : priority_)
-      fn(entry);
-    for (auto &entry : queue_)
-      fn(entry);
+    fn(queue_);
   }
 
   mutable std::mutex mutex_;