fix(queue-worker/kickoff): make crawls wait for kickoff to finish (ma…

…tters on big sitemapped sites)
mendableai · Jan 17, 2025 · d5929af · d5929af
1 parent 23bb172
commit d5929af
Show file tree

Hide file tree

Showing 2 changed files with 9 additions and 0 deletions.
diff --git a/apps/api/src/lib/crawl-redis.ts b/apps/api/src/lib/crawl-redis.ts
@@ -128,13 +128,18 @@ export async function isCrawlFinished(id: string) {
   return (
     (await redisConnection.scard("crawl:" + id + ":jobs_done")) ===
     (await redisConnection.scard("crawl:" + id + ":jobs"))
+    && (await redisConnection.get("crawl:" + id + ":kickoff:finish")) !== null
   );
 }
 
 export async function isCrawlFinishedLocked(id: string) {
   return await redisConnection.exists("crawl:" + id + ":finish");
 }
 
+export async function finishCrawlKickoff(id: string) {
+  await redisConnection.set("crawl:" + id + ":kickoff:finish", "yes", "EX", 24 * 60 * 60);
+}
+
 export async function finishCrawl(id: string) {
   if (await isCrawlFinished(id)) {
     _logger.debug("Marking crawl as finished.", {

diff --git a/apps/api/src/services/queue-worker.ts b/apps/api/src/services/queue-worker.ts
@@ -23,6 +23,7 @@ import {
   addCrawlJobs,
   crawlToCrawler,
   finishCrawl,
+  finishCrawlKickoff,
   generateURLPermutations,
   getCrawl,
   getCrawlJobCount,
@@ -675,6 +676,9 @@ async function processKickoffJob(job: Job & { id: string }, token: string) {
 
     logger.debug("Done queueing jobs!");
 
+    await finishCrawlKickoff(job.data.crawl_id);
+    await finishCrawlIfNeeded(job, sc);
+
     return { success: true };
   } catch (error) {
     logger.error("An error occurred!", { error });