fix(crawler): bad urls from sitemap

mendableai · Jan 17, 2025 · 5992c57 · 5992c57
1 parent 237d0dc
commit 5992c57
Showing 1 changed file with 1 addition and 1 deletion.
diff --git a/apps/api/src/scraper/WebScraper/crawler.ts b/apps/api/src/scraper/WebScraper/crawler.ts
@@ -228,7 +228,7 @@ export class WebCrawler {
         );
         leftOfLimit -= filteredLinks.length;
         let uniqueURLs: string[] = [];
-        for (const url of urls) {
+        for (const url of filteredLinks) {
           if (
             await redisConnection.sadd(
               "sitemap:" + this.jobId + ":links",