fix(removeUnwantedElements): post-includeTags excludeTags

Fixes #700
mendableai · Jan 17, 2025 · faf58df · faf58df
1 parent de08b37
commit faf58df
Showing 1 changed file with 3 additions and 2 deletions.
diff --git a/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts b/apps/api/src/scraper/scrapeURL/lib/removeUnwantedElements.ts
@@ -54,7 +54,7 @@ export const htmlTransform = (
   url: string,
   scrapeOptions: ScrapeOptions,
 ) => {
-  const soup = load(html);
+  let soup = load(html);
 
   // remove unwanted elements
   if (
@@ -68,7 +68,8 @@ export const htmlTransform = (
         newRoot.append(soup(element).clone());
       });
     });
-    return newRoot.html() ?? "";
+
+    soup = load(newRoot.html() ?? "");
   }
 
   soup("script, style, noscript, meta, head").remove();