Improve API error handling and enforce Node.js >=18.0.0 requirement

TODO.md

@@ -0,0 +1,73 @@
+# TODO: Fix Run Issues
+
+## Issue Summary
+- Next.js 14 requires Node.js 18.0.0+
+- Older Node versions cause NPM "BadEngine" errors
+- Bun installs successfully but 'next dev' hangs due to experimental Next.js support
+- API errors (422/403) cause crashes due to unhandled exceptions
+
+## Fixes Applied
+- Added "engines": {"node": ">=18.0.0"} to package.json to specify Node requirement
+- Fixed error handling in searchProviders.tsx to return [] instead of throwing on API failures
+
+## Recommended: Run with Docker (Easiest, No Node Version Issues)
+
+### Prerequisites
+- Install Docker Desktop for Windows: https://www.docker.com/products/docker-desktop
+
+### Steps
+1. Clone the repository:
+   ```
+   git clone https://github.com/developersdigest/llm-answer-engine.git
+   cd llm-answer-engine
+   ```
+
+2. Edit `docker-compose.yml` and replace the placeholder API keys with your actual keys:
+   ```
+   OPENAI_API_KEY=your_actual_openai_key
+   GROQ_API_KEY=your_actual_groq_key
+   BRAVE_SEARCH_API_KEY=your_actual_brave_key
+   SERPER_API=your_actual_serper_key
+   ```
+
+3. Run the application:
+   ```
+   docker compose up -d
+   ```
+
+4. Access the app at: http://localhost:3000
+
+## Alternative: Manual Setup on Windows (No WSL Required)
+
+### 1. Install Node.js 18+ from https://nodejs.org/
+
+### 2. Clone the repository:
+```
+git clone https://github.com/developersdigest/llm-answer-engine.git
+cd llm-answer-engine
+```
+
+### 3. Install dependencies:
+```
+npm install --legacy-peer-deps
+```
+
+### 4. Create .env file with your API keys:
+```
+OPENAI_API_KEY=your_openai_api_key
+GROQ_API_KEY=your_groq_api_key
+BRAVE_SEARCH_API_KEY=your_brave_search_api_key
+SERPER_API=your_serper_api_key
+```
+
+### 5. Run the development server:
+```
+npm run dev
+```
+
+### 6. Open browser to http://localhost:3000
+
+## Notes
+- Docker is recommended as it avoids Node version conflicts
+- Avoid using Bun for now as it's experimental with Next.js
+- The app now handles API errors gracefully without crashing

app/tools/contentProcessing.tsx

@@ -6,102 +6,118 @@ import { MemoryVectorStore } from 'langchain/vectorstores/memory';
 import { Document as DocumentInterface } from 'langchain/document';
 import { OpenAIEmbeddings } from '@langchain/openai';
 import { OllamaEmbeddings } from "@langchain/community/embeddings/ollama";
+
 let embeddings: OllamaEmbeddings | OpenAIEmbeddings;
 if (config.useOllamaEmbeddings) {
-    embeddings = new OllamaEmbeddings({
-        model: config.embeddingsModel,
-        baseUrl: "http://localhost:11434"
-    });
+  embeddings = new OllamaEmbeddings({
+    model: config.embeddingsModel,
+    baseUrl: "http://localhost:11434",
+  });
 } else {
-    embeddings = new OpenAIEmbeddings({
-        modelName: config.embeddingsModel
-    });
+  embeddings = new OpenAIEmbeddings({
+    modelName: config.embeddingsModel,
+  });
 }
 
 interface SearchResult {
-    title: string;
-    link: string;
-    favicon: string;
+  title: string;
+  link: string;
+  favicon: string;
 }
 interface ContentResult extends SearchResult {
-    html: string;
+  html: string;
 }
 
-
-//  Fetch contents of top 10 search results
-export async function get10BlueLinksContents(sources: SearchResult[]): Promise<ContentResult[]> {
-    async function fetchWithTimeout(url: string, options: RequestInit = {}, timeout = 800): Promise<Response> {
-        try {
-            const controller = new AbortController();
-            const timeoutId = setTimeout(() => controller.abort(), timeout);
-            const response = await fetch(url, { ...options, signal: controller.signal });
-            clearTimeout(timeoutId);
-            return response;
-        } catch (error) {
-            if (error) {
-                console.log(`Skipping ${url}!`);
-            }
-            throw error;
-        }
+// Fetch contents of top 10 search results
+export async function get10BlueLinksContents(
+  sources: SearchResult[]
+): Promise<ContentResult[]> {
+  async function fetchWithTimeout(
+    url: string,
+    options: RequestInit = {},
+    timeout = 800
+  ): Promise<Response> {
+    try {
+      const controller = new AbortController();
+      const timeoutId = setTimeout(() => controller.abort(), timeout);
+      const response = await fetch(url, { ...options, signal: controller.signal });
+      clearTimeout(timeoutId);
+      return response;
+    } catch (error) {
+      if (error) {
+        console.log(`Skipping ${url}!`);
+      }
+      throw error;
     }
-    function extractMainContent(html: string): string {
-        try {
-            const $ = cheerio.load(html);
-            $("script, style, head, nav, footer, iframe, img").remove();
-            return $("body").text().replace(/\s+/g, " ").trim();
-        } catch (error) {
-            console.error('Error extracting main content:', error);
-            throw error;
-        }
+  }
+  function extractMainContent(html: string): string {
+    try {
+      const $ = cheerio.load(html);
+      $("script, style, head, nav, footer, iframe, img").remove();
+      return $("body").text().replace(/\s+/g, " ").trim();
+    } catch (error) {
+      console.error("Error extracting main content:", error);
+      throw error;
     }
-    const promises = sources.map(async (source): Promise<ContentResult | null> => {
-        try {
-            const response = await fetchWithTimeout(source.link, {}, 800);
-            if (!response.ok) {
-                throw new Error(`Failed to fetch ${source.link}. Status: ${response.status}`);
-            }
-            const html = await response.text();
-            const mainContent = extractMainContent(html);
-            return { ...source, html: mainContent };
-        } catch (error) {
-            // console.error(`Error processing ${source.link}:`, error);
-            return null;
-        }
-    });
+  }
+  const promises = sources.map(async (source): Promise<ContentResult | null> => {
     try {
-        const results = await Promise.all(promises);
-        return results.filter((source): source is ContentResult => source !== null);
+      const response = await fetchWithTimeout(source.link, {}, 800);
+      if (!response.ok) {
+        throw new Error(`Failed to fetch ${source.link}. Status: ${response.status}`);
+      }
+      const html = await response.text();
+      const mainContent = extractMainContent(html);
+      return { ...source, html: mainContent };
     } catch (error) {
-        console.error('Error fetching and processing blue links contents:', error);
-        throw error;
+      // console.error(`Error processing ${source.link}:`, error);
+      return null;
     }
+  });
+  try {
+    const results = await Promise.all(promises);
+    return results.filter((source): source is ContentResult => source !== null);
+  } catch (error) {
+    console.error("Error fetching and processing blue links contents:", error);
+    return [];
+  }
 }
-// rocess and vectorize content using LangChain
+
+// Process and vectorize content using LangChain
 export async function processAndVectorizeContent(
-    contents: ContentResult[],
-    query: string,
-    textChunkSize = config.textChunkSize,
-    textChunkOverlap = config.textChunkOverlap,
-    numberOfSimilarityResults = config.numberOfSimilarityResults,
+  contents: ContentResult[],
+  query: string,
+  textChunkSize = config.textChunkSize,
+  textChunkOverlap = config.textChunkOverlap,
+  numberOfSimilarityResults = config.numberOfSimilarityResults
 ): Promise<DocumentInterface[]> {
-    const allResults: DocumentInterface[] = [];
-    try {
-        for (let i = 0; i < contents.length; i++) {
-            const content = contents[i];
-            if (content.html.length > 0) {
-                try {
-                    const splitText = await new RecursiveCharacterTextSplitter({ chunkSize: textChunkSize, chunkOverlap: textChunkOverlap }).splitText(content.html);
-                    const vectorStore = await MemoryVectorStore.fromTexts(splitText, { title: content.title, link: content.link }, embeddings);
-                    const contentResults = await vectorStore.similaritySearch(query, numberOfSimilarityResults);
-                    allResults.push(...contentResults);
-                } catch (error) {
-                    console.error(`Error processing content for ${content.link}:`, error);
-                }
-            }
+  const allResults: DocumentInterface[] = [];
+  try {
+    for (let i = 0; i < contents.length; i++) {
+      const content = contents[i];
+      if (content.html.length > 0) {
+        try {
+          const splitText = await new RecursiveCharacterTextSplitter({
+            chunkSize: textChunkSize,
+            chunkOverlap: textChunkOverlap,
+          }).splitText(content.html);
+          const vectorStore = await MemoryVectorStore.fromTexts(
+            splitText,
+            { title: content.title, link: content.link },
+            embeddings
+          );
+          const contentResults = await vectorStore.similaritySearch(
+            query,
+            numberOfSimilarityResults
+          );
+          allResults.push(...contentResults);
+        } catch (error) {
+          console.error(`Error processing content for ${content.link}:`, error);
         }
-        return allResults;
-    } catch (error) {
-        console.error('Error processing and vectorizing content:', error);
-        throw error;
+      }
     }
-}
+    return allResults;
+  } catch (error) {
+    console.error("Error processing and vectorizing content:", error);
+    return [];
+  }