Retry DataForSEO Google AI Overview requests on transient server errors

.changeset/retry-dataforseo-ai-overview.md

@@ -0,0 +1,5 @@
+---
+"@elmohq/cli": patch
+---
+
+Google AI Overview tracking via DataForSEO now retries transient server errors instead of failing the request.

packages/lib/src/providers/registry/dataforseo.test.ts

@@ -50,8 +50,30 @@ import { dataforseo } from "./dataforseo";
 afterEach(() => {
 	vi.clearAllMocks();
 	vi.unstubAllGlobals();
+	vi.useRealTimers();
 });
 
+const AI_OVERVIEW_OK = {
+	tasks: [
+		{
+			status_code: 20000,
+			status_message: "Ok.",
+			result: [
+				{
+					items: [
+						{ type: "organic", title: "some result" },
+						{
+							type: "ai_overview",
+							markdown: "The Sonos Era 300 is a well-reviewed speaker released recently.",
+							references: [{ url: "https://www.whathifi.com/reviews/sonos-era-300", title: "Sonos Era 300 review" }],
+						},
+					],
+				},
+			],
+		},
+	],
+};
+
 describe("dataforseo provider", () => {
 	it("rejects prompts longer than DataForSEO's 500 character limit before calling the API", async () => {
 		await expect(dataforseo.run("chatgpt", "x".repeat(501), { webSearch: true })).rejects.toThrow(
@@ -104,28 +126,7 @@ describe("dataforseo provider", () => {
 	});
 
 	it("fetches Google AI Overview from the organic SERP endpoint with async loading on", async () => {
-		dataforseoClient.googleOrganicLiveAdvanced.mockResolvedValueOnce({
-			tasks: [
-				{
-					status_code: 20000,
-					status_message: "Ok.",
-					result: [
-						{
-							items: [
-								{ type: "organic", title: "some result" },
-								{
-									type: "ai_overview",
-									markdown: "The Sonos Era 300 is a well-reviewed speaker released recently.",
-									references: [
-										{ url: "https://www.whathifi.com/reviews/sonos-era-300", title: "Sonos Era 300 review" },
-									],
-								},
-							],
-						},
-					],
-				},
-			],
-		});
+		dataforseoClient.googleOrganicLiveAdvanced.mockResolvedValueOnce(AI_OVERVIEW_OK);
 
 		const result = await dataforseo.run("google-ai-overview", "What is a well-reviewed speaker released last month?", {
 			webSearch: true,
@@ -144,6 +145,41 @@ describe("dataforseo provider", () => {
 		expect(result.webQueries).toEqual(["unavailable"]);
 	});
 
+	it("retries the AI Overview request when DataForSEO returns a transient server error", async () => {
+		vi.useFakeTimers();
+		dataforseoClient.googleOrganicLiveAdvanced
+			.mockResolvedValueOnce({
+				tasks: [{ status_code: 40602, status_message: "Internal SE Server Error.", result: null }],
+			})
+			.mockResolvedValueOnce(AI_OVERVIEW_OK);
+
+		const promise = dataforseo.run("google-ai-overview", "What is a well-reviewed speaker released last month?", {
+			webSearch: true,
+		});
+		await vi.runAllTimersAsync();
+		const result = await promise;
+
+		expect(dataforseoClient.googleOrganicLiveAdvanced).toHaveBeenCalledTimes(2);
+		expect(result.textContent).toContain("Sonos Era 300");
+		expect(result.citations).toHaveLength(1);
+	});
+
+	it("throws after exhausting retries when every AI Overview attempt fails", async () => {
+		vi.useFakeTimers();
+		dataforseoClient.googleOrganicLiveAdvanced.mockResolvedValue({
+			tasks: [{ status_code: 40602, status_message: "Internal SE Server Error.", result: null }],
+		});
+
+		const promise = dataforseo.run("google-ai-overview", "What is a well-reviewed speaker released last month?", {
+			webSearch: true,
+		});
+		const assertion = expect(promise).rejects.toThrow("DataForSEO API Error: Internal SE Server Error.");
+		await vi.runAllTimersAsync();
+		await assertion;
+
+		expect(dataforseoClient.googleOrganicLiveAdvanced).toHaveBeenCalledTimes(3);
+	});
+
 	it("resolves Gemini Vertex grounding-redirect citation URLs to the real source", async () => {
 		const realUrl = "https://www.whathifi.com/best-buys/hi-fi/best-hi-fi-speakers";
 		const redirectUrl = "https://vertexaisearch.cloud.google.com/grounding-api-redirect/ABC123";

packages/lib/src/providers/registry/dataforseo.ts

@@ -133,27 +133,34 @@ async function runGoogleAiOverview(prompt: string): Promise<ScrapeResult> {
 		load_async_ai_overview: true,
 	});
 
-	const response = await api.googleOrganicLiveAdvanced([requestInfo]);
-
-	if (!response?.tasks?.length) {
-		throw new Error(`DataForSEO API Error: No response or tasks.`);
-	}
-
-	const task = response.tasks[0];
-	if (task.status_code !== 20000 || !task.result?.length) {
-		throw new Error(`DataForSEO API Error: ${task.status_message}`);
+	// Loading the AI Overview asynchronously intermittently fails on DataForSEO's
+	// side with a task-level "Internal SE Server Error"; a couple of retries clear
+	// it, so a transient blip doesn't fail the run (matches the BrightData AI
+	// Overview runner).
+	let lastError = "No response or tasks.";
+	for (let attempt = 0; attempt < 3; attempt++) {
+		try {
+			const response = await api.googleOrganicLiveAdvanced([requestInfo]);
+			const task = response?.tasks?.[0];
+			if (task?.status_code === 20000 && task.result?.length) {
+				// The SERP response carries the AI Overview as an items[].type
+				// "ai_overview" element, which the shared Google extractors understand.
+				const citations = extractCitationsFromGoogle(response);
+				return {
+					rawOutput: sanitizeForJson(response),
+					webQueries: citations.length > 0 ? [WEB_QUERIES_UNAVAILABLE] : [],
+					textContent: extractTextFromGoogle(response),
+					citations,
+					modelVersion: "dataforseo",
+				};
+			}
+			lastError = task?.status_message ?? "No response or tasks.";
+		} catch (error) {
+			lastError = error instanceof Error ? error.message : String(error);
+		}
+		if (attempt < 2) await new Promise((resolve) => setTimeout(resolve, 1500 * (attempt + 1)));
 	}
-
-	// The SERP response carries the AI Overview as an items[].type "ai_overview"
-	// element, which the shared Google extractors already understand.
-	const citations = extractCitationsFromGoogle(response);
-	return {
-		rawOutput: sanitizeForJson(response),
-		webQueries: citations.length > 0 ? [WEB_QUERIES_UNAVAILABLE] : [],
-		textContent: extractTextFromGoogle(response),
-		citations,
-		modelVersion: "dataforseo",
-	};
+	throw new Error(`DataForSEO API Error: ${lastError}`);
 }
 
 /**