From 405b52a7153ad70dd97ce0d4dc20b0f904b1bfdd Mon Sep 17 00:00:00 2001
From: "craigbidenbot[bot]"
 <260814249+craigbidenbot[bot]@users.noreply.github.com>
Date: Sat, 28 Mar 2026 22:07:52 +0000
Subject: [PATCH] fix: add User-Agent to server-side probes for Cloudflare
 compatibility

Some endpoints behind Cloudflare Bot Fight Mode block requests without a
recognizable User-Agent, causing registration to fail with 'No valid x402
response found' even though the endpoint returns valid 402 challenges.

- Send 'x402scan/1.0 (+https://x402scan.com; server-probe)' on all probes
- Include cause details in probe error messages for better diagnostics

Closes #742
---
 apps/scan/src/lib/discovery/probe.ts           | 10 +++++++---
 apps/scan/src/lib/discovery/utils/constants.ts | 11 +++++++++++
 apps/scan/src/lib/discovery/utils/index.ts     |  2 +-
 3 files changed, 19 insertions(+), 4 deletions(-)

diff --git a/apps/scan/src/lib/discovery/probe.ts b/apps/scan/src/lib/discovery/probe.ts
index 0ac527682..655845ed8 100644
--- a/apps/scan/src/lib/discovery/probe.ts
+++ b/apps/scan/src/lib/discovery/probe.ts
@@ -3,7 +3,7 @@ import type {
   AuditWarning,
   EndpointMethodAdvisory,
 } from '@agentcash/discovery';
-import { PROBE_TIMEOUT_MS } from './utils';
+import { PROBE_TIMEOUT_MS, PROBE_USER_AGENT } from './utils';
 
 export type ProbeX402Result =
   | {
@@ -28,17 +28,21 @@ export async function probeX402Endpoint(
     url,
     probe: true,
     signal: AbortSignal.timeout(PROBE_TIMEOUT_MS),
+    headers: { 'User-Agent': PROBE_USER_AGENT },
   });
 
   if (!result.found) {
-    const causeMessages = {
+    const causeMessages: Record<string, string> = {
       not_found: 'Endpoint did not return a 402 payment challenge',
       network: 'Network error reaching endpoint',
       timeout: 'Endpoint timed out',
     };
+    const base =
+      causeMessages[result.cause] ?? `Probe failed (${result.cause})`;
+    const detail = result.message ? `${base}: ${result.message}` : base;
     return {
       success: false,
-      error: result.message ?? causeMessages[result.cause],
+      error: detail,
     };
   }
 
diff --git a/apps/scan/src/lib/discovery/utils/constants.ts b/apps/scan/src/lib/discovery/utils/constants.ts
index bfa4ca2bb..c08a848f2 100644
--- a/apps/scan/src/lib/discovery/utils/constants.ts
+++ b/apps/scan/src/lib/discovery/utils/constants.ts
@@ -1,2 +1,13 @@
 /** Default timeout for probing a live endpoint via checkEndpointSchema. */
 export const PROBE_TIMEOUT_MS = 15000;
+
+/**
+ * User-Agent sent with server-side probes.
+ *
+ * Some endpoints sit behind Cloudflare Bot Fight Mode or similar WAFs that
+ * block requests without a recognizable User-Agent. Using a well-known
+ * identifier lets server operators allowlist x402scan's crawler while still
+ * looking like a legitimate HTTP client.
+ */
+export const PROBE_USER_AGENT =
+  'x402scan/1.0 (+https://x402scan.com; server-probe)';
diff --git a/apps/scan/src/lib/discovery/utils/index.ts b/apps/scan/src/lib/discovery/utils/index.ts
index 9758cbd25..4354c4754 100644
--- a/apps/scan/src/lib/discovery/utils/index.ts
+++ b/apps/scan/src/lib/discovery/utils/index.ts
@@ -1,3 +1,3 @@
 export { isX402PaymentOption } from './is-x402-option';
 export { getRegistrationErrorMessage } from './registration-error-message';
-export { PROBE_TIMEOUT_MS } from './constants';
+export { PROBE_TIMEOUT_MS, PROBE_USER_AGENT } from './constants';