Add explicit AI crawler allowlist to docs robots.txt (#43599)

Author: Copilot

docs/public/robots.txt

@@ -1,89 +1,31 @@
-# Allow all crawlers by default
 User-agent: *
 Allow: /
 
-# Explicitly list major AI crawlers for clarity and signaling.
-# The wildcard rule above already permits all bots; the entries below
-# make it unambiguous to AI indexers that they are welcome here.
 User-agent: GPTBot
 Allow: /
 
-User-agent: ChatGPT-User
-Allow: /
-
 User-agent: OAI-SearchBot
 Allow: /
 
-User-agent: Google-Extended
+User-agent: ChatGPT-User
 Allow: /
 
-User-agent: Googlebot
+User-agent: anthropic-ai
 Allow: /
 
 User-agent: ClaudeBot
 Allow: /
 
-User-agent: Claude-Web
-Allow: /
-
-User-agent: anthropic-ai
-Allow: /
-
 User-agent: PerplexityBot
 Allow: /
 
 User-agent: Perplexity-User
 Allow: /
 
-User-agent: Applebot
-Allow: /
-
-User-agent: Applebot-Extended
-Allow: /
-
-User-agent: Amazonbot
-Allow: /
-
-User-agent: cohere-ai
-Allow: /
-
-User-agent: Omgilibot
-Allow: /
-
-User-agent: FacebookBot
-Allow: /
-
-User-agent: Bytespider
-Allow: /
-
-User-agent: bingbot
-Allow: /
-
-User-agent: BingPreview
-Allow: /
-
-User-agent: DuckDuckBot
-Allow: /
-
-User-agent: Slurp
-Allow: /
-
-User-agent: YandexBot
-Allow: /
-
-User-agent: Baiduspider
-Allow: /
-
-User-agent: ia_archiver
-Allow: /
-
-User-agent: CCBot
-Allow: /
-
-User-agent: DataForSeoBot
+User-agent: Google-Extended
 Allow: /
 
-User-agent: Diffbot
+User-agent: Google-CloudVertexBot
 Allow: /
 
-Sitemap: https://github.github.com/gh-aw/sitemap.xml
+Sitemap: https://github.github.com/gh-aw/sitemap-index.xml

docs/tests/robots-txt.spec.ts

@@ -0,0 +1,47 @@
+import { test, expect } from '@playwright/test';
+
+const EXPECTED_ROBOTS_TXT = [
+  'User-agent: *',
+  'Allow: /',
+  '',
+  'User-agent: GPTBot',
+  'Allow: /',
+  '',
+  'User-agent: OAI-SearchBot',
+  'Allow: /',
+  '',
+  'User-agent: ChatGPT-User',
+  'Allow: /',
+  '',
+  'User-agent: anthropic-ai',
+  'Allow: /',
+  '',
+  'User-agent: ClaudeBot',
+  'Allow: /',
+  '',
+  'User-agent: PerplexityBot',
+  'Allow: /',
+  '',
+  'User-agent: Perplexity-User',
+  'Allow: /',
+  '',
+  'User-agent: Google-Extended',
+  'Allow: /',
+  '',
+  'User-agent: Google-CloudVertexBot',
+  'Allow: /',
+  '',
+  'Sitemap: https://github.github.com/gh-aw/sitemap-index.xml',
+  '',
+].join('\n');
+
+test.describe('robots.txt', () => {
+  test('should contain only the expected AI crawler directives and sitemap index', async ({ request }) => {
+    const response = await request.get('/gh-aw/robots.txt');
+    expect(response.ok()).toBeTruthy();
+
+    const body = (await response.text()).replace(/\r\n/g, '\n');
+
+    expect(body).toBe(EXPECTED_ROBOTS_TXT);
+  });
+});