feat(publisher): generate sitemap.xml and robots.txt

docs/features/publisher.md

@@ -56,6 +56,7 @@ server/publish/
 ├── mediaPresentation.ts            — media URL materialization for originals + responsive variants
 ├── renderTreeWalk.ts               — walkRenderTree: visits every node that contributes to a rendered page (page nodes + VC definition trees, cycle-guarded); single source of truth for loop-prefetch and media-prefetch
 ├── mediaPrefetch.ts, loopPrefetch.ts — pre-warm caches needed by the renderer
+├── sitemap.ts                      — generated /sitemap.xml + /robots.txt (enumerate published routes)
 ├── republish.ts                    — bulk re-publish on site-level changes
 ├── publishScheduler.ts             — scheduled publish jobs
 ├── runtime/                        — per-site bun install workspace serving
@@ -365,6 +366,7 @@ Because `serializeCsp` sorts, the same plugins + adapters always emit a **byte-i
 | File                                            | Role                                                                |
 |-------------------------------------------------|---------------------------------------------------------------------|
 | `server/publish/publicRouter.ts`                | Gateway: Layer A disk fast-path → Layer B LRU → live `resolvePublicRoute` + `renderPublicResolution`. |
+| `server/publish/sitemap.ts`                     | Generated `GET /sitemap.xml` + `GET /robots.txt`. Enumerates published pages (excluding template pages) + data-row routes from the DB on each request, anchored to the request origin. Owned before `tryServePublicRoute` so these paths can't be shadowed by a same-slug page. |
 | `server/publish/staticArtefact.ts`              | Two-slot symlink swap (`swapSlot`), per-file atomic writes (`writeArtefact`, `updateArtefactInPlace`), and reads (`readArtefact`). Layer A. |
 | `server/publish/renderCache.ts`                 | In-memory LRU keyed by `(urlPath, canonicalQuery)`, entries versioned. `getOrRender` (single-flight). Reads the version from `publishState`; version captured at render start — a publish landing mid-render discards the result rather than caching stale HTML. Layer B. |
 | `server/publish/publishState.ts`                | Publish-time process state: `publishVersion` (`bumpPublishVersion`/`getPublishVersion`), `withPublishLock` (ISS-038 publish serializer), and `createVersionedSingleFlight` — the generalized version-keyed single-flight memo the hole endpoint reuses. Repositories import the version + lock from here (not from the cache). |
@@ -388,6 +390,37 @@ Because `serializeCsp` sorts, the same plugins + adapters always emit a **byte-i
 | `server/handlers/cms/moduleJs.ts`               | `GET /_instatic/module-js/<moduleId>.js?v=<publishVersion>` — serves a module's render-emitted JS from the memoised site map; validates the untrusted moduleId segment; 404 unknown; `text/javascript`; `cache-control: public, max-age=3600`. |
 | `server/richtextSanitizer.ts`                   | Installs the server's happy-dom-backed DOMPurify runtime without global DOM objects. |
 
+### Sitemap & robots.txt
+
+`sitemap.ts` owns two reserved public routes, registered in `server/router.ts`
+as `tryServeSitemap` **before** `tryServePublicRoute` (so a published page with
+slug `sitemap.xml`/`robots.txt` can never shadow them), and opted into the Vite
+dev proxy in `vite.config.ts` (both carry a file extension, so — like the
+`/_instatic/` routes — they need an explicit allow past the extension-rejection
+rule):
+
+- `GET /sitemap.xml` — a `urlset` of every published, **directly-routable** URL.
+  `collectSitemapEntries(db)` unions:
+  - Published `pages` rows (`listPublishedPageRoutes`), keyed by `data_rows.slug`
+    (the exact slug the resolver matches; the `index` slug → `/`), **minus**
+    template pages — a page id flagged `isTemplatePage` in the latest published
+    `SiteDocument` is dropped, since entry templates / layouts / the notFound
+    page are never routable.
+  - Published data-row routes (`listPublishedRowRoutes` → `publicDataPath`),
+    using each row's active-version slug.
+  Pages win on a path collision (mirrors resolver order). Each `<url>` carries a
+  `<lastmod>` from the row's `updated_at`.
+- `GET /robots.txt` — allow-all except `/admin`, plus a `Sitemap:` line.
+
+Both are rebuilt from the DB per request (crawled infrequently, cheap to
+recompute; `Cache-Control: public, max-age=3600`) and anchored to
+`canonicalPublicOrigin(url)` (`server/auth/security.ts`): the configured
+`PUBLIC_ORIGIN` entry whose host matches the request wins (so a
+TLS-terminating edge that hands the container plain HTTP still yields
+`https://` locs), else the canonical first entry, else the request's own
+origin. The pure builders (`buildSitemapXml`, `buildRobotsTxt`), enumeration,
+and origin anchoring are gated by `src/__tests__/server/sitemap.test.ts`.
+
 ### `publishedHtmlPipeline.ts` — the plugin filter point
 
 After `publishPage` returns, the server runs:

server/auth/security.ts

@@ -100,6 +100,22 @@ export function publicOriginIsHttps(): boolean {
   return configured?.startsWith('https://') ?? false
 }
 
+/**
+ * Canonical absolute origin for URLs embedded in generated public documents
+ * (sitemap `<loc>`, robots `Sitemap:`). Behind a TLS-terminating edge the
+ * request URL's scheme is plain http, so raw `url.origin` would leak
+ * `http://` URLs into crawler-facing output; the configured public origins
+ * are authoritative instead. A configured entry whose host matches the
+ * request host wins (multi-domain installs emit the domain the crawler
+ * actually fetched, upgraded to its configured scheme), else the canonical
+ * first entry, else — nothing configured — the request's own origin.
+ */
+export function canonicalPublicOrigin(url: URL): string {
+  const host = url.host.toLowerCase()
+  const match = publicOrigins.find((origin) => new URL(origin).host === host)
+  return match ?? publicOrigins[0] ?? url.origin
+}
+
 /**
  * True when the request's `Origin` header is acceptable for a state-changing
  * action. The check is a CSRF defense-in-depth on top of `SameSite=Lax`:

server/publish/sitemap.ts

@@ -0,0 +1,159 @@
+/**
+ * Sitemap + robots.txt for the published site.
+ *
+ * Two reserved public routes, owned before the public-slug resolver:
+ *
+ *   - `GET /sitemap.xml` — a `urlset` of every published, directly-routable
+ *     URL (standalone pages + content-row routes), each with a `<lastmod>`.
+ *   - `GET /robots.txt`  — allow-all except `/admin`, plus a `Sitemap:` line.
+ *
+ * Both are derived from the published database on each request (sitemaps are
+ * crawled infrequently and the response is cheap to rebuild). Absolute URLs
+ * are anchored to `canonicalPublicOrigin` — the configured public origin
+ * matching the request host (falling back to the request's own origin when
+ * none is configured), so a TLS-terminating edge that hands the container
+ * plain HTTP can never leak `http://` locs into crawler-facing output.
+ *
+ * The route set mirrors what the public router (`publicRouter.ts`) will
+ * actually serve a 200 for:
+ *   - Published `pages` rows, EXCLUDING template pages (entry templates /
+ *     layouts / the notFound page are never directly routable), keyed by
+ *     `data_rows.slug` — the exact slug the resolver matches.
+ *   - Published data-row routes (`/<table-route>/<row-slug>`), via
+ *     `listPublishedRowRoutes` — the same list the full publish bakes.
+ *
+ * Template-page exclusion reuses the published `SiteDocument`: a page id present
+ * in the snapshot and flagged `isTemplatePage` is dropped. A published page that
+ * isn't in the latest snapshot (an unusual incremental-publish edge) is kept —
+ * it resolves by slug, so omitting it would hide a live URL.
+ */
+
+import type { DbClient } from '../db/client'
+import { canonicalPublicOrigin } from '../auth/security'
+import { isTemplatePage } from '@core/templates'
+import {
+  getLatestPublishedSiteSnapshot,
+  listPublishedPageRoutes,
+} from '../repositories/publish'
+import { listPublishedRowRoutes, publicDataPath } from '../repositories/data/publish'
+
+const SITEMAP_PATH = '/sitemap.xml'
+const ROBOTS_PATH = '/robots.txt'
+
+/** One entry in the sitemap: an absolute-path route plus its last-modified time. */
+export interface SitemapEntry {
+  /** Root-relative URL path, e.g. `/`, `/about`, `/posts/hello`. */
+  path: string
+  /** ISO 8601 last-modified timestamp (W3C datetime) for `<lastmod>`. */
+  lastmod: string
+}
+
+/**
+ * Convert a published page's stored slug to its public URL path. Mirrors the
+ * resolver's `publicSlugFromPath` in reverse: the canonical `index` slug is the
+ * site root, everything else is `/<slug>`. Stray leading/trailing slashes on the
+ * stored slug are normalised away so the emitted `<loc>` is canonical.
+ */
+export function pageSlugToPath(slug: string): string {
+  const trimmed = slug.replace(/^\/+|\/+$/g, '')
+  return trimmed === '' || trimmed === 'index' ? '/' : `/${trimmed}`
+}
+
+/**
+ * Enumerate every published, directly-routable URL for the sitemap. Pages win
+ * over content rows on a path collision (the resolver looks up pages first), so
+ * a row route that resolves to the same path as a page is dropped.
+ */
+export async function collectSitemapEntries(db: DbClient): Promise<SitemapEntry[]> {
+  const [snapshot, pageRoutes, rowRoutes] = await Promise.all([
+    getLatestPublishedSiteSnapshot(db),
+    listPublishedPageRoutes(db),
+    listPublishedRowRoutes(db),
+  ])
+
+  const templatePageIds = new Set<string>(
+    (snapshot?.site.pages ?? []).filter(isTemplatePage).map((page) => page.id),
+  )
+
+  const entries: SitemapEntry[] = []
+  const seen = new Set<string>()
+
+  for (const route of pageRoutes) {
+    if (templatePageIds.has(route.pageId)) continue
+    const path = pageSlugToPath(route.slug)
+    if (seen.has(path)) continue
+    seen.add(path)
+    entries.push({ path, lastmod: route.updatedAt })
+  }
+
+  for (const route of rowRoutes) {
+    const path = publicDataPath(route.tableRouteBase, route.rowSlug)
+    if (seen.has(path)) continue
+    seen.add(path)
+    entries.push({ path, lastmod: route.updatedAt })
+  }
+
+  return entries
+}
+
+/** XML-escape a value for use inside `<loc>` / element text. */
+function escapeXml(value: string): string {
+  return value
+    .replace(/&/g, '&amp;')
+    .replace(/</g, '&lt;')
+    .replace(/>/g, '&gt;')
+    .replace(/"/g, '&quot;')
+    .replace(/'/g, '&apos;')
+}
+
+/**
+ * Build the `sitemap.xml` document. Pure: `origin` is the scheme+host the loc
+ * URLs are anchored to (no trailing slash), `entries` are root-relative paths.
+ */
+export function buildSitemapXml(origin: string, entries: readonly SitemapEntry[]): string {
+  const urls = entries
+    .map((entry) => {
+      const loc = escapeXml(`${origin}${entry.path}`)
+      return `  <url>\n    <loc>${loc}</loc>\n    <lastmod>${escapeXml(entry.lastmod)}</lastmod>\n  </url>`
+    })
+    .join('\n')
+  return `<?xml version="1.0" encoding="UTF-8"?>\n<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">\n${urls}\n</urlset>\n`
+}
+
+/** Build `robots.txt`: allow all except `/admin`, and advertise the sitemap. */
+export function buildRobotsTxt(origin: string): string {
+  return `User-agent: *\nDisallow: /admin\n\nSitemap: ${origin}${SITEMAP_PATH}\n`
+}
+
+/**
+ * Own the two reserved SEO routes. Returns `null` for anything else (or a
+ * non-GET request) so the dispatcher keeps walking.
+ */
+export async function handleSitemapRequest(
+  req: Request,
+  url: URL,
+  ctx: { db: DbClient },
+): Promise<Response | null> {
+  if (req.method !== 'GET') return null
+
+  if (url.pathname === SITEMAP_PATH) {
+    const entries = await collectSitemapEntries(ctx.db)
+    return new Response(buildSitemapXml(canonicalPublicOrigin(url), entries), {
+      headers: {
+        'content-type': 'application/xml; charset=utf-8',
+        'cache-control': 'public, max-age=3600',
+      },
+    })
+  }
+
+  if (url.pathname === ROBOTS_PATH) {
+    return new Response(buildRobotsTxt(canonicalPublicOrigin(url)), {
+      headers: {
+        'content-type': 'text/plain; charset=utf-8',
+        'cache-control': 'public, max-age=3600',
+      },
+    })
+  }
+
+  return null
+}

server/repositories/data/publish.ts

@@ -294,26 +294,30 @@ interface PublishedRowRoute {
   rowSlug: string
   tableSlug: string
   tableRouteBase: string
+  /** ISO timestamp of the row's last change — becomes the sitemap `<lastmod>`. */
+  updatedAt: string
 }
 
 /**
  * Every published, non-deleted data row (excluding the `pages` table) with
  * its active version's slug and its table's route info. The full publish uses
  * this to bake a Layer A artefact for each row route into the fresh slot —
  * without it, the slot swap would strand every row artefact written by
- * incremental publishes.
+ * incremental publishes. The sitemap builder consumes the same list.
  */
 export async function listPublishedRowRoutes(db: DbClient): Promise<PublishedRowRoute[]> {
   const { rows } = await db<{
     row_id: string
     row_slug: string
     table_slug: string
     table_route_base: string
+    updated_at: string | Date
   }>`
     select data_rows.id as row_id,
            data_row_versions.slug as row_slug,
            data_tables.slug as table_slug,
-           data_tables.route_base as table_route_base
+           data_tables.route_base as table_route_base,
+           data_rows.updated_at as updated_at
     from data_rows
     join data_tables on data_tables.id = data_rows.table_id
     join data_row_versions on data_row_versions.id = data_rows.active_version_id
@@ -328,6 +332,7 @@ export async function listPublishedRowRoutes(db: DbClient): Promise<PublishedRow
     rowSlug: row.row_slug,
     tableSlug: row.table_slug,
     tableRouteBase: normalizeRouteBase(row.table_route_base),
+    updatedAt: isoDate(row.updated_at),
   }))
 }
 

server/repositories/publish.ts

@@ -19,10 +19,12 @@
  *   getPublishedPageBySlug    — look up a published page snapshot by slug
  *   getPublishedPageSnapshotById — same, by page row id
  *   getLatestPublishedSiteSnapshot — first published page snapshot (for 404s etc.)
+ *   listPublishedPageRoutes   — every published page's slug + lastmod (for the sitemap)
  *   getDraftPublishStatus     — compare draft vs published state for the UI
  */
 import { createHash } from 'node:crypto'
 import type { DataRow } from '@core/data/schemas'
+import { isoDate } from '@core/utils/isoDate'
 import type { SiteDocument } from '@core/page-tree'
 import type { PublishedPageRuntimeAssets } from '@core/site-runtime'
 import type { PublishedRuntimePackageImportmap } from '@core/publisher'
@@ -370,3 +372,35 @@ export async function getLatestPublishedSiteSnapshot(
   const row = rows[0]
   return row ? snapshotFromQueryRow({ ...row, runtime_assets_json: null }) : null
 }
+
+export interface PublishedPageRoute {
+  /** id of the `data_rows` row — used to locate the page in the site snapshot. */
+  pageId: string
+  /** Public URL slug (`data_rows.slug` — what the public router matches). */
+  slug: string
+  /** ISO timestamp of the row's last change — becomes the sitemap `<lastmod>`. */
+  updatedAt: string
+}
+
+/**
+ * Every published, non-deleted page row (`table_id = 'pages'`) with its public
+ * slug and last-updated timestamp. Lightweight — it does NOT pull `cells_json`
+ * or the site snapshot, so template-page filtering is left to the caller (which
+ * already holds the published `SiteDocument` and checks `isTemplatePage`). Used
+ * to enumerate routes for the sitemap.
+ */
+export async function listPublishedPageRoutes(db: DbClient): Promise<PublishedPageRoute[]> {
+  const { rows } = await db<{ page_id: string; slug: string; updated_at: string | Date }>`
+    select id as page_id, slug, updated_at
+    from data_rows
+    where table_id = 'pages'
+      and status = 'published'
+      and deleted_at is null
+    order by created_at asc
+  `
+  return rows.map((row) => ({
+    pageId: row.page_id,
+    slug: row.slug,
+    updatedAt: isoDate(row.updated_at),
+  }))
+}

server/router.ts

@@ -4,6 +4,7 @@ import { tryHandleMcpOAuth } from './ai/mcp/oauth/handler'
 import { handleCmsRequest } from './handlers/cms'
 import type { DbClient } from './db/client'
 import { renderNotFoundResponse, renderPublicResolution } from './publish/publicRouter'
+import { handleSitemapRequest } from './publish/sitemap'
 import { readStaticAsset } from './publish/staticArtefact'
 import { getLatestSnapshotForVersion } from './publish/publishedSnapshotCache'
 import { getPublishVersion, registerVersionedCacheReset } from './publish/publishState'
@@ -91,6 +92,7 @@ const routes: readonly RouteHandler[] = [
   tryServeStaticAsset,
   tryServeUpload,
   tryServeAdminApp,
+  tryServeSitemap,
   tryServePublicRoute,
   trySetupRedirect,
   tryServeNotFoundPage,
@@ -484,6 +486,17 @@ async function tryServePublicRoute(req: Request, runtime: ServerRuntime, url: UR
   return await renderPublicResolution(runtime.db, url, runtime.uploadsDir)
 }
 
+/**
+ * Reserved SEO routes — `/sitemap.xml` and `/robots.txt`. Placed before
+ * `tryServePublicRoute` so these paths are always the generated documents and
+ * can't be shadowed by a published page that happens to use the same slug.
+ * Both are derived from the published DB on each request, anchored to the
+ * request's own origin. Resolution lives in `server/publish/sitemap.ts`.
+ */
+async function tryServeSitemap(req: Request, runtime: ServerRuntime, url: URL, _pathname: string): Promise<Response | null> {
+  return await handleSitemapRequest(req, url, { db: runtime.db })
+}
+
 /**
  * On a fresh install with no admin user yet, bounce the visitor to /admin so
  * they land in the setup wizard instead of seeing a confusing 404. Returns