diff --git a/docs/features/publisher.md b/docs/features/publisher.md index c87e8eb12..0e094d65d 100644 --- a/docs/features/publisher.md +++ b/docs/features/publisher.md @@ -56,6 +56,7 @@ server/publish/ ├── mediaPresentation.ts — media URL materialization for originals + responsive variants ├── renderTreeWalk.ts — walkRenderTree: visits every node that contributes to a rendered page (page nodes + VC definition trees, cycle-guarded); single source of truth for loop-prefetch and media-prefetch ├── mediaPrefetch.ts, loopPrefetch.ts — pre-warm caches needed by the renderer +├── sitemap.ts — generated /sitemap.xml + /robots.txt (enumerate published routes) ├── republish.ts — bulk re-publish on site-level changes ├── publishScheduler.ts — scheduled publish jobs ├── runtime/ — per-site bun install workspace serving @@ -365,6 +366,7 @@ Because `serializeCsp` sorts, the same plugins + adapters always emit a **byte-i | File | Role | |-------------------------------------------------|---------------------------------------------------------------------| | `server/publish/publicRouter.ts` | Gateway: Layer A disk fast-path → Layer B LRU → live `resolvePublicRoute` + `renderPublicResolution`. | +| `server/publish/sitemap.ts` | Generated `GET /sitemap.xml` + `GET /robots.txt`. Enumerates published pages (excluding template pages) + data-row routes from the DB on each request, anchored to the request origin. Owned before `tryServePublicRoute` so these paths can't be shadowed by a same-slug page. | | `server/publish/staticArtefact.ts` | Two-slot symlink swap (`swapSlot`), per-file atomic writes (`writeArtefact`, `updateArtefactInPlace`), and reads (`readArtefact`). Layer A. | | `server/publish/renderCache.ts` | In-memory LRU keyed by `(urlPath, canonicalQuery)`, entries versioned. `getOrRender` (single-flight). Reads the version from `publishState`; version captured at render start — a publish landing mid-render discards the result rather than caching stale HTML. Layer B. | | `server/publish/publishState.ts` | Publish-time process state: `publishVersion` (`bumpPublishVersion`/`getPublishVersion`), `withPublishLock` (ISS-038 publish serializer), and `createVersionedSingleFlight` — the generalized version-keyed single-flight memo the hole endpoint reuses. Repositories import the version + lock from here (not from the cache). | @@ -388,6 +390,37 @@ Because `serializeCsp` sorts, the same plugins + adapters always emit a **byte-i | `server/handlers/cms/moduleJs.ts` | `GET /_instatic/module-js/.js?v=` — serves a module's render-emitted JS from the memoised site map; validates the untrusted moduleId segment; 404 unknown; `text/javascript`; `cache-control: public, max-age=3600`. | | `server/richtextSanitizer.ts` | Installs the server's happy-dom-backed DOMPurify runtime without global DOM objects. | +### Sitemap & robots.txt + +`sitemap.ts` owns two reserved public routes, registered in `server/router.ts` +as `tryServeSitemap` **before** `tryServePublicRoute` (so a published page with +slug `sitemap.xml`/`robots.txt` can never shadow them), and opted into the Vite +dev proxy in `vite.config.ts` (both carry a file extension, so — like the +`/_instatic/` routes — they need an explicit allow past the extension-rejection +rule): + +- `GET /sitemap.xml` — a `urlset` of every published, **directly-routable** URL. + `collectSitemapEntries(db)` unions: + - Published `pages` rows (`listPublishedPageRoutes`), keyed by `data_rows.slug` + (the exact slug the resolver matches; the `index` slug → `/`), **minus** + template pages — a page id flagged `isTemplatePage` in the latest published + `SiteDocument` is dropped, since entry templates / layouts / the notFound + page are never routable. + - Published data-row routes (`listPublishedRowRoutes` → `publicDataPath`), + using each row's active-version slug. + Pages win on a path collision (mirrors resolver order). Each `` carries a + `` from the row's `updated_at`. +- `GET /robots.txt` — allow-all except `/admin`, plus a `Sitemap:` line. + +Both are rebuilt from the DB per request (crawled infrequently, cheap to +recompute; `Cache-Control: public, max-age=3600`) and anchored to +`canonicalPublicOrigin(url)` (`server/auth/security.ts`): the configured +`PUBLIC_ORIGIN` entry whose host matches the request wins (so a +TLS-terminating edge that hands the container plain HTTP still yields +`https://` locs), else the canonical first entry, else the request's own +origin. The pure builders (`buildSitemapXml`, `buildRobotsTxt`), enumeration, +and origin anchoring are gated by `src/__tests__/server/sitemap.test.ts`. + ### `publishedHtmlPipeline.ts` — the plugin filter point After `publishPage` returns, the server runs: diff --git a/server/auth/security.ts b/server/auth/security.ts index dea2bdd1f..a5a9d687d 100644 --- a/server/auth/security.ts +++ b/server/auth/security.ts @@ -100,6 +100,22 @@ export function publicOriginIsHttps(): boolean { return configured?.startsWith('https://') ?? false } +/** + * Canonical absolute origin for URLs embedded in generated public documents + * (sitemap ``, robots `Sitemap:`). Behind a TLS-terminating edge the + * request URL's scheme is plain http, so raw `url.origin` would leak + * `http://` URLs into crawler-facing output; the configured public origins + * are authoritative instead. A configured entry whose host matches the + * request host wins (multi-domain installs emit the domain the crawler + * actually fetched, upgraded to its configured scheme), else the canonical + * first entry, else — nothing configured — the request's own origin. + */ +export function canonicalPublicOrigin(url: URL): string { + const host = url.host.toLowerCase() + const match = publicOrigins.find((origin) => new URL(origin).host === host) + return match ?? publicOrigins[0] ?? url.origin +} + /** * True when the request's `Origin` header is acceptable for a state-changing * action. The check is a CSRF defense-in-depth on top of `SameSite=Lax`: diff --git a/server/publish/sitemap.ts b/server/publish/sitemap.ts new file mode 100644 index 000000000..5e967fcaf --- /dev/null +++ b/server/publish/sitemap.ts @@ -0,0 +1,159 @@ +/** + * Sitemap + robots.txt for the published site. + * + * Two reserved public routes, owned before the public-slug resolver: + * + * - `GET /sitemap.xml` — a `urlset` of every published, directly-routable + * URL (standalone pages + content-row routes), each with a ``. + * - `GET /robots.txt` — allow-all except `/admin`, plus a `Sitemap:` line. + * + * Both are derived from the published database on each request (sitemaps are + * crawled infrequently and the response is cheap to rebuild). Absolute URLs + * are anchored to `canonicalPublicOrigin` — the configured public origin + * matching the request host (falling back to the request's own origin when + * none is configured), so a TLS-terminating edge that hands the container + * plain HTTP can never leak `http://` locs into crawler-facing output. + * + * The route set mirrors what the public router (`publicRouter.ts`) will + * actually serve a 200 for: + * - Published `pages` rows, EXCLUDING template pages (entry templates / + * layouts / the notFound page are never directly routable), keyed by + * `data_rows.slug` — the exact slug the resolver matches. + * - Published data-row routes (`//`), via + * `listPublishedRowRoutes` — the same list the full publish bakes. + * + * Template-page exclusion reuses the published `SiteDocument`: a page id present + * in the snapshot and flagged `isTemplatePage` is dropped. A published page that + * isn't in the latest snapshot (an unusual incremental-publish edge) is kept — + * it resolves by slug, so omitting it would hide a live URL. + */ + +import type { DbClient } from '../db/client' +import { canonicalPublicOrigin } from '../auth/security' +import { isTemplatePage } from '@core/templates' +import { + getLatestPublishedSiteSnapshot, + listPublishedPageRoutes, +} from '../repositories/publish' +import { listPublishedRowRoutes, publicDataPath } from '../repositories/data/publish' + +const SITEMAP_PATH = '/sitemap.xml' +const ROBOTS_PATH = '/robots.txt' + +/** One entry in the sitemap: an absolute-path route plus its last-modified time. */ +export interface SitemapEntry { + /** Root-relative URL path, e.g. `/`, `/about`, `/posts/hello`. */ + path: string + /** ISO 8601 last-modified timestamp (W3C datetime) for ``. */ + lastmod: string +} + +/** + * Convert a published page's stored slug to its public URL path. Mirrors the + * resolver's `publicSlugFromPath` in reverse: the canonical `index` slug is the + * site root, everything else is `/`. Stray leading/trailing slashes on the + * stored slug are normalised away so the emitted `` is canonical. + */ +export function pageSlugToPath(slug: string): string { + const trimmed = slug.replace(/^\/+|\/+$/g, '') + return trimmed === '' || trimmed === 'index' ? '/' : `/${trimmed}` +} + +/** + * Enumerate every published, directly-routable URL for the sitemap. Pages win + * over content rows on a path collision (the resolver looks up pages first), so + * a row route that resolves to the same path as a page is dropped. + */ +export async function collectSitemapEntries(db: DbClient): Promise { + const [snapshot, pageRoutes, rowRoutes] = await Promise.all([ + getLatestPublishedSiteSnapshot(db), + listPublishedPageRoutes(db), + listPublishedRowRoutes(db), + ]) + + const templatePageIds = new Set( + (snapshot?.site.pages ?? []).filter(isTemplatePage).map((page) => page.id), + ) + + const entries: SitemapEntry[] = [] + const seen = new Set() + + for (const route of pageRoutes) { + if (templatePageIds.has(route.pageId)) continue + const path = pageSlugToPath(route.slug) + if (seen.has(path)) continue + seen.add(path) + entries.push({ path, lastmod: route.updatedAt }) + } + + for (const route of rowRoutes) { + const path = publicDataPath(route.tableRouteBase, route.rowSlug) + if (seen.has(path)) continue + seen.add(path) + entries.push({ path, lastmod: route.updatedAt }) + } + + return entries +} + +/** XML-escape a value for use inside `` / element text. */ +function escapeXml(value: string): string { + return value + .replace(/&/g, '&') + .replace(//g, '>') + .replace(/"/g, '"') + .replace(/'/g, ''') +} + +/** + * Build the `sitemap.xml` document. Pure: `origin` is the scheme+host the loc + * URLs are anchored to (no trailing slash), `entries` are root-relative paths. + */ +export function buildSitemapXml(origin: string, entries: readonly SitemapEntry[]): string { + const urls = entries + .map((entry) => { + const loc = escapeXml(`${origin}${entry.path}`) + return ` \n ${loc}\n ${escapeXml(entry.lastmod)}\n ` + }) + .join('\n') + return `\n\n${urls}\n\n` +} + +/** Build `robots.txt`: allow all except `/admin`, and advertise the sitemap. */ +export function buildRobotsTxt(origin: string): string { + return `User-agent: *\nDisallow: /admin\n\nSitemap: ${origin}${SITEMAP_PATH}\n` +} + +/** + * Own the two reserved SEO routes. Returns `null` for anything else (or a + * non-GET request) so the dispatcher keeps walking. + */ +export async function handleSitemapRequest( + req: Request, + url: URL, + ctx: { db: DbClient }, +): Promise { + if (req.method !== 'GET') return null + + if (url.pathname === SITEMAP_PATH) { + const entries = await collectSitemapEntries(ctx.db) + return new Response(buildSitemapXml(canonicalPublicOrigin(url), entries), { + headers: { + 'content-type': 'application/xml; charset=utf-8', + 'cache-control': 'public, max-age=3600', + }, + }) + } + + if (url.pathname === ROBOTS_PATH) { + return new Response(buildRobotsTxt(canonicalPublicOrigin(url)), { + headers: { + 'content-type': 'text/plain; charset=utf-8', + 'cache-control': 'public, max-age=3600', + }, + }) + } + + return null +} diff --git a/server/repositories/data/publish.ts b/server/repositories/data/publish.ts index 9575e9566..47b449aae 100644 --- a/server/repositories/data/publish.ts +++ b/server/repositories/data/publish.ts @@ -294,6 +294,8 @@ interface PublishedRowRoute { rowSlug: string tableSlug: string tableRouteBase: string + /** ISO timestamp of the row's last change — becomes the sitemap ``. */ + updatedAt: string } /** @@ -301,7 +303,7 @@ interface PublishedRowRoute { * its active version's slug and its table's route info. The full publish uses * this to bake a Layer A artefact for each row route into the fresh slot — * without it, the slot swap would strand every row artefact written by - * incremental publishes. + * incremental publishes. The sitemap builder consumes the same list. */ export async function listPublishedRowRoutes(db: DbClient): Promise { const { rows } = await db<{ @@ -309,11 +311,13 @@ export async function listPublishedRowRoutes(db: DbClient): Promise` select data_rows.id as row_id, data_row_versions.slug as row_slug, data_tables.slug as table_slug, - data_tables.route_base as table_route_base + data_tables.route_base as table_route_base, + data_rows.updated_at as updated_at from data_rows join data_tables on data_tables.id = data_rows.table_id join data_row_versions on data_row_versions.id = data_rows.active_version_id @@ -328,6 +332,7 @@ export async function listPublishedRowRoutes(db: DbClient): Promise`. */ + updatedAt: string +} + +/** + * Every published, non-deleted page row (`table_id = 'pages'`) with its public + * slug and last-updated timestamp. Lightweight — it does NOT pull `cells_json` + * or the site snapshot, so template-page filtering is left to the caller (which + * already holds the published `SiteDocument` and checks `isTemplatePage`). Used + * to enumerate routes for the sitemap. + */ +export async function listPublishedPageRoutes(db: DbClient): Promise { + const { rows } = await db<{ page_id: string; slug: string; updated_at: string | Date }>` + select id as page_id, slug, updated_at + from data_rows + where table_id = 'pages' + and status = 'published' + and deleted_at is null + order by created_at asc + ` + return rows.map((row) => ({ + pageId: row.page_id, + slug: row.slug, + updatedAt: isoDate(row.updated_at), + })) +} diff --git a/server/router.ts b/server/router.ts index 6b9ef0dfb..7622f2695 100644 --- a/server/router.ts +++ b/server/router.ts @@ -4,6 +4,7 @@ import { tryHandleMcpOAuth } from './ai/mcp/oauth/handler' import { handleCmsRequest } from './handlers/cms' import type { DbClient } from './db/client' import { renderNotFoundResponse, renderPublicResolution } from './publish/publicRouter' +import { handleSitemapRequest } from './publish/sitemap' import { readStaticAsset } from './publish/staticArtefact' import { getLatestSnapshotForVersion } from './publish/publishedSnapshotCache' import { getPublishVersion, registerVersionedCacheReset } from './publish/publishState' @@ -91,6 +92,7 @@ const routes: readonly RouteHandler[] = [ tryServeStaticAsset, tryServeUpload, tryServeAdminApp, + tryServeSitemap, tryServePublicRoute, trySetupRedirect, tryServeNotFoundPage, @@ -484,6 +486,17 @@ async function tryServePublicRoute(req: Request, runtime: ServerRuntime, url: UR return await renderPublicResolution(runtime.db, url, runtime.uploadsDir) } +/** + * Reserved SEO routes — `/sitemap.xml` and `/robots.txt`. Placed before + * `tryServePublicRoute` so these paths are always the generated documents and + * can't be shadowed by a published page that happens to use the same slug. + * Both are derived from the published DB on each request, anchored to the + * request's own origin. Resolution lives in `server/publish/sitemap.ts`. + */ +async function tryServeSitemap(req: Request, runtime: ServerRuntime, url: URL, _pathname: string): Promise { + return await handleSitemapRequest(req, url, { db: runtime.db }) +} + /** * On a fresh install with no admin user yet, bounce the visitor to /admin so * they land in the setup wizard instead of seeing a confusing 404. Returns diff --git a/src/__tests__/server/sitemap.test.ts b/src/__tests__/server/sitemap.test.ts new file mode 100644 index 000000000..ec8b70cb9 --- /dev/null +++ b/src/__tests__/server/sitemap.test.ts @@ -0,0 +1,187 @@ +import { afterEach, describe, expect, it } from 'bun:test' +import { createTestDb } from '../helpers/createTestDb' +import { + buildRobotsTxt, + buildSitemapXml, + collectSitemapEntries, + handleSitemapRequest, + pageSlugToPath, + type SitemapEntry, +} from '../../../server/publish/sitemap' +import { + canonicalPublicOrigin, + configurePublicOrigins, + resetPublicOrigins, +} from '../../../server/auth/security' +import type { DbClient } from '../../../server/db/client' + +describe('pageSlugToPath', () => { + it('maps the canonical index slug to the site root', () => { + expect(pageSlugToPath('index')).toBe('/') + expect(pageSlugToPath('')).toBe('/') + }) + + it('prefixes other slugs with a single slash and strips stray ones', () => { + expect(pageSlugToPath('about')).toBe('/about') + expect(pageSlugToPath('/about/')).toBe('/about') + expect(pageSlugToPath('blog/post')).toBe('/blog/post') + }) +}) + +describe('buildSitemapXml', () => { + it('emits a urlset with absolute locs and lastmod', () => { + const entries: SitemapEntry[] = [ + { path: '/', lastmod: '2026-07-19T10:00:00.000Z' }, + { path: '/about', lastmod: '2026-07-18T09:00:00.000Z' }, + ] + const xml = buildSitemapXml('https://example.com', entries) + expect(xml).toContain('') + expect(xml).toContain('xmlns="http://www.sitemaps.org/schemas/sitemap/0.9"') + expect(xml).toContain('https://example.com/') + expect(xml).toContain('https://example.com/about') + expect(xml).toContain('2026-07-19T10:00:00.000Z') + }) + + it('xml-escapes special characters in the loc', () => { + const xml = buildSitemapXml('https://example.com', [ + { path: '/a&b', lastmod: '2026-01-01T00:00:00.000Z' }, + ]) + expect(xml).toContain('https://example.com/a&b<c>') + expect(xml).not.toContain('/a&b') + }) +}) + +describe('buildRobotsTxt', () => { + it('allows all except /admin and advertises the sitemap', () => { + const txt = buildRobotsTxt('https://example.com') + expect(txt).toContain('User-agent: *') + expect(txt).toContain('Disallow: /admin') + expect(txt).toContain('Sitemap: https://example.com/sitemap.xml') + }) +}) + +describe('canonicalPublicOrigin', () => { + afterEach(() => { + resetPublicOrigins() + }) + + it('upgrades a host-matched request to its configured origin (https behind a TLS edge)', () => { + configurePublicOrigins(['https://app.example.com', 'https://www.example.com']) + // The edge terminates TLS; the container sees plain http. + expect(canonicalPublicOrigin(new URL('http://www.example.com/sitemap.xml'))).toBe( + 'https://www.example.com', + ) + }) + + it('falls back to the canonical first entry when no host matches', () => { + configurePublicOrigins(['https://www.example.com', 'https://app.example.com']) + expect(canonicalPublicOrigin(new URL('http://internal:8080/sitemap.xml'))).toBe( + 'https://www.example.com', + ) + }) + + it('uses the request origin when nothing is configured', () => { + expect(canonicalPublicOrigin(new URL('http://localhost:3001/sitemap.xml'))).toBe( + 'http://localhost:3001', + ) + }) +}) + +describe('handleSitemapRequest origin anchoring', () => { + afterEach(() => { + resetPublicOrigins() + }) + + it('anchors robots.txt to the canonical public origin, not the request scheme', async () => { + configurePublicOrigins(['https://www.example.com']) + // robots.txt path never touches the db — a bare sentinel suffices. + const db = {} as DbClient + const url = new URL('http://www.example.com/robots.txt') + const res = await handleSitemapRequest(new Request(url), url, { db }) + expect(res).not.toBeNull() + expect(await res!.text()).toContain('Sitemap: https://www.example.com/sitemap.xml') + }) +}) + +describe('collectSitemapEntries', () => { + it('enumerates published pages + row routes, excluding templates/drafts/deleted', async () => { + const { db, cleanup } = await createTestDb() + try { + // One published snapshot flags `p-tmpl` as a template page. + await db` + insert into site_snapshots (id, site_json, content_hash) + values (${'s1'}, ${{ + pages: [ + { id: 'p-index', slug: 'index' }, + { id: 'p-about', slug: 'about' }, + { id: 'p-tmpl', slug: 'post-template', template: { enabled: true } }, + ], + }}, ${'hash'})` + + async function seedPage( + id: string, + slug: string, + status: string, + opts: { deleted?: boolean; createdAt: string; updatedAt: string }, + ): Promise { + // data_rows.active_version_id ↔ data_row_versions.row_id are mutually + // FK-referencing, so seed the row first (null version), then the + // version, then link them — mirrors the real publish write order. + const versionId = `v-${id}` + await db` + insert into data_rows (id, table_id, slug, status, created_at, updated_at, deleted_at) + values (${id}, ${'pages'}, ${slug}, ${status}, ${opts.createdAt}, ${opts.updatedAt}, ${opts.deleted ? '2026-07-19T00:00:00.000Z' : null})` + await db` + insert into data_row_versions (id, row_id, version_number, slug, site_snapshot_id) + values (${versionId}, ${id}, ${1}, ${slug}, ${'s1'})` + await db`update data_rows set active_version_id = ${versionId} where id = ${id}` + } + + await seedPage('p-index', 'index', 'published', { + createdAt: '2026-07-01T00:00:00.000Z', + updatedAt: '2026-07-19T10:00:00.000Z', + }) + await seedPage('p-about', 'about', 'published', { + createdAt: '2026-07-02T00:00:00.000Z', + updatedAt: '2026-07-18T09:00:00.000Z', + }) + await seedPage('p-tmpl', 'post-template', 'published', { + createdAt: '2026-07-03T00:00:00.000Z', + updatedAt: '2026-07-17T00:00:00.000Z', + }) + await seedPage('p-draft', 'secret', 'draft', { + createdAt: '2026-07-04T00:00:00.000Z', + updatedAt: '2026-07-16T00:00:00.000Z', + }) + await seedPage('p-del', 'gone', 'published', { + deleted: true, + createdAt: '2026-07-05T00:00:00.000Z', + updatedAt: '2026-07-15T00:00:00.000Z', + }) + + // A content table with a route base + one published row. + await db` + insert into data_tables (id, name, slug, kind, route_base, singular_label, plural_label) + values (${'blog'}, ${'Blog'}, ${'blog'}, ${'postType'}, ${'/blog'}, ${'Post'}, ${'Posts'})` + await db` + insert into data_rows (id, table_id, slug, status, created_at, updated_at) + values (${'r1'}, ${'blog'}, ${'hello-draft-slug'}, ${'published'}, ${'2026-07-06T00:00:00.000Z'}, ${'2026-07-14T00:00:00.000Z'})` + await db` + insert into data_row_versions (id, row_id, version_number, slug, site_snapshot_id) + values (${'vr1'}, ${'r1'}, ${1}, ${'hello-world'}, ${'s1'})` + await db`update data_rows set active_version_id = ${'vr1'} where id = ${'r1'}` + + const entries = await collectSitemapEntries(db) + const byPath = new Map(entries.map((e) => [e.path, e.lastmod])) + + expect(new Set(byPath.keys())).toEqual(new Set(['/', '/about', '/blog/hello-world'])) + // Page lastmod is the row's updated_at. + expect(byPath.get('/')).toBe('2026-07-19T10:00:00.000Z') + expect(byPath.get('/about')).toBe('2026-07-18T09:00:00.000Z') + // Row route uses the published version's slug, not the draft data_rows.slug. + expect(byPath.get('/blog/hello-world')).toBe('2026-07-14T00:00:00.000Z') + } finally { + await cleanup() + } + }) +}) diff --git a/vite.config.ts b/vite.config.ts index fff274486..e79f74613 100644 --- a/vite.config.ts +++ b/vite.config.ts @@ -41,6 +41,10 @@ function shouldProxyPublicSiteRequest(req: IncomingMessage): boolean { // /_instatic/css/ → per-site published CSS bundle (reset / framework / style) if (pathname.startsWith('/_instatic/assets/')) return true if (pathname.startsWith('/_instatic/css/')) return true + // Generated SEO documents live on the Bun server; opt them in past the + // file-extension rejection below (same reason as the `/_instatic/` routes). + if (pathname === '/sitemap.xml') return true + if (pathname === '/robots.txt') return true return pathname === '/' || !FILE_EXTENSION_RE.test(pathname) }