diff --git a/internal/handler/sitemap.go b/internal/handler/sitemap.go index a0af702e6..fbe861603 100644 --- a/internal/handler/sitemap.go +++ b/internal/handler/sitemap.go @@ -76,11 +76,16 @@ const companySitemapChunk = 10000 // ?limit= and ?chunk=, and the value web/src/lib/sitemap.ts JOB_SITEMAP_CHUNK must // equal so each offset in the index opens exactly one file's worth. // -// Kept below the protocol's 50k cap with room to spare: a 25k page measured ~2.5s -// against the prod index, and doubling it would put a single sub-sitemap's render -// near the SSR timeout for no gain — a sitemap index carries the extra files for -// free (its own cap is 50k entries). -const jobSitemapChunk = 25000 +// Sized by the SSR fetch timeout, not by the protocol cap. At 25k a page measured +// ~2.5s warm against the prod index but 8s on the deepest offset with the host under +// load — against a 10s timeout, i.e. a file that renders fine until the box is busy +// and then 500s. 10k restores the margin at ~1-3s per page. +// +// The cost is 127 sub-sitemaps instead of 51, which a sitemap index carries for free +// (its own cap is 50,000 entries). Cheap files beat a narrow deadline: a slow page is +// one missing file, but there is no partial credit — the crawler either gets it or +// gets an error. +const jobSitemapChunk = 10000 // sitemapEntry is the slim wire shape a sitemap URL needs — the slug and a lastmod. // Nothing wider (no full job or company document) crosses the wire. diff --git a/web/src/lib/sitemap.ts b/web/src/lib/sitemap.ts index 495839003..4872046f3 100644 --- a/web/src/lib/sitemap.ts +++ b/web/src/lib/sitemap.ts @@ -15,7 +15,10 @@ export const SITEMAP_CHUNK = 10000; // index, not of the jobs table: the index already holds exactly the postings worth // crawling, and it can address any offset in it directly — which is what let the // sitemap stop asking Postgres to number 3.4M rows on every render. -export const JOB_SITEMAP_CHUNK = 25000; +// +// 10k rather than 25k because the deepest 25k page measured 8s against this route's +// 10s fetch timeout under load — see the backend constant for the trade. +export const JOB_SITEMAP_CHUNK = 10000; /** The site's static, always-present pages (relative paths). */ export const STATIC_PATHS = [