From d996c0dae61562f4316eb5b6a2d00000b48e1982 Mon Sep 17 00:00:00 2001 From: strelov1 Date: Sun, 16 Aug 2026 18:27:33 -0300 Subject: [PATCH] Halve the job sitemap chunk to restore the timeout margin MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Measured on prod after #1990 shipped: a 25k page is ~2.5s warm, but 8s on the deepest offset with the host under load — against this route's 10s fetch timeout. That is a file which renders fine until the box is busy and then 500s, which is the same shape of latent failure /sitemap.xml itself was in before #1990 (it answered 200 from a stale nginx cache while every uncached request errored). 10k puts a page at ~1-3s. The cost is 127 sub-sitemaps instead of 51, which a sitemap index carries for free — its own cap is 50,000 entries. Cheap files beat a narrow deadline here because there is no partial credit: the crawler either gets a file or gets an error, so the failure mode of being slightly too slow is losing 25,000 URLs rather than delivering them late. --- internal/handler/sitemap.go | 15 ++++++++++----- web/src/lib/sitemap.ts | 5 ++++- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/internal/handler/sitemap.go b/internal/handler/sitemap.go index a0af702e6..fbe861603 100644 --- a/internal/handler/sitemap.go +++ b/internal/handler/sitemap.go @@ -76,11 +76,16 @@ const companySitemapChunk = 10000 // ?limit= and ?chunk=, and the value web/src/lib/sitemap.ts JOB_SITEMAP_CHUNK must // equal so each offset in the index opens exactly one file's worth. // -// Kept below the protocol's 50k cap with room to spare: a 25k page measured ~2.5s -// against the prod index, and doubling it would put a single sub-sitemap's render -// near the SSR timeout for no gain — a sitemap index carries the extra files for -// free (its own cap is 50k entries). -const jobSitemapChunk = 25000 +// Sized by the SSR fetch timeout, not by the protocol cap. At 25k a page measured +// ~2.5s warm against the prod index but 8s on the deepest offset with the host under +// load — against a 10s timeout, i.e. a file that renders fine until the box is busy +// and then 500s. 10k restores the margin at ~1-3s per page. +// +// The cost is 127 sub-sitemaps instead of 51, which a sitemap index carries for free +// (its own cap is 50,000 entries). Cheap files beat a narrow deadline: a slow page is +// one missing file, but there is no partial credit — the crawler either gets it or +// gets an error. +const jobSitemapChunk = 10000 // sitemapEntry is the slim wire shape a sitemap URL needs — the slug and a lastmod. // Nothing wider (no full job or company document) crosses the wire. diff --git a/web/src/lib/sitemap.ts b/web/src/lib/sitemap.ts index 495839003..4872046f3 100644 --- a/web/src/lib/sitemap.ts +++ b/web/src/lib/sitemap.ts @@ -15,7 +15,10 @@ export const SITEMAP_CHUNK = 10000; // index, not of the jobs table: the index already holds exactly the postings worth // crawling, and it can address any offset in it directly — which is what let the // sitemap stop asking Postgres to number 3.4M rows on every render. -export const JOB_SITEMAP_CHUNK = 25000; +// +// 10k rather than 25k because the deepest 25k page measured 8s against this route's +// 10s fetch timeout under load — see the backend constant for the trade. +export const JOB_SITEMAP_CHUNK = 10000; /** The site's static, always-present pages (relative paths). */ export const STATIC_PATHS = [