From 775325dd092c382e816ab3602e64eaeba717c93b Mon Sep 17 00:00:00 2001 From: Elie Habib Date: Thu, 2 Apr 2026 18:58:00 +0400 Subject: [PATCH 1/5] fix(fuel): enable week-on-week price diff computation The WoW computation was silently skipped for most countries because the observedAt guard required different source observation dates between consecutive weekly seeder runs. Since the EU XLSX often embeds the same date across weekly publishes, nearly all 27 EU countries got 0 WoW. Also fix prev snapshot rotation: only overwrite :prev when the existing snapshot is 6+ days old, preventing the seeder from clobbering a freshly backfilled prev snapshot. --- scripts/seed-fuel-prices.mjs | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/scripts/seed-fuel-prices.mjs b/scripts/seed-fuel-prices.mjs index 62938358af..f6cdeb4d41 100644 --- a/scripts/seed-fuel-prices.mjs +++ b/scripts/seed-fuel-prices.mjs @@ -652,8 +652,7 @@ if (wowAvailable) { const prev = prevMap.get(country.code); if (!prev) continue; - if (country.gasoline && prev.gasoline?.usdPrice > 0 && country.gasoline.usdPrice > 0 - && country.gasoline.observedAt !== prev.gasoline?.observedAt) { + if (country.gasoline && prev.gasoline?.usdPrice > 0 && country.gasoline.usdPrice > 0) { const raw = +((country.gasoline.usdPrice - prev.gasoline.usdPrice) / prev.gasoline.usdPrice * 100).toFixed(2); if (Math.abs(raw) > WOW_ANOMALY_THRESHOLD) { console.warn(` [WoW] ANOMALY ${country.flag} ${country.name} gasoline: ${raw}% — omitting`); @@ -661,8 +660,7 @@ if (wowAvailable) { country.gasoline.wowPct = raw; } } - if (country.diesel && prev.diesel?.usdPrice > 0 && country.diesel.usdPrice > 0 - && country.diesel.observedAt !== prev.diesel?.observedAt) { + if (country.diesel && prev.diesel?.usdPrice > 0 && country.diesel.usdPrice > 0) { const raw = +((country.diesel.usdPrice - prev.diesel.usdPrice) / prev.diesel.usdPrice * 100).toFixed(2); if (Math.abs(raw) > WOW_ANOMALY_THRESHOLD) { console.warn(` [WoW] ANOMALY ${country.flag} ${country.name} diesel: ${raw}% — omitting`); @@ -707,13 +705,15 @@ const data = { countryCount: countries.length, }; +const shouldRotatePrev = !hasPrevData || !prevTooRecent; + await runSeed('economic', 'fuel-prices', CANONICAL_KEY, async () => data, { ttlSeconds: CACHE_TTL, validateFn: (d) => d?.countries?.length >= 1, recordCount: (d) => d?.countries?.length || 0, - extraKeys: [{ + extraKeys: shouldRotatePrev ? [{ key: `${CANONICAL_KEY}:prev`, transform: () => data, ttl: CACHE_TTL * 2, - }], + }] : [], }); From b6084f85d01827108ec3cd0574794fd571b4650e Mon Sep 17 00:00:00 2001 From: Elie Habib Date: Thu, 2 Apr 2026 19:02:27 +0400 Subject: [PATCH 2/5] fix(fuel): only rotate prev snapshot after WoW is consumed The previous shouldRotatePrev logic checked if prev data was "old enough" but this used the data's fetchedAt, not when the key was written. A backfill with an older fetchedAt would be immediately overwritten by the seeder. Simpler and correct: rotate prev only when wowAvailable is true, meaning the prev data was successfully consumed for WoW computation. --- scripts/seed-fuel-prices.mjs | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/scripts/seed-fuel-prices.mjs b/scripts/seed-fuel-prices.mjs index f6cdeb4d41..0bdc8d5e36 100644 --- a/scripts/seed-fuel-prices.mjs +++ b/scripts/seed-fuel-prices.mjs @@ -705,13 +705,11 @@ const data = { countryCount: countries.length, }; -const shouldRotatePrev = !hasPrevData || !prevTooRecent; - await runSeed('economic', 'fuel-prices', CANONICAL_KEY, async () => data, { ttlSeconds: CACHE_TTL, validateFn: (d) => d?.countries?.length >= 1, recordCount: (d) => d?.countries?.length || 0, - extraKeys: shouldRotatePrev ? [{ + extraKeys: wowAvailable ? [{ key: `${CANONICAL_KEY}:prev`, transform: () => data, ttl: CACHE_TTL * 2, From 8f8b4446b8d6c35cd60d5dc3147db1ec0b9b99bc Mon Sep 17 00:00:00 2001 From: Elie Habib Date: Thu, 2 Apr 2026 19:39:20 +0400 Subject: [PATCH 3/5] fix(fuel): filter Malaysia API for level rows only The data.gov.my API returns both series_type=level (actual prices) and series_type=change_weekly (delta) rows. The change_weekly row has ron95=0 when price is unchanged, which the seeder rejected as "out of range". --- scripts/seed-fuel-prices.mjs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/scripts/seed-fuel-prices.mjs b/scripts/seed-fuel-prices.mjs index 0bdc8d5e36..313412da53 100644 --- a/scripts/seed-fuel-prices.mjs +++ b/scripts/seed-fuel-prices.mjs @@ -79,7 +79,7 @@ async function fetchMalaysia() { if (!resp.ok) throw new Error(`HTTP ${resp.status}`); const data = await resp.json(); if (!Array.isArray(data) || data.length === 0) return []; - const row = data[0]; + const row = data.find(r => r.series_type === 'level') ?? data[0]; const observedAt = row.date ?? ''; const ron95 = typeof row.ron95 === 'number' ? row.ron95 : null; const diesel = typeof row.diesel === 'number' ? row.diesel : null; From accd65317b509f2d10c270b3b6b0b996f1b3e1db Mon Sep 17 00:00:00 2001 From: Elie Habib Date: Thu, 2 Apr 2026 19:48:16 +0400 Subject: [PATCH 4/5] fix(fuel): drop separate Spain source, use EU XLSX consistently Spain was fetched from minetur.gob.es (live station average) AND the EU XLSX (weekly weighted average). Since minetur.gob.es ran first, it won the merge, causing a source mismatch with the :prev snapshot (EU XLSX). This produced a phantom -10% WoW diff. Removing fetchSpain() makes Spain consistent with all other EU countries, sourced entirely from the EU Oil Bulletin XLSX. --- scripts/seed-fuel-prices.mjs | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/scripts/seed-fuel-prices.mjs b/scripts/seed-fuel-prices.mjs index 313412da53..de25dc8912 100644 --- a/scripts/seed-fuel-prices.mjs +++ b/scripts/seed-fuel-prices.mjs @@ -565,7 +565,6 @@ console.log(' [FX] Rates loaded:', Object.keys(fxRates).join(', ')); const fetchResults = await Promise.allSettled([ fetchMalaysia(), - fetchSpain(), fetchMexico(), fetchUS_EIA(), fetchEU_CSV(), @@ -574,7 +573,7 @@ const fetchResults = await Promise.allSettled([ fetchUK_ModeA(), ]); -const sourceNames = ['Malaysia', 'Spain', 'Mexico', 'US-EIA', 'EU-CSV', 'Brazil', 'New Zealand', 'UK-ModeA']; +const sourceNames = ['Malaysia', 'Mexico', 'US-EIA', 'EU-CSV', 'Brazil', 'New Zealand', 'UK-ModeA']; let successfulSources = 0; const countryMap = new Map(); From 44f5317680fdc67c88b848c8ad6343110e36bc0c Mon Sep 17 00:00:00 2001 From: Elie Habib Date: Thu, 2 Apr 2026 20:03:50 +0400 Subject: [PATCH 5/5] fix(fuel): replace UK station scrape with DESNZ weekly CSV MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The UK source was fetchUK_ModeA (live retailer station scrape, ~2650 stations) which produced a £0.105/L gap vs the gov.uk DESNZ published weekly average. This caused a phantom +7.3% WoW when compared against the DESNZ-sourced prev snapshot. Replace with fetchUK_DESNZ which reads the official weekly road fuel prices CSV from gov.uk. URL is discovered via the Content API since it changes weekly. This gives consistent WoW and is the authoritative UK national average. --- scripts/seed-fuel-prices.mjs | 108 ++++++++++++++--------------------- 1 file changed, 43 insertions(+), 65 deletions(-) diff --git a/scripts/seed-fuel-prices.mjs b/scripts/seed-fuel-prices.mjs index de25dc8912..5ec52e388c 100644 --- a/scripts/seed-fuel-prices.mjs +++ b/scripts/seed-fuel-prices.mjs @@ -485,72 +485,50 @@ async function fetchNewZealand() { } } -async function fetchUK_ModeA() { - // CMA voluntary scheme: each retailer hosts their own JSON feed. No auth required. - // Prices in pence/litre (integer). Divide by 100 -> GBP/litre. - // E10 = standard unleaded (gasoline), B7 = standard diesel. - // Aggregate across all working retailers for a national average. - const RETAILER_URLS = [ - 'https://storelocator.asda.com/fuel_prices_data.json', - 'https://www.bp.com/en_gb/united-kingdom/home/fuelprices/fuel_prices_data.json', - 'https://jetlocal.co.uk/fuel_prices_data.json', - 'https://fuel.motorfuelgroup.com/fuel_prices_data.json', - 'https://api.sainsburys.co.uk/v1/exports/latest/fuel_prices_data.json', - 'https://www.morrisons.com/fuel-prices/fuel.json', - ]; - - const allE10 = []; - const allB7 = []; - let observedAt = new Date().toISOString().slice(0, 10); - - const results = await Promise.allSettled( - RETAILER_URLS.map(url => - globalThis.fetch(url, { headers: { 'User-Agent': CHROME_UA }, signal: AbortSignal.timeout(15000) }) - .then(r => r.ok ? r.json() : Promise.reject(new Error(`HTTP ${r.status} ${url}`))) - ) - ); - - for (let i = 0; i < results.length; i++) { - const r = results[i]; - if (r.status === 'rejected') { - console.warn(` [UK] ${RETAILER_URLS[i]}: ${r.reason?.message ?? r.reason}`); - continue; - } - const body = r.value; - // CMA format: { last_updated, stations: [{ prices: { E10, B7, ... } }] } - const stations = body?.stations ?? body?.data ?? []; - if (!Array.isArray(stations)) continue; - if (body.last_updated) { - // CMA feeds use "DD/MM/YYYY HH:mm:ss" — convert to ISO YYYY-MM-DD for comparison - const raw = String(body.last_updated); - const ddmmyyyy = raw.match(/^(\d{2})\/(\d{2})\/(\d{4})/); - const iso = ddmmyyyy ? `${ddmmyyyy[3]}-${ddmmyyyy[2]}-${ddmmyyyy[1]}` : raw.slice(0, 10); - if (iso > observedAt) observedAt = iso; - } - for (const s of stations) { - const prices = s?.prices ?? s?.fuel_prices ?? {}; - const e10 = prices?.E10 ?? prices?.['E10_STANDARD']; - const b7 = prices?.B7 ?? prices?.['B7_STANDARD']; - if (e10 > 0) allE10.push(e10); - if (b7 > 0) allB7.push(b7); - } - } +async function fetchUK_DESNZ() { + // Gov.uk DESNZ weekly road fuel prices CSV. Published weekly, covers 2018-present. + // ULSP = unleaded petrol (gasoline), ULSD = diesel. Prices in pence/litre. + // URL changes weekly; discover via Content API. + try { + console.log(' [GB] Discovering DESNZ CSV URL...'); + const apiResp = await globalThis.fetch('https://www.gov.uk/api/content/government/statistics/weekly-road-fuel-prices', { + headers: { 'User-Agent': CHROME_UA }, signal: AbortSignal.timeout(15000), + }); + if (!apiResp.ok) throw new Error(`Content API HTTP ${apiResp.status}`); + const apiData = await apiResp.json(); + const csvAttach = apiData?.details?.attachments?.find(a => a.content_type?.includes('csv') && a.title?.includes('2018')); + if (!csvAttach?.url) throw new Error('CSV attachment not found in Content API'); - if (!allE10.length && !allB7.length) { - console.warn(' [UK] No stations with E10/B7 data from any retailer'); + const csvResp = await globalThis.fetch(csvAttach.url, { + headers: { 'User-Agent': CHROME_UA }, signal: AbortSignal.timeout(20000), + }); + if (!csvResp.ok) throw new Error(`CSV HTTP ${csvResp.status}`); + const lines = (await csvResp.text()).split('\n').filter(l => l.trim()); + // Header: Date,ULSP Pump price pence/litre,ULSD Pump price pence/litre,... + const dataLines = lines.slice(1).filter(l => l.split(',').length >= 3); + if (!dataLines.length) throw new Error('No data rows in CSV'); + + const lastLine = dataLines.at(-1).split(','); + const dateStr = lastLine[0]?.trim(); + const ulsp = parseFloat(lastLine[1]); + const ulsd = parseFloat(lastLine[2]); + const gasPrice = ulsp > 0 ? +(ulsp / 100).toFixed(4) : null; + const dslPrice = ulsd > 0 ? +(ulsd / 100).toFixed(4) : null; + + // Parse DD/MM/YYYY -> YYYY-MM-DD + const dm = dateStr?.match(/(\d{2})\/(\d{2})\/(\d{4})/); + const observedAt = dm ? `${dm[3]}-${dm[2]}-${dm[1]}` : dateStr; + + console.log(` [GB] ULSP=${gasPrice} GBP/L, ULSD=${dslPrice} GBP/L (${observedAt})`); + return [{ + code: 'GB', name: 'United Kingdom', currency: 'GBP', flag: '🇬🇧', + gasoline: gasPrice != null ? { localPrice: gasPrice, grade: 'E10', source: 'gov.uk/desnz', observedAt } : null, + diesel: dslPrice != null ? { localPrice: dslPrice, grade: 'B7', source: 'gov.uk/desnz', observedAt } : null, + }]; + } catch (err) { + console.warn(` [GB] fetchUK_DESNZ error: ${err.message}`); return []; } - - // Prices are in pence/litre -> divide by 100 for GBP/litre - const avgE10 = allE10.length ? +(allE10.reduce((a, b) => a + b, 0) / allE10.length / 100).toFixed(4) : null; - const avgB7 = allB7.length ? +(allB7.reduce((a, b) => a + b, 0) / allB7.length / 100).toFixed(4) : null; - - console.log(` [GB] E10=${avgE10} GBP/L (${allE10.length} stations), B7=${avgB7} GBP/L (${allB7.length} stations)`); - return [{ - code: 'GB', name: 'United Kingdom', currency: 'GBP', flag: '🇬🇧', - gasoline: avgE10 != null ? { localPrice: avgE10, grade: 'E10', source: 'gov.uk/fuel-finder', observedAt } : null, - diesel: avgB7 != null ? { localPrice: avgB7, grade: 'B7', source: 'gov.uk/fuel-finder', observedAt } : null, - }]; } const prevSnapshot = await readSeedSnapshot(`${CANONICAL_KEY}:prev`); @@ -570,10 +548,10 @@ const fetchResults = await Promise.allSettled([ fetchEU_CSV(), fetchBrazil(), fetchNewZealand(), - fetchUK_ModeA(), + fetchUK_DESNZ(), ]); -const sourceNames = ['Malaysia', 'Mexico', 'US-EIA', 'EU-CSV', 'Brazil', 'New Zealand', 'UK-ModeA']; +const sourceNames = ['Malaysia', 'Mexico', 'US-EIA', 'EU-CSV', 'Brazil', 'New Zealand', 'UK-DESNZ']; let successfulSources = 0; const countryMap = new Map();