Skip to content

fix(fuel): enable week-on-week price diff computation #2622

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
koala73 merged 5 commits into from
Apr 2, 2026
Merged

fix(fuel): enable week-on-week price diff computation #2622

Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
775325d
fix(fuel): enable week-on-week price diff computation
koala73 Apr 2, 2026
b6084f8
fix(fuel): only rotate prev snapshot after WoW is consumed
koala73 Apr 2, 2026
8f8b444
fix(fuel): filter Malaysia API for level rows only
koala73 Apr 2, 2026
accd653
fix(fuel): drop separate Spain source, use EU XLSX consistently
koala73 Apr 2, 2026
44f5317
fix(fuel): replace UK station scrape with DESNZ weekly CSV
koala73 Apr 2, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
121 changes: 48 additions & 73 deletions scripts/seed-fuel-prices.mjs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -79,7 +79,7 @@ async function fetchMalaysia() {
if (!resp.ok) throw new Error(`HTTP ${resp.status}`);
const data = await resp.json();
if (!Array.isArray(data) || data.length === 0) return [];
const row = data[0];
const row = data.find(r => r.series_type === 'level') ?? data[0];
const observedAt = row.date ?? '';
const ron95 = typeof row.ron95 === 'number' ? row.ron95 : null;
const diesel = typeof row.diesel === 'number' ? row.diesel : null;
Expand Down Expand Up @@ -485,72 +485,50 @@ async function fetchNewZealand() {
}
}

async function fetchUK_ModeA() {
// CMA voluntary scheme: each retailer hosts their own JSON feed. No auth required.
// Prices in pence/litre (integer). Divide by 100 -> GBP/litre.
// E10 = standard unleaded (gasoline), B7 = standard diesel.
// Aggregate across all working retailers for a national average.
const RETAILER_URLS = [
'https://storelocator.asda.com/fuel_prices_data.json',
'https://www.bp.com/en_gb/united-kingdom/home/fuelprices/fuel_prices_data.json',
'https://jetlocal.co.uk/fuel_prices_data.json',
'https://fuel.motorfuelgroup.com/fuel_prices_data.json',
'https://api.sainsburys.co.uk/v1/exports/latest/fuel_prices_data.json',
'https://www.morrisons.com/fuel-prices/fuel.json',
];

const allE10 = [];
const allB7 = [];
let observedAt = new Date().toISOString().slice(0, 10);

const results = await Promise.allSettled(
RETAILER_URLS.map(url =>
globalThis.fetch(url, { headers: { 'User-Agent': CHROME_UA }, signal: AbortSignal.timeout(15000) })
.then(r => r.ok ? r.json() : Promise.reject(new Error(`HTTP ${r.status} ${url}`)))
)
);

for (let i = 0; i < results.length; i++) {
const r = results[i];
if (r.status === 'rejected') {
console.warn(` [UK] ${RETAILER_URLS[i]}: ${r.reason?.message ?? r.reason}`);
continue;
}
const body = r.value;
// CMA format: { last_updated, stations: [{ prices: { E10, B7, ... } }] }
const stations = body?.stations ?? body?.data ?? [];
if (!Array.isArray(stations)) continue;
if (body.last_updated) {
// CMA feeds use "DD/MM/YYYY HH:mm:ss" — convert to ISO YYYY-MM-DD for comparison
const raw = String(body.last_updated);
const ddmmyyyy = raw.match(/^(\d{2})\/(\d{2})\/(\d{4})/);
const iso = ddmmyyyy ? `${ddmmyyyy[3]}-${ddmmyyyy[2]}-${ddmmyyyy[1]}` : raw.slice(0, 10);
if (iso > observedAt) observedAt = iso;
}
for (const s of stations) {
const prices = s?.prices ?? s?.fuel_prices ?? {};
const e10 = prices?.E10 ?? prices?.['E10_STANDARD'];
const b7 = prices?.B7 ?? prices?.['B7_STANDARD'];
if (e10 > 0) allE10.push(e10);
if (b7 > 0) allB7.push(b7);
}
}
async function fetchUK_DESNZ() {
// Gov.uk DESNZ weekly road fuel prices CSV. Published weekly, covers 2018-present.
// ULSP = unleaded petrol (gasoline), ULSD = diesel. Prices in pence/litre.
// URL changes weekly; discover via Content API.
try {
console.log(' [GB] Discovering DESNZ CSV URL...');
const apiResp = await globalThis.fetch('https://www.gov.uk/api/content/government/statistics/weekly-road-fuel-prices', {
headers: { 'User-Agent': CHROME_UA }, signal: AbortSignal.timeout(15000),
});
if (!apiResp.ok) throw new Error(`Content API HTTP ${apiResp.status}`);
const apiData = await apiResp.json();
const csvAttach = apiData?.details?.attachments?.find(a => a.content_type?.includes('csv') && a.title?.includes('2018'));
if (!csvAttach?.url) throw new Error('CSV attachment not found in Content API');

if (!allE10.length && !allB7.length) {
console.warn(' [UK] No stations with E10/B7 data from any retailer');
const csvResp = await globalThis.fetch(csvAttach.url, {
headers: { 'User-Agent': CHROME_UA }, signal: AbortSignal.timeout(20000),
});
if (!csvResp.ok) throw new Error(`CSV HTTP ${csvResp.status}`);
const lines = (await csvResp.text()).split('\n').filter(l => l.trim());
// Header: Date,ULSP Pump price pence/litre,ULSD Pump price pence/litre,...
const dataLines = lines.slice(1).filter(l => l.split(',').length >= 3);
if (!dataLines.length) throw new Error('No data rows in CSV');

const lastLine = dataLines.at(-1).split(',');
const dateStr = lastLine[0]?.trim();
const ulsp = parseFloat(lastLine[1]);
const ulsd = parseFloat(lastLine[2]);
const gasPrice = ulsp > 0 ? +(ulsp / 100).toFixed(4) : null;
const dslPrice = ulsd > 0 ? +(ulsd / 100).toFixed(4) : null;

// Parse DD/MM/YYYY -> YYYY-MM-DD
const dm = dateStr?.match(/(\d{2})\/(\d{2})\/(\d{4})/);
const observedAt = dm ? `${dm[3]}-${dm[2]}-${dm[1]}` : dateStr;

console.log(` [GB] ULSP=${gasPrice} GBP/L, ULSD=${dslPrice} GBP/L (${observedAt})`);
return [{
code: 'GB', name: 'United Kingdom', currency: 'GBP', flag: '🇬🇧',
gasoline: gasPrice != null ? { localPrice: gasPrice, grade: 'E10', source: 'gov.uk/desnz', observedAt } : null,
diesel: dslPrice != null ? { localPrice: dslPrice, grade: 'B7', source: 'gov.uk/desnz', observedAt } : null,
}];
} catch (err) {
console.warn(` [GB] fetchUK_DESNZ error: ${err.message}`);
return [];
}

// Prices are in pence/litre -> divide by 100 for GBP/litre
const avgE10 = allE10.length ? +(allE10.reduce((a, b) => a + b, 0) / allE10.length / 100).toFixed(4) : null;
const avgB7 = allB7.length ? +(allB7.reduce((a, b) => a + b, 0) / allB7.length / 100).toFixed(4) : null;

console.log(` [GB] E10=${avgE10} GBP/L (${allE10.length} stations), B7=${avgB7} GBP/L (${allB7.length} stations)`);
return [{
code: 'GB', name: 'United Kingdom', currency: 'GBP', flag: '🇬🇧',
gasoline: avgE10 != null ? { localPrice: avgE10, grade: 'E10', source: 'gov.uk/fuel-finder', observedAt } : null,
diesel: avgB7 != null ? { localPrice: avgB7, grade: 'B7', source: 'gov.uk/fuel-finder', observedAt } : null,
}];
}

const prevSnapshot = await readSeedSnapshot(`${CANONICAL_KEY}:prev`);
Expand All @@ -565,16 +543,15 @@ console.log(' [FX] Rates loaded:', Object.keys(fxRates).join(', '));

const fetchResults = await Promise.allSettled([
fetchMalaysia(),
fetchSpain(),
fetchMexico(),
fetchUS_EIA(),
fetchEU_CSV(),
fetchBrazil(),
fetchNewZealand(),
fetchUK_ModeA(),
fetchUK_DESNZ(),
]);

const sourceNames = ['Malaysia', 'Spain', 'Mexico', 'US-EIA', 'EU-CSV', 'Brazil', 'New Zealand', 'UK-ModeA'];
const sourceNames = ['Malaysia', 'Mexico', 'US-EIA', 'EU-CSV', 'Brazil', 'New Zealand', 'UK-DESNZ'];
let successfulSources = 0;

const countryMap = new Map();
Expand Down Expand Up @@ -652,17 +629,15 @@ if (wowAvailable) {
const prev = prevMap.get(country.code);
if (!prev) continue;

if (country.gasoline && prev.gasoline?.usdPrice > 0 && country.gasoline.usdPrice > 0
&& country.gasoline.observedAt !== prev.gasoline?.observedAt) {
if (country.gasoline && prev.gasoline?.usdPrice > 0 && country.gasoline.usdPrice > 0) {
const raw = +((country.gasoline.usdPrice - prev.gasoline.usdPrice) / prev.gasoline.usdPrice * 100).toFixed(2);
if (Math.abs(raw) > WOW_ANOMALY_THRESHOLD) {
console.warn(` [WoW] ANOMALY ${country.flag} ${country.name} gasoline: ${raw}% — omitting`);
} else {
country.gasoline.wowPct = raw;
}
}
if (country.diesel && prev.diesel?.usdPrice > 0 && country.diesel.usdPrice > 0
&& country.diesel.observedAt !== prev.diesel?.observedAt) {
if (country.diesel && prev.diesel?.usdPrice > 0 && country.diesel.usdPrice > 0) {
const raw = +((country.diesel.usdPrice - prev.diesel.usdPrice) / prev.diesel.usdPrice * 100).toFixed(2);
if (Math.abs(raw) > WOW_ANOMALY_THRESHOLD) {
console.warn(` [WoW] ANOMALY ${country.flag} ${country.name} diesel: ${raw}% — omitting`);
Expand Down Expand Up @@ -711,9 +686,9 @@ await runSeed('economic', 'fuel-prices', CANONICAL_KEY, async () => data, {
ttlSeconds: CACHE_TTL,
validateFn: (d) => d?.countries?.length >= 1,
recordCount: (d) => d?.countries?.length || 0,
extraKeys: [{
extraKeys: wowAvailable ? [{
key: `${CANONICAL_KEY}:prev`,
transform: () => data,
ttl: CACHE_TTL * 2,
}],
}] : [],
});
Loading