diff --git a/scripts/_shared-av.mjs b/scripts/_shared-av.mjs new file mode 100644 index 0000000000..141f16377b --- /dev/null +++ b/scripts/_shared-av.mjs @@ -0,0 +1,162 @@ +// @ts-check +/** + * Shared Alpha Vantage fetch helpers for seed scripts. + * Single implementation used by seed-market-quotes, seed-commodity-quotes, seed-etf-flows. + */ + +import { CHROME_UA, sleep } from './_seed-utils.mjs'; + +export const AV_PHYSICAL_MAP = { + 'CL=F': 'WTI', + 'BZ=F': 'BRENT', + 'NG=F': 'NATURAL_GAS', + 'HG=F': 'COPPER', + 'ALI=F': 'ALUMINUM', + 'GC=F': 'GOLD', + 'SI=F': 'SILVER', +}; + +const AV_BATCH_DELAY_MS = 500; +const AV_TIMEOUT_MS = 15_000; + +async function avFetch(url, label) { + for (let attempt = 0; attempt < 2; attempt++) { + try { + if (attempt > 0) await sleep(1000); + const resp = await fetch(url, { + headers: { 'User-Agent': CHROME_UA, Accept: 'application/json' }, + signal: AbortSignal.timeout(AV_TIMEOUT_MS), + }); + if (!resp.ok) { + console.warn(` [AV] ${label} HTTP ${resp.status}`); + if (attempt === 0) continue; + return null; + } + return resp; + } catch (err) { + console.warn(` [AV] ${label} error: ${err.message}`); + if (attempt === 0) continue; + return null; + } + } + return null; +} + +/** + * Fetch physical commodity quote from AV daily series. + * Returns price, day-over-day change, and a 7-point sparkline from daily closes. + * + * Note: AV physical commodity endpoints are daily-close data (not intraday). + * Prices reflect the most recent market-day close. + * + * @param {string} yahooSymbol + * @param {string} apiKey + * @returns {Promise<{ price: number; change: number; sparkline: number[] } | null>} + */ +export async function fetchAvPhysicalCommodity(yahooSymbol, apiKey) { + const fn = AV_PHYSICAL_MAP[yahooSymbol]; + if (!fn) return null; + const url = `https://www.alphavantage.co/query?function=${fn}&interval=daily&apikey=${encodeURIComponent(apiKey)}`; + const resp = await avFetch(url, fn); + if (!resp) return null; + try { + const json = await resp.json(); + if (json.Information) { console.warn(` [AV] Rate limit: ${String(json.Information).slice(0, 100)}`); return null; } + const data = json.data; + if (!Array.isArray(data) || data.length < 2) return null; + const latest = parseFloat(data[0].value); + const prev = parseFloat(data[1].value); + if (!Number.isFinite(latest) || latest <= 0) return null; + const change = (Number.isFinite(prev) && prev > 0) ? ((latest - prev) / prev) * 100 : 0; + // Build sparkline from last 7 daily closes (oldest → newest) + const sparkline = data.slice(0, 7).map(d => parseFloat(d.value)).filter(Number.isFinite).reverse(); + return { price: latest, change, sparkline }; + } catch (err) { + console.warn(` [AV] ${fn} parse error: ${err.message}`); + return null; + } +} + +/** + * Fetch daily FX time series for a currency pair (FROM → USD). + * Returns price (latest close), day-over-day change %, and 7-point sparkline. + * Use this when you need change% and sparkline (e.g. gulf panel currencies). + * + * @param {string} fromCurrency e.g. 'SAR', 'EUR', 'JPY' + * @param {string} apiKey + * @returns {Promise<{ price: number; change: number; sparkline: number[] } | null>} + */ +export async function fetchAvFxDaily(fromCurrency, apiKey) { + if (fromCurrency === 'USD') return { price: 1.0, change: 0, sparkline: [] }; + const url = `https://www.alphavantage.co/query?function=FX_DAILY&from_symbol=${encodeURIComponent(fromCurrency)}&to_symbol=USD&outputsize=compact&apikey=${encodeURIComponent(apiKey)}`; + const resp = await avFetch(url, `FX_DAILY/${fromCurrency}`); + if (!resp) return null; + try { + const json = await resp.json(); + if (json.Information) { console.warn(` [AV] Rate limit: ${String(json.Information).slice(0, 100)}`); return null; } + const series = json['Time Series FX (Daily)']; + if (!series || typeof series !== 'object') return null; + const dates = Object.keys(series).sort().reverse(); // newest first + if (dates.length < 2) return null; + const latest = parseFloat(series[dates[0]]['4. close']); + const prev = parseFloat(series[dates[1]]['4. close']); + if (!Number.isFinite(latest) || latest <= 0) return null; + const change = (Number.isFinite(prev) && prev > 0) ? ((latest - prev) / prev) * 100 : 0; + const sparkline = dates.slice(0, 7).map(d => parseFloat(series[d]['4. close'])).filter(Number.isFinite).reverse(); + return { price: latest, change, sparkline }; + } catch (err) { + console.warn(` [AV] FX_DAILY/${fromCurrency} parse error: ${err.message}`); + return null; + } +} + +/** + * Fetch real-time bulk quotes from AV. Batches up to 100 symbols per call. + * Returns a Map of symbol → { price, change, volume, prevClose }. + * + * @param {string[]} symbols + * @param {string} apiKey + * @returns {Promise>} + */ +export async function fetchAvBulkQuotes(symbols, apiKey) { + if (symbols.length === 0) return new Map(); + const results = new Map(); + const BATCH = 100; + for (let i = 0; i < symbols.length; i += BATCH) { + if (i > 0) await sleep(AV_BATCH_DELAY_MS); + const chunk = symbols.slice(i, i + BATCH); + const url = `https://www.alphavantage.co/query?function=REALTIME_BULK_QUOTES&symbol=${encodeURIComponent(chunk.join(','))}&apikey=${encodeURIComponent(apiKey)}`; + const resp = await avFetch(url, 'REALTIME_BULK_QUOTES'); + if (!resp) continue; + try { + const json = await resp.json(); + if (json.Information) { + const remaining = symbols.length - i - chunk.length; + console.warn(` [AV] Rate limit hit${remaining > 0 ? ` — dropping ${remaining} remaining symbols` : ''}: ${String(json.Information).slice(0, 80)}`); + break; + } + if (!Array.isArray(json.data)) { + console.warn(' [AV] Unexpected response:', JSON.stringify(json).slice(0, 200)); + continue; + } + for (const item of json.data) { + const price = parseFloat(item.price); + const prevClose = parseFloat(item['previous close']); + const volume = parseInt(item.volume || '0', 10); + if (!Number.isFinite(price) || price <= 0) continue; + const changePct = (Number.isFinite(prevClose) && prevClose > 0) + ? ((price - prevClose) / prevClose) * 100 + : parseFloat((item['change percent'] || '0').replace('%', '')); + results.set(item.symbol, { + price, + change: Number.isFinite(changePct) ? changePct : 0, + volume: Number.isFinite(volume) ? volume : 0, + prevClose: Number.isFinite(prevClose) ? prevClose : null, + }); + } + } catch (err) { + console.warn(` [AV] Bulk quotes parse error: ${err.message}`); + } + } + return results; +} diff --git a/scripts/seed-commodity-quotes.mjs b/scripts/seed-commodity-quotes.mjs index 6743039481..1436a03137 100644 --- a/scripts/seed-commodity-quotes.mjs +++ b/scripts/seed-commodity-quotes.mjs @@ -1,6 +1,7 @@ #!/usr/bin/env node -import { loadEnvFile, loadSharedConfig, CHROME_UA, sleep, runSeed, parseYahooChart, writeExtraKey } from './_seed-utils.mjs'; +import { loadEnvFile, loadSharedConfig, sleep, runSeed, parseYahooChart, writeExtraKey } from './_seed-utils.mjs'; +import { AV_PHYSICAL_MAP, fetchAvPhysicalCommodity, fetchAvBulkQuotes } from './_shared-av.mjs'; const commodityConfig = loadSharedConfig('commodities.json'); @@ -37,22 +38,50 @@ const COMMODITY_SYMBOLS = commodityConfig.commodities.map(c => c.symbol); async function fetchCommodityQuotes() { const quotes = []; let misses = 0; + const avKey = process.env.ALPHA_VANTAGE_API_KEY; + // --- Primary: Alpha Vantage --- + if (avKey) { + // Physical commodity functions for WTI, BRENT, NATURAL_GAS, COPPER, ALUMINUM + const physicalSymbols = COMMODITY_SYMBOLS.filter(s => AV_PHYSICAL_MAP[s]); + for (const sym of physicalSymbols) { + const q = await fetchAvPhysicalCommodity(sym, avKey); + if (q) { + const meta = commodityConfig.commodities.find(c => c.symbol === sym); + quotes.push({ symbol: sym, name: meta?.name || sym, display: meta?.display || sym, ...q }); + console.log(` [AV:physical] ${sym}: $${q.price} (${q.change > 0 ? '+' : ''}${q.change.toFixed(2)}%)`); + } + } + + // REALTIME_BULK_QUOTES for ETF-style symbols (URA, LIT) + const bulkCandidates = COMMODITY_SYMBOLS.filter(s => !AV_PHYSICAL_MAP[s] && !quotes.some(q => q.symbol === s) && !s.includes('=F') && !s.startsWith('^')); + const bulkResults = await fetchAvBulkQuotes(bulkCandidates, avKey); + for (const [sym, q] of bulkResults) { + const meta = commodityConfig.commodities.find(c => c.symbol === sym); + quotes.push({ symbol: sym, name: meta?.name || sym, display: meta?.display || sym, price: q.price, change: q.change, sparkline: [] }); + console.log(` [AV:bulk] ${sym}: $${q.price} (${q.change > 0 ? '+' : ''}${q.change.toFixed(2)}%)`); + } + } + + const covered = new Set(quotes.map(q => q.symbol)); + + // --- Fallback: Yahoo (for remaining symbols: futures not covered by AV, ^VIX, Indian markets) --- + let yahooIdx = 0; for (let i = 0; i < COMMODITY_SYMBOLS.length; i++) { const symbol = COMMODITY_SYMBOLS[i]; - if (i > 0) await sleep(YAHOO_DELAY_MS); + if (covered.has(symbol)) continue; + if (yahooIdx > 0) await sleep(YAHOO_DELAY_MS); + yahooIdx++; try { const url = `https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(symbol)}`; const resp = await fetchYahooWithRetry(url, symbol); - if (!resp) { - misses++; - continue; - } + if (!resp) { misses++; continue; } const parsed = parseYahooChart(await resp.json(), symbol); if (parsed) { quotes.push(parsed); - console.log(` ${symbol}: $${parsed.price} (${parsed.change > 0 ? '+' : ''}${parsed.change}%)`); + covered.add(symbol); + console.log(` [Yahoo] ${symbol}: $${parsed.price} (${parsed.change > 0 ? '+' : ''}${parsed.change}%)`); } else { misses++; } @@ -83,7 +112,7 @@ async function fetchAndStash() { runSeed('market', 'commodities', CANONICAL_KEY, fetchAndStash, { validateFn: validate, ttlSeconds: CACHE_TTL, - sourceVersion: 'yahoo-chart', + sourceVersion: 'alphavantage+yahoo-chart', }).then(async (result) => { if (result?.skipped || !seedData) return; const commodityKey = `market:commodities:v1:${[...COMMODITY_SYMBOLS].sort().join(',')}`; diff --git a/scripts/seed-etf-flows.mjs b/scripts/seed-etf-flows.mjs index 020409f552..bf1a6a8ff3 100755 --- a/scripts/seed-etf-flows.mjs +++ b/scripts/seed-etf-flows.mjs @@ -1,6 +1,7 @@ #!/usr/bin/env node -import { loadEnvFile, loadSharedConfig, CHROME_UA, runSeed } from './_seed-utils.mjs'; +import { loadEnvFile, loadSharedConfig, runSeed } from './_seed-utils.mjs'; +import { fetchAvBulkQuotes } from './_shared-av.mjs'; const etfConfig = loadSharedConfig('etfs.json'); @@ -81,23 +82,44 @@ function parseEtfChartData(chart, ticker, issuer) { async function fetchEtfFlows() { const etfs = []; let misses = 0; + const avKey = process.env.ALPHA_VANTAGE_API_KEY; + const covered = new Set(); + + // --- Primary: Alpha Vantage REALTIME_BULK_QUOTES --- + if (avKey) { + const tickers = ETF_LIST.map(e => e.ticker); + const avData = await fetchAvBulkQuotes(tickers, avKey); + for (const { ticker, issuer } of ETF_LIST) { + const av = avData.get(ticker); + if (!av) continue; + const { price, change: priceChange, volume } = av; + const direction = priceChange > 0.1 ? 'inflow' : priceChange < -0.1 ? 'outflow' : 'neutral'; + const estFlow = Math.round(volume * price * (priceChange > 0 ? 1 : -1) * 0.1); + // avgVolume and volumeRatio require 5-day history not available from REALTIME_BULK_QUOTES + etfs.push({ ticker, issuer, price: +price.toFixed(2), priceChange: +priceChange.toFixed(2), volume, avgVolume: 0, volumeRatio: 0, direction, estFlow }); + covered.add(ticker); + console.log(` [AV] ${ticker}: $${price.toFixed(2)} (${direction})`); + } + } + // --- Fallback: Yahoo (for any ETFs not covered by AV) --- + let yahooIdx = 0; for (let i = 0; i < ETF_LIST.length; i++) { const { ticker, issuer } = ETF_LIST[i]; - if (i > 0) await sleep(YAHOO_DELAY_MS); + if (covered.has(ticker)) continue; + if (yahooIdx > 0) await sleep(YAHOO_DELAY_MS); + yahooIdx++; try { const url = `https://query1.finance.yahoo.com/v8/finance/chart/${ticker}?range=5d&interval=1d`; const resp = await fetchYahooWithRetry(url, ticker); - if (!resp) { - misses++; - continue; - } + if (!resp) { misses++; continue; } const chart = await resp.json(); const parsed = parseEtfChartData(chart, ticker, issuer); if (parsed) { etfs.push(parsed); - console.log(` ${ticker}: $${parsed.price} (${parsed.direction})`); + covered.add(ticker); + console.log(` [Yahoo] ${ticker}: $${parsed.price} (${parsed.direction})`); } else { misses++; } @@ -142,7 +164,7 @@ function validate(data) { runSeed('market', 'etf-flows', CANONICAL_KEY, fetchEtfFlows, { validateFn: validate, ttlSeconds: CACHE_TTL, - sourceVersion: 'yahoo-chart-5d', + sourceVersion: 'alphavantage+yahoo-chart-5d', }).catch((err) => { const _cause = err.cause ? ` (cause: ${err.cause.message || err.cause.code || err.cause})` : ''; console.error('FATAL:', (err.message || err) + _cause); process.exit(1); diff --git a/scripts/seed-gulf-quotes.mjs b/scripts/seed-gulf-quotes.mjs index b0d0fa7a38..2db7d6c8fd 100755 --- a/scripts/seed-gulf-quotes.mjs +++ b/scripts/seed-gulf-quotes.mjs @@ -1,6 +1,7 @@ #!/usr/bin/env node -import { loadEnvFile, loadSharedConfig, CHROME_UA, runSeed } from './_seed-utils.mjs'; +import { loadEnvFile, loadSharedConfig, CHROME_UA, runSeed, sleep } from './_seed-utils.mjs'; +import { fetchAvPhysicalCommodity, fetchAvFxDaily } from './_shared-av.mjs'; const gulfConfig = loadSharedConfig('gulf.json'); @@ -12,10 +13,6 @@ const YAHOO_DELAY_MS = 200; const GULF_SYMBOLS = gulfConfig.symbols; -function sleep(ms) { - return new Promise((r) => setTimeout(r, ms)); -} - async function fetchYahooWithRetry(url, label, maxAttempts = 4) { for (let i = 0; i < maxAttempts; i++) { const resp = await fetch(url, { @@ -65,23 +62,50 @@ function parseYahooChart(data, meta) { async function fetchGulfQuotes() { const quotes = []; let misses = 0; + const avKey = process.env.ALPHA_VANTAGE_API_KEY; + const covered = new Set(); + + // --- Primary: Alpha Vantage --- + if (avKey) { + for (const meta of GULF_SYMBOLS) { + if (meta.type === 'oil') { + const q = await fetchAvPhysicalCommodity(meta.symbol, avKey); + if (q) { + quotes.push({ symbol: meta.symbol, name: meta.name, country: meta.country, flag: meta.flag, type: meta.type, price: q.price, change: +q.change.toFixed(2), sparkline: q.sparkline }); + covered.add(meta.symbol); + console.log(` [AV:physical] ${meta.symbol}: $${q.price} (${q.change > 0 ? '+' : ''}${q.change.toFixed(2)}%)`); + } + } else if (meta.type === 'currency') { + const fromCurrency = meta.symbol.replace('USD=X', ''); // 'SARUSD=X' → 'SAR' + const q = await fetchAvFxDaily(fromCurrency, avKey); + if (q) { + quotes.push({ symbol: meta.symbol, name: meta.name, country: meta.country, flag: meta.flag, type: meta.type, price: q.price, change: +q.change.toFixed(2), sparkline: q.sparkline }); + covered.add(meta.symbol); + console.log(` [AV:fx] ${meta.symbol}: ${q.price} (${q.change > 0 ? '+' : ''}${q.change.toFixed(2)}%)`); + } + } + // type === 'index' → no AV equivalent, falls through to Yahoo + } + } + // --- Fallback: Yahoo (for indices and any AV misses) --- + let yahooIdx = 0; for (let i = 0; i < GULF_SYMBOLS.length; i++) { const meta = GULF_SYMBOLS[i]; - if (i > 0) await sleep(YAHOO_DELAY_MS); + if (covered.has(meta.symbol)) continue; + if (yahooIdx > 0) await sleep(YAHOO_DELAY_MS); + yahooIdx++; try { const url = `https://query1.finance.yahoo.com/v8/finance/chart/${encodeURIComponent(meta.symbol)}`; const resp = await fetchYahooWithRetry(url, meta.symbol); - if (!resp) { - misses++; - continue; - } + if (!resp) { misses++; continue; } const chart = await resp.json(); const parsed = parseYahooChart(chart, meta); if (parsed) { quotes.push(parsed); - console.log(` ${meta.symbol}: $${parsed.price} (${parsed.change > 0 ? '+' : ''}${parsed.change}%)`); + covered.add(meta.symbol); + console.log(` [Yahoo] ${meta.symbol}: $${parsed.price} (${parsed.change > 0 ? '+' : ''}${parsed.change}%)`); } else { misses++; } @@ -105,7 +129,7 @@ function validate(data) { runSeed('market', 'gulf-quotes', CANONICAL_KEY, fetchGulfQuotes, { validateFn: validate, ttlSeconds: CACHE_TTL, - sourceVersion: 'yahoo-chart', + sourceVersion: 'alphavantage+yahoo-chart', }).catch((err) => { const _cause = err.cause ? ` (cause: ${err.cause.message || err.cause.code || err.cause})` : ''; console.error('FATAL:', (err.message || err) + _cause); process.exit(1); diff --git a/scripts/seed-market-quotes.mjs b/scripts/seed-market-quotes.mjs index 70718a3c84..4f27f91bab 100644 --- a/scripts/seed-market-quotes.mjs +++ b/scripts/seed-market-quotes.mjs @@ -1,6 +1,7 @@ #!/usr/bin/env node -import { loadEnvFile, loadSharedConfig, CHROME_UA, sleep, runSeed, parseYahooChart, writeExtraKey } from './_seed-utils.mjs'; +import { loadEnvFile, loadSharedConfig, sleep, runSeed, parseYahooChart, writeExtraKey } from './_seed-utils.mjs'; +import { fetchAvBulkQuotes } from './_shared-av.mjs'; const stocksConfig = loadSharedConfig('stocks.json'); @@ -67,34 +68,48 @@ async function fetchYahooQuote(symbol) { async function fetchMarketQuotes() { const quotes = []; - const apiKey = process.env.FINNHUB_API_KEY; - const finnhubSymbols = MARKET_SYMBOLS.filter((s) => !YAHOO_ONLY.has(s)); - const yahooSymbols = MARKET_SYMBOLS.filter((s) => YAHOO_ONLY.has(s)); + const avKey = process.env.ALPHA_VANTAGE_API_KEY; + const finnhubKey = process.env.FINNHUB_API_KEY; + + // --- Primary: Alpha Vantage REALTIME_BULK_QUOTES --- + if (avKey) { + // AV doesn't support Indian NSE symbols or Yahoo-only indices — skip those + const avSymbols = MARKET_SYMBOLS.filter((s) => !YAHOO_ONLY.has(s) && !s.endsWith('.NS')); + const avResults = await fetchAvBulkQuotes(avSymbols, avKey); + for (const [sym, q] of avResults) { + const meta = stocksConfig.symbols.find(s => s.symbol === sym); + quotes.push({ symbol: sym, name: meta?.name || sym, display: meta?.display || sym, price: q.price, change: q.change, sparkline: [] }); + console.log(` [AV] ${sym}: $${q.price} (${q.change > 0 ? '+' : ''}${q.change.toFixed(2)}%)`); + } + } + + const covered = new Set(quotes.map((q) => q.symbol)); - if (apiKey && finnhubSymbols.length > 0) { + // --- Secondary: Finnhub (for any stocks not covered by AV or if AV key not set) --- + if (finnhubKey) { + const finnhubSymbols = MARKET_SYMBOLS.filter((s) => !covered.has(s) && !YAHOO_ONLY.has(s)); for (let i = 0; i < finnhubSymbols.length; i++) { if (i > 0 && i % 10 === 0) await sleep(100); - const r = await fetchFinnhubQuote(finnhubSymbols[i], apiKey); + const r = await fetchFinnhubQuote(finnhubSymbols[i], finnhubKey); if (r) { quotes.push(r); + covered.add(r.symbol); console.log(` [Finnhub] ${r.symbol}: $${r.price} (${r.change > 0 ? '+' : ''}${r.change}%)`); } } } - const missedFinnhub = apiKey - ? finnhubSymbols.filter((s) => !quotes.some((q) => q.symbol === s)) - : finnhubSymbols; - const allYahoo = [...yahooSymbols, ...missedFinnhub]; - + // --- Fallback: Yahoo (for remaining symbols including Yahoo-only and Indian markets) --- + const allYahoo = MARKET_SYMBOLS.filter((s) => !covered.has(s)); for (let i = 0; i < allYahoo.length; i++) { const s = allYahoo[i]; - if (quotes.some((q) => q.symbol === s)) continue; if (i > 0) await sleep(YAHOO_DELAY_MS); const q = await fetchYahooQuote(s); if (q) { - quotes.push(q); - console.log(` [Yahoo] ${q.symbol}: $${q.price} (${q.change > 0 ? '+' : ''}${q.change}%)`); + const meta = stocksConfig.symbols.find(x => x.symbol === s); + quotes.push({ ...q, symbol: s, name: meta?.name || s, display: meta?.display || s }); + covered.add(s); + console.log(` [Yahoo] ${s}: $${q.price} (${q.change > 0 ? '+' : ''}${q.change}%)`); } } @@ -102,13 +117,10 @@ async function fetchMarketQuotes() { throw new Error('All market quote fetches failed'); } - const coveredByYahoo = finnhubSymbols.every((s) => quotes.some((q) => q.symbol === s)); - const skipped = !apiKey && !coveredByYahoo; - return { quotes, - finnhubSkipped: skipped, - skipReason: skipped ? 'FINNHUB_API_KEY not configured' : '', + finnhubSkipped: !finnhubKey && !avKey, + skipReason: (!finnhubKey && !avKey) ? 'ALPHA_VANTAGE_API_KEY and FINNHUB_API_KEY not configured' : '', rateLimited: false, }; } @@ -127,7 +139,7 @@ async function fetchAndStash() { runSeed('market', 'quotes', CANONICAL_KEY, fetchAndStash, { validateFn: validate, ttlSeconds: CACHE_TTL, - sourceVersion: 'yahoo+finnhub', + sourceVersion: 'alphavantage+finnhub+yahoo', }).then(async (result) => { if (result?.skipped || !seedData) return; const rpcKey = `market:quotes:v1:${[...MARKET_SYMBOLS].sort().join(',')}`; diff --git a/server/worldmonitor/market/v1/_shared.ts b/server/worldmonitor/market/v1/_shared.ts index 2322da3c7b..96eee7d27b 100644 --- a/server/worldmonitor/market/v1/_shared.ts +++ b/server/worldmonitor/market/v1/_shared.ts @@ -79,6 +79,121 @@ export interface CoinGeckoMarketItem { image?: string; } +// ======================================================================== +// Alpha Vantage fetchers +// ======================================================================== + +// Physical commodity function names for Alpha Vantage (no futures notation needed) +export const AV_PHYSICAL_COMMODITY_MAP: Record = { + 'CL=F': 'WTI', + 'BZ=F': 'BRENT', + 'NG=F': 'NATURAL_GAS', + 'HG=F': 'COPPER', + 'ALI=F': 'ALUMINUM', + 'GC=F': 'GOLD', + 'SI=F': 'SILVER', +}; + +export async function fetchAlphaVantageQuotesBatch( + symbols: string[], + apiKey: string, +): Promise> { + const results = new Map(); + const BATCH = 100; + const AV_BATCH_DELAY_MS = 500; + for (let i = 0; i < symbols.length; i += BATCH) { + if (i > 0) await new Promise(r => setTimeout(r, AV_BATCH_DELAY_MS)); + const chunk = symbols.slice(i, i + BATCH); + const url = `https://www.alphavantage.co/query?function=REALTIME_BULK_QUOTES&symbol=${encodeURIComponent(chunk.join(','))}&apikey=${encodeURIComponent(apiKey)}`; + let resp: Response | null = null; + for (let attempt = 0; attempt < 2; attempt++) { + try { + if (attempt > 0) await new Promise(r => setTimeout(r, 1000)); + resp = await fetch(url, { + headers: { 'User-Agent': CHROME_UA, Accept: 'application/json' }, + signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS), + }); + break; + } catch (err) { + console.warn(`[AV] Bulk quotes fetch error (attempt ${attempt + 1}):`, (err as Error).message); + } + } + if (!resp) continue; + if (!resp.ok) { + console.warn(`[AV] Bulk quotes HTTP ${resp.status}`); + continue; + } + try { + const json = await resp.json() as { data?: Array<{ symbol: string; price: string; 'previous close': string; 'change percent': string }>; Information?: string }; + if (json.Information) { + const remaining = symbols.length - i - chunk.length; + console.warn(`[AV] Rate limit hit${remaining > 0 ? ` — dropping ${remaining} remaining symbols` : ''}: ${json.Information.slice(0, 80)}`); + break; + } + if (!Array.isArray(json.data)) continue; + for (const item of json.data) { + const price = parseFloat(item.price); + const prevClose = parseFloat(item['previous close']); + const changePct = Number.isFinite(prevClose) && prevClose > 0 + ? ((price - prevClose) / prevClose) * 100 + : parseFloat((item['change percent'] || '0').replace('%', '')); + if (Number.isFinite(price) && price > 0) { + results.set(item.symbol, { price, change: Number.isFinite(changePct) ? changePct : 0, sparkline: [] }); + } + } + } catch (err) { + console.warn(`[AV] Bulk quotes parse error:`, (err as Error).message); + } + } + return results; +} + +export async function fetchAlphaVantagePhysicalCommodity( + yahooSymbol: string, + apiKey: string, +): Promise<{ price: number; change: number; sparkline: number[] } | null> { + const fn = AV_PHYSICAL_COMMODITY_MAP[yahooSymbol]; + if (!fn) return null; + const url = `https://www.alphavantage.co/query?function=${fn}&interval=daily&apikey=${encodeURIComponent(apiKey)}`; + let resp: Response | null = null; + for (let attempt = 0; attempt < 2; attempt++) { + try { + if (attempt > 0) await new Promise(r => setTimeout(r, 1000)); + resp = await fetch(url, { + headers: { 'User-Agent': CHROME_UA, Accept: 'application/json' }, + signal: AbortSignal.timeout(UPSTREAM_TIMEOUT_MS), + }); + break; + } catch (err) { + console.warn(`[AV] ${fn} fetch error (attempt ${attempt + 1}):`, (err as Error).message); + } + } + if (!resp) return null; + if (!resp.ok) { + console.warn(`[AV] ${fn} HTTP ${resp.status}`); + return null; + } + try { + const json = await resp.json() as { data?: Array<{ date: string; value: string }>; Information?: string }; + if (json.Information) { + console.warn(`[AV] Rate limit hit: ${json.Information.slice(0, 100)}`); + return null; + } + const data = json.data; + if (!Array.isArray(data) || data.length < 2) return null; + const latest = parseFloat(data[0]!.value); + const prev = parseFloat(data[1]!.value); + if (!Number.isFinite(latest) || latest <= 0) return null; + const change = Number.isFinite(prev) && prev > 0 ? ((latest - prev) / prev) * 100 : 0; + // Build sparkline from last 7 daily closes (oldest → newest) + const sparkline = data.slice(0, 7).map(d => parseFloat(d.value)).filter(Number.isFinite).reverse(); + return { price: latest, change, sparkline }; + } catch (err) { + console.warn(`[AV] ${fn} parse error:`, (err as Error).message); + return null; + } +} + // ======================================================================== // Finnhub quote fetcher // ========================================================================