Update the omni search for romanized gurmukhi

.dockerignore

@@ -1,3 +1,2 @@
 node_modules/
-package-lock.json
 .env

.github/workflows/pr-build.yml

@@ -10,12 +10,12 @@ jobs:
 
     strategy:
       matrix:
-        node-version: [10.x, 12.x, 14.x, 18.x]
+        node-version: [22.x]
 
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v4
       - name: Use Node.js ${{ matrix.node-version }}
-        uses: actions/setup-node@v1
+        uses: actions/setup-node@v4
         with:
           node-version: ${{ matrix.node-version }}
       - run: npm install --legacy-peer-deps

Dockerfile

@@ -1,4 +1,4 @@
-FROM node:18.18.0
+FROM node:22
 
 # Set working directory
 WORKDIR /app

api/controllers/amritkeertan.js

@@ -72,7 +72,7 @@ exports.headers = async (req, res) => {
       'SELECT HeaderID, Gurmukhi, GurmukhiUni, Translations, Transliterations, Updated FROM AKHeaders ORDER BY HeaderID ASC';
     const rows = await conn.query(q, []);
     res.json({
-      headers: rows.map(items => lib.prepAKIndex(items)),
+      headers: rows.map((items) => lib.prepAKIndex(items)),
     });
   } catch (err) {
     lib.error(err, res, 500);
@@ -107,7 +107,7 @@ exports.index = async (req, res) => {
 
     const q = `SELECT ${allIndexColumns} WHERE 1 ${header} ${sinceQuery} ORDER BY IndexID ASC`;
     const rows = await conn.query(q, parameters);
-    out.index = rows.map(items => lib.prepAKIndex(items));
+    out.index = rows.map((items) => lib.prepAKIndex(items));
     res.json(out);
   } catch (err) {
     lib.error(err, res, 500);
@@ -135,7 +135,7 @@ exports.shabad = async (req, res) => {
 
     if (rows && rows.length > 0) {
       const header = await getHeaderInfo(rows[0].HeaderID, conn);
-      const verses = rows.map(row => lib.prepVerse(row));
+      const verses = rows.map((row) => lib.prepVerse(row));
 
       res.json({
         header,
@@ -157,7 +157,7 @@ const getHeaderInfo = async (headerID, conn, res) => {
     const q =
       'SELECT HeaderID, Gurmukhi, GurmukhiUni, Translations, Transliterations, Updated FROM AKHeaders WHERE HeaderID = ? ORDER BY HeaderID ASC';
     const row = await conn.query(q, [headerID]);
-    return row.map(items => lib.prepAKIndex(items));
+    return row.map((items) => lib.prepAKIndex(items));
   } catch (err) {
     lib.error(err, res, 500);
   } finally {

api/controllers/banis.js

@@ -46,15 +46,15 @@ LEFT JOIN Writer w USING(WriterID)
 LEFT JOIN Raag r USING(RaagID)
 LEFT JOIN Source src USING(SourceID)`;
 
-const getAll = async req => {
+const getAll = async (req) => {
   let conn;
   try {
     conn = await req.app.locals.pool.getConnection();
     const q =
       'SELECT ID, Token as token, Gurmukhi as gurmukhi, GurmukhiUni as gurmukhiUni, Transliterations as transliterations, Updated as updated FROM Banis WHERE ID < 1000 ORDER BY ID ASC';
     const rows = await conn.query(q, []);
 
-    return { rows: rows.map(banis => lib.prepBanis(banis)) };
+    return { rows: rows.map((banis) => lib.prepBanis(banis)) };
   } catch (err) {
     throw new Error(err);
   } finally {
@@ -115,7 +115,7 @@ const prepResults = ({ rows, exists, BaniID }) => {
     writer: lib.getWriter(rows[0]),
   };
 
-  const verses = rows.map(row => prepBaniVerse(row, exists));
+  const verses = rows.map((row) => prepBaniVerse(row, exists));
 
   return {
     baniInfo,

api/controllers/limiter.js

@@ -2,18 +2,17 @@ const { RateLimiter } = require('limiter');
 const cache = require('memory-cache');
 
 const isLimited = (req, res, next, rate) => {
-  if (cache.get(req.ip)) {
-    const cachedLimiter = cache.get(req.ip);
-    if (cachedLimiter.getTokensRemaining() > 1) {
-      cachedLimiter.removeTokens(1, () => {});
-      cache.put(req.ip, cachedLimiter, 10000);
-      return next();
-    }
-    return res.status(429).send('Too Many Requests');
+  let cachedLimiter = cache.get(req.ip);
+  if (!cachedLimiter) {
+    // limiter v2: constructor takes { tokensPerInterval, interval }
+    cachedLimiter = new RateLimiter({ tokensPerInterval: rate, interval: 'minute' });
+    cache.put(req.ip, cachedLimiter, 10000);
   }
-  const cachedLimiter = new RateLimiter(rate, 'minute');
-  cache.put(req.ip, cachedLimiter, 10000);
-  return next();
+  // tryRemoveTokens is sync and returns true if a token was consumed
+  if (cachedLimiter.tryRemoveTokens(1)) {
+    return next();
+  }
+  return res.status(429).send('Too Many Requests');
 };
 
 module.exports = {

api/controllers/metadata.js

@@ -27,7 +27,7 @@ exports.raags = async (req, res) => {
     const q = 'SELECT * FROM Raag ORDER BY RaagID';
 
     const rows = await conn.query(q);
-    rows.forEach(row => lib.getRaagExtended(row));
+    rows.forEach((row) => lib.getRaagExtended(row));
 
     res.json({
       rows,

api/controllers/omni.js

@@ -1,74 +1,170 @@
+/* eslint-disable no-underscore-dangle */
+
+/**
+ * omniSearch.js — Auto-detect search for Gurbani
+ *
+ * Search modes (auto-detected):
+ *   Gurmukhi mode (isGurmukhi = true)
+ *     • Single word, pure Gurmukhi chars → FirstLetterChar (char-code prefix)
+ *     • Otherwise → FirstLetterStr, MainLetters, Gurmukhi, GurmukhiUnicode
+ *
+ *   English / Romanized mode (isGurmukhi = false)
+ *     • No spaces → FirstLetterEng  (first-letter English)
+ *     • Spaces     → Romanized transliteration + consonant skeleton fallback
+ *                    + translation search, all merged and re-ranked
+ */
+
 const { MeiliSearch } = require('meilisearch');
 const lib = require('../lib');
+const { normalizePunjabiRoman, toConsonantSkeleton, searchRank } = require('./phonetic-helpers');
 
 const client = new MeiliSearch({
   host: process.env.MEILI_HOST,
   apiKey: process.env.MEILI_API_KEY,
 });
 
+// Characters that are valid in the strict Gurmukhi romanization encoding
 const GURMUKHI_CHARS = 'aAeshkKgG|cCjJ\\tTfFxqQdDnpPbBmXrlvS^Zz&LV';
 
 const omniSearch = async (req, query, isGurmukhi, SourceID, writer, liveSearch) => {
   try {
-    let processedQuery = query.trim().replaceAll('*', ',');
+    const rawQuery = query.trim().replaceAll('*', ',');
+    let results = [];
 
     const activeFilters = [];
-    if (SourceID !== 'a') {
-      activeFilters.push(`Source=${SourceID}`);
-    }
-    if (writer !== null) {
-      activeFilters.push(`Writer=${writer}`);
-    }
-
-    const searchParams = {
-      limit: 20,
-      attributesToRetrieve: ['ID'],
-    };
+    if (SourceID !== 'a') activeFilters.push(`Source=${SourceID}`);
+    if (writer !== null) activeFilters.push(`Writer=${writer}`);
+    const filterStr = activeFilters.length > 0 ? activeFilters.join(' AND ') : undefined;
 
-    if (activeFilters.length > 0) {
-      searchParams.filter = activeFilters.join(' AND ');
-    }
+    const withFilter = (params) => (filterStr ? { ...params, filter: filterStr } : params);
 
+    // ── GURMUKHI MODE ─────────────────────────────────────────────────────────
     if (isGurmukhi) {
-      if (query.split('').every(char => GURMUKHI_CHARS.includes(char))) {
-        processedQuery = '';
-        query.split('').forEach(q => {
-          const code = q.charCodeAt(0);
-          const padded = code.toString().padStart(3, '0');
-          processedQuery += `${padded},`;
-        });
+      let processedQuery = rawQuery;
+      const searchParams = {
+        limit: 20,
+        attributesToRetrieve: ['ID', 'RankingScore'],
+        showRankingScore: true,
+      };
+
+      const isStrictGurmukhi = rawQuery.split('').every((char) => GURMUKHI_CHARS.includes(char));
+
+      if (isStrictGurmukhi) {
+        processedQuery = rawQuery
+          .split('')
+          .map((char) => char.charCodeAt(0).toString().padStart(3, '0'))
+          .join(',');
+        processedQuery += ',';
         searchParams.attributesToSearchOn = ['FirstLetterChar'];
       } else {
-        searchParams.attributesToSearchOn = ['FirstLetterStr', 'MainLetters', 'Gurmukhi'];
+        searchParams.attributesToSearchOn = [
+          'FirstLetterStr',
+          'MainLetters',
+          'Gurmukhi',
+          'GurmukhiUnicode',
+        ];
       }
+
+      const gurmukhi = await client
+        .index('verses')
+        .search(processedQuery || '', withFilter(searchParams));
+
+      results = gurmukhi.hits;
+
+      // ── ENGLISH / ROMANIZED MODE ──────────────────────────────────────────────
     } else {
-      searchParams.attributesToSearchOn = [
-        'FirstLetterEng',
-        'Translation_bdb',
-        'Translation_ms',
-        'Translation_ssk',
-      ];
-    }
+      const isSingleWord = !rawQuery.includes(' ');
+
+      if (isSingleWord) {
+        const singleWordResults = await client.index('verses').search(
+          rawQuery,
+          withFilter({
+            limit: 20,
+            attributesToRetrieve: ['ID', 'RankingScore'],
+            showRankingScore: true,
+            attributesToSearchOn: [
+              'FirstLetterEng',
+              'Translation_bdb',
+              'Translation_ms',
+              'Translation_ssk',
+            ],
+          }),
+        );
 
-    let results = await client.index('verses').search(processedQuery || '', searchParams);
-    if (!isGurmukhi && results.estimatedTotalHits === 0) {
-      const words = (processedQuery || '')
-        .trim()
-        .split(/\s+/)
-        .filter(Boolean);
-      const firstLetters = words.map(w => w[0]).join('');
-      if (firstLetters) {
-        const fallbackParams = {
-          limit: 20,
-          attributesToRetrieve: ['ID'],
-          attributesToSearchOn: ['FirstLetterEng'],
-        };
-        results = await client.index('verses').search(firstLetters, fallbackParams);
+        results = singleWordResults.hits;
+      } else {
+        // ── Multi-word romanized: the main path ───────────────────────────────
+        const userNorm = normalizePunjabiRoman(rawQuery);
+        const userSkeleton = toConsonantSkeleton(userNorm);
+        const firstLetters = rawQuery
+          .trim()
+          .split(/\s+/)
+          .filter(Boolean)
+          .map((w) => w[0])
+          .join('');
+
+        const multiSearchResult = await client.multiSearch({
+          queries: [
+            // Search 1: Normalized romanization
+            withFilter({
+              q: userNorm,
+              indexUid: 'verses',
+              limit: 40,
+              attributesToRetrieve: ['ID', 'RankingScore', 'ManualPhonetic'],
+              attributesToSearchOn: ['ManualPhonetic', 'Transliteration'],
+              showRankingScore: true,
+              showRankingScoreDetails: true,
+              matchingStrategy: 'frequency',
+            }),
+            // Search 2: Consonant skeleton
+            withFilter({
+              q: userSkeleton,
+              indexUid: 'verses',
+              limit: 40,
+              attributesToRetrieve: ['ID', 'RankingScore', 'ManualPhonetic', 'ConsonantSkeleton'],
+              attributesToSearchOn: ['ConsonantSkeleton'],
+              showRankingScore: true,
+              showRankingScoreDetails: true,
+              matchingStrategy: 'frequency',
+            }),
+            // Search 3: First letter English fallback
+            withFilter({
+              q: firstLetters,
+              indexUid: 'verses',
+              limit: 20,
+              attributesToRetrieve: ['ID', 'RankingScore'],
+              attributesToSearchOn: ['FirstLetterEng'],
+              showRankingScore: true,
+              matchingStrategy: 'all',
+            }),
+          ],
+        });
+
+        // searchRank now handles all 3 result sets
+        const ranked = searchRank(rawQuery, multiSearchResult.results);
+
+        // Translation search (meaning-based, not romanized)
+        const translationResults = await client.index('verses').search(
+          rawQuery,
+          withFilter({
+            limit: 15,
+            attributesToRetrieve: ['ID', 'RankingScore'],
+            showRankingScore: true,
+            attributesToSearchOn: ['Translation_bdb', 'Translation_ms', 'Translation_ssk'],
+            matchingStrategy: 'frequency',
+          }),
+        );
+
+        const rankedIds = new Set(ranked.map((r) => r.ID));
+        const translationOnly = translationResults.hits
+          .filter((h) => !rankedIds.has(h.ID))
+          .map((h) => ({ ...h, _rankingScore: h._rankingScore * 0.5 }));
+
+        results = [...ranked, ...translationOnly];
       }
     }
-    const verseArray = results.hits
-      .map(hit => hit.ID)
-      .filter(id => id !== null && id !== undefined);
+
+    const verseArray = results.map((hit) => hit.ID).filter((id) => id !== null && id !== undefined);
 
     const preppedResults = await lib.prepResults(req, verseArray, liveSearch);
     return preppedResults;