Migrate to Dividend Chain of Events and Improve Regex Parsing

backend/ingest/field_mapper.py

@@ -285,17 +285,17 @@ def _parse_dividend(cls, purpose: str, face_value: Optional[float]) -> tuple[Opt
 
         # Try Rs format: sum all amounts if multiple exist (e.g. "Dividend - Rs 3 & Special - Rs 3")
         # 1. Aggressively remove 'face value' and 'fv' context blocks
-        _clean_purpose = re.sub(r'(?:face value|fv|equity shares? of|shares? of)\s*(?:of\s*)?(?:rs\.?|re\.?|rupees?|inr|[-/]|\s)*\d+(?:\.\d+)?', '', purpose_lower, flags=re.IGNORECASE)
+        _clean_purpose = re.sub(r'(?:face value|fv|paid-up capital|paid up capital|equity shares? of|shares? of)\s*(?:of\s*)?(?:rs\.?|re\.?|rupees?|inr|[-/]|\s|\u20b9)*\d+(?:\.\d+)?(?:/-)?(?:\s*each)?', '', purpose_lower, flags=re.IGNORECASE)
 
         # 2. Check for the 'including' or 'includes' pattern to avoid double counting
         # e.g. 'Dividend Rs 16/- (including Rs 10 special dividend)' -> We should just extract the 16.
         if 'including' in _clean_purpose or 'includes' in _clean_purpose:
-            match = re.search(r'(?:rs\.?|re\.?|rupees?|inr)\s*(\d+(?:\.\d+)?)', _clean_purpose)
+            match = re.search(r'(?:rs\.?|re\.?|rupees?|inr|\u20b9)\s*(\d+(?:\.\d+)?)', _clean_purpose)
             if match:
                 return float(match.group(1)), dividend_type
 
         # 3. Standard extraction: find all Rs matches and sum them up (for explicitly separate components joined by &)
-        rs_matches = re.findall(r'(?:rs\.?|re\.?|rupees?|inr)\s*(\d+(?:\.\d+)?)', _clean_purpose)
+        rs_matches = re.findall(r'(?:rs\.?|re\.?|rupees?|inr|\u20b9)\s*(\d+(?:\.\d+)?)', _clean_purpose)
         if rs_matches:
             total_amount = sum(float(m) for m in rs_matches)
             return total_amount, dividend_type

backend/ingest/nse_importer.py

@@ -479,24 +479,15 @@ def _process_file(self, db: Session, key: str, trade_date: date, results: dict,
                     from sqlalchemy import delete
                     # To effectively deduplicate synthesized corporate actions that might have
                     # drifted across different `trade_date` imports but belong to the same symbol/purpose:
-                    for rec in synthesized_ca_records:
-                        from sqlalchemy import or_
-                        # Crucially, do not filter deletions by `parsed_dividend_amount`, to ensure intimation records
-                        # (no amount) are properly overwritten by subsequent announcement records (with amount).
-                        # Crucial fix to preserve actual historical dividends!
-                        # We only want to delete the synthesized records that are being replaced BY THIS EXACT EVENT.
-                        # So we only delete synthesized placeholders from the SAME date or later (which means it's the exact same lifecycle event).
-                        from datetime import timedelta
-                        threshold_date = rec['date'] - timedelta(days=60) # Lifecycle events happen closely
+                    from sqlalchemy import or_
+                    from datetime import timedelta
 
+                    for rec in synthesized_ca_records:
+                        # Find potential duplicate placeholders to delete for this specific symbol
+                        # We NEVER include `ca_model.purpose == 'Dividend'` broadly as it wipes out official historical dividends.
                         stmt = delete(ca_model).where(
                             ca_model.symbol == rec['symbol'],
-                            ca_model.date >= threshold_date,
-                            or_(
-                                ca_model.purpose.like('%not yet declared%'),
-                                ca_model.purpose == 'Dividend',
-                                ca_model.purpose.like('Dividend (%')
-                            )
+                            ca_model.purpose.like('%not yet declared%')
                         )
                         db.execute(stmt)
 

backend/ingest/nse_lib.py

@@ -648,13 +648,13 @@ def get_board_meetings(self, trade_date: date) -> pd.DataFrame:
                                 subject = str(ca.get('subject', ''))
 
                                 # Extract amount from the CA subject: e.g. 'Dividend - Rs 31 Per Share'
-                                _clean_subject = re.sub(r'(?:face value|fv|equity shares? of|shares? of)\s*(?:of\s*)?(?:rs\.?|re\.?|rupees?|inr|[-/]|\s)*\d+(?:\.\d+)?', '', subject, flags=re.IGNORECASE)
+                                _clean_subject = re.sub(r'(?:face value|fv|paid-up capital|paid up capital|equity shares? of|shares? of)\s*(?:of\s*)?(?:rs\.?|re\.?|rupees?|inr|[-/]|\s|\u20b9)*\d+(?:\.\d+)?(?:/-)?(?:\s*each)?', '', subject, flags=re.IGNORECASE)
                                 if 'including' in _clean_subject.lower() or 'includes' in _clean_subject.lower():
-                                    match = re.search(r'(?:rs\.?|re\.?|rupees?|inr)\s*(\d+(?:\.\d+)?)', _clean_subject, re.IGNORECASE)
+                                    match = re.search(r'(?:rs\.?|re\.?|rupees?|inr|\u20b9)\s*(\d+(?:\.\d+)?)', _clean_subject, re.IGNORECASE)
                                     if match:
                                         found_amount = float(match.group(1))
                                 else:
-                                    matches = re.findall(r'(?:rs\.?|re\.?|rupees?|inr)\s*(\d+(?:\.\d+)?)', _clean_subject, re.IGNORECASE)
+                                    matches = re.findall(r'(?:rs\.?|re\.?|rupees?|inr|\u20b9)\s*(\d+(?:\.\d+)?)', _clean_subject, re.IGNORECASE)
                                     if matches:
                                         found_amount = sum(float(m) for m in matches)
 
@@ -687,14 +687,14 @@ def get_board_meetings(self, trade_date: date) -> pd.DataFrame:
 
                                         # Extract Amount
                                         if found_amount is None:
-                                            _clean_text = re.sub(r'(?:face value|fv|equity shares? of|shares? of)\s*(?:of\s*)?(?:rs\.?|re\.?|rupees?|inr|[-/]|\s)*\d+(?:\.\d+)?', '', attchmntText, flags=re.IGNORECASE)
+                                            _clean_text = re.sub(r'(?:face value|fv|paid-up capital|paid up capital|equity shares? of|shares? of)\s*(?:of\s*)?(?:rs\.?|re\.?|rupees?|inr|[-/]|\s|\u20b9)*\d+(?:\.\d+)?(?:/-)?(?:\s*each)?', '', attchmntText, flags=re.IGNORECASE)
 
                                             if 'including' in _clean_text.lower() or 'includes' in _clean_text.lower():
-                                                match = re.search(r'(?:rs\.?|re\.?|rupees?|inr)\s*(\d+(?:\.\d+)?)', _clean_text, re.IGNORECASE)
+                                                match = re.search(r'(?:rs\.?|re\.?|rupees?|inr|\u20b9)\s*(\d+(?:\.\d+)?)', _clean_text, re.IGNORECASE)
                                                 if match:
                                                     found_amount = float(match.group(1))
                                             else:
-                                                div_pattern = re.compile(r'(?:rs\.?|re\.?|rupees?|inr)\s*(\d+(?:\.\d+)?)', re.IGNORECASE)
+                                                div_pattern = re.compile(r'(?:rs\.?|re\.?|rupees?|inr|\u20b9)\s*(\d+(?:\.\d+)?)', re.IGNORECASE)
                                                 matches = div_pattern.findall(_clean_text)
                                                 if matches:
                                                     found_amount = sum(float(m) for m in matches)
@@ -713,16 +713,16 @@ def get_board_meetings(self, trade_date: date) -> pd.DataFrame:
                         # Fallback 2: Extracting from bm_desc and bm_purpose
                         if found_amount is None:
                             text_to_search = f"{purpose} {desc}"
-                            _clean_text_2 = re.sub(r'(?:face value|fv|equity shares? of|shares? of)\s*(?:of\s*)?(?:rs\.?|re\.?|rupees?|inr|[-/]|\s)*\d+(?:\.\d+)?', '', text_to_search, flags=re.IGNORECASE)
+                            _clean_text_2 = re.sub(r'(?:face value|fv|paid-up capital|paid up capital|equity shares? of|shares? of)\s*(?:of\s*)?(?:rs\.?|re\.?|rupees?|inr|[-/]|\s|\u20b9)*\d+(?:\.\d+)?(?:/-)?(?:\s*each)?', '', text_to_search, flags=re.IGNORECASE)
 
                             if 'including' in _clean_text_2.lower() or 'includes' in _clean_text_2.lower():
-                                match = re.search(r'(?:rs\.?|re\.?|rupees?|inr)\s*(\d+(?:\.\d+)?)', _clean_text_2, re.IGNORECASE)
+                                match = re.search(r'(?:rs\.?|re\.?|rupees?|inr|\u20b9)\s*(\d+(?:\.\d+)?)', _clean_text_2, re.IGNORECASE)
                                 if match:
                                     found_amount = float(match.group(1))
                             else:
                                 # Extract using the common UI regex patterns
                                 ui_patterns = [
-                                    r'(?:rs\.?|re\.?|rupees?|inr)\s*(\d+(?:\.\d+)?)',
+                                    r'(?:rs\.?|re\.?|rupees?|inr|\u20b9)\s*(\d+(?:\.\d+)?)',
                                     r'(\d+(?:\.\d+)?)\s*\/\-',
                                     r'dividend\s+of\s+(\d+(?:\.\d+)?)',
                                     r'dividend.*?\s+(\d+(?:\.\d+)?)\s+per'

backend/ui/static/js/specialSitTool.js

@@ -1369,6 +1369,9 @@ function renderSSDividends() {
 
         if (isOverridden) {
             expectedAmountHTML = `${expectedAmountHTML} <span class="asterisk-mark" style="color: #ffeb3b; font-size: 1.2em; font-weight: bold; margin-left: 4px;" title="Manually Edited">*</span>`;
+        } else if (item.expected_highly_likely && typeof item.expected_highly_likely === 'string' && item.expected_highly_likely.includes('Announced:')) {
+            // If it's already officially announced, we strictly show the announced value without trend arrows
+            // Just use the base expectedAmountHTML which is the announced value.
         } else if (item.expected_amount && item.expected_amount_compare) {
             let numExpected = parseFloat(item.expected_amount);
             let numLast = parseFloat(item.expected_amount_compare);