feature_engine.py

import nltk
nltk.download('punkt', quiet=True)

import pandas as pd
import numpy as np
import re
from nltk.tokenize import sent_tokenize, word_tokenize


VOWELS = set("aeiouy")
PUNCTUATION_TARGETS = [',', ';', ':', '?', '!', '-', '(', '"']
MARKDOWN_TARGETS = ['#', '*', '`']

# Word banks for semantic intent and style
HEDGING_WORDS = ['might', 'may', 'could', 'perhaps', 'possibly', 'likely', 'suggests', 'appears', 'arguably', 'seemingly', 'tend', 'almost']
CONFIDENCE_WORDS = ['definitely', 'clearly', 'obviously', 'absolutely', 'certainly', 'proven', 'undoubtedly', 'inevitably', 'always', 'must', 'ensure']
TRANSITION_WORDS = ['furthermore', 'moreover', 'however', 'consequently', 'notably', 'additionally', 'conversely', 'therefore', 'thus', 'hence', 'meanwhile']
REASONING_WORDS = ['because', 'since', 'due to', 'implies', 'shows', 'proves', 'means', 'indicates', 'results']
FORMAL_WORDS = ['utilize', 'implement', 'facilitate', 'demonstrate', 'elucidate', 'substantiate', 'delineate', 'ascertain', 'analyze', 'commence']

# Pronoun banks to track POV
FIRST_PERSON = ['i', 'me', 'my', 'mine', 'myself', 'we', 'us', 'our', 'ours', 'ourselves']
SECOND_PERSON = ['you', 'your', 'yours', 'yourself', 'yourselves']
THIRD_PERSON = ['he', 'him', 'his', 'she', 'her', 'hers', 'it', 'its', 'they', 'them', 'their', 'theirs']


def count_syllables(word):
    word = str(word).lower()
    count = 0
    if len(word) == 0: return 0
    if word[0] in VOWELS: count += 1
    for index in range(1, len(word)):
        if word[index] in VOWELS and word[index - 1] not in VOWELS:
            count += 1
    if word.endswith("e"): count -= 1
    return max(1, count)

def extract_stylometric_features(text):
    if pd.isna(text) or str(text).strip() == "":
        return [0.0] * 22 # Return 22 zeros if text is missing

    raw_text = str(text)
    lower_text = raw_text.lower()

    # Tokenize
    word_tokens = word_tokenize(lower_text)
    words = [w for w in word_tokens if w.isalpha()]
    sentences = [s for s in sent_tokenize(raw_text) if s.strip()]

    word_count = len(words)
    char_count = len(raw_text)
    sentence_count = len(sentences) if len(sentences) > 0 else 1

    if word_count == 0:
        return [0.0] * 22

    # Base Complexity Ratios
    avg_word_length = sum(len(w) for w in words) / word_count
    avg_sentence_length = word_count / sentence_count
    type_token_ratio = len(set(words)) / word_count
    capital_ratio = sum(1 for c in raw_text if c.isupper()) / char_count
    digit_ratio = sum(1 for c in raw_text if c.isdigit()) / char_count
    syllable_density = sum(count_syllables(w) for w in words) / word_count

    # Punctuation Densities (Per 100 words)
    scale = 100.0 / word_count
    comma_dns = raw_text.count(',') * scale
    semi_dns = raw_text.count(';') * scale
    colon_dns = raw_text.count(':') * scale
    question_dns = raw_text.count('?') * scale
    exclaim_dns = raw_text.count('!') * scale
    dash_dns = raw_text.count('-') * scale
    paren_dns = (raw_text.count('(') + raw_text.count(')')) * scale
    quote_dns = (raw_text.count('"') + raw_text.count("'")) * scale

    # Markdown Formatting Densities (Per 100 words)
    hash_dns = raw_text.count('#') * scale
    asterisk_dns = raw_text.count('*') * scale
    backtick_dns = raw_text.count('`') * scale

    # Academic Readability Indices
    flesch = 206.835 - 1.015 * (avg_sentence_length) - 84.6 * (syllable_density)
    flesch_reading_ease = max(0.0, min(100.0, flesch))

    complex_words = sum(1 for w in words if count_syllables(w) > 2)
    gunning_fog = 0.4 * (avg_sentence_length + (100.0 * (complex_words / word_count)))

    # THE QWEN RESCUE METRICS
    code_blocks = text.count("```") 
    bullet_points = text.count("•") + text.count("- ")
    math_vars = text.count("$")

    return [
        word_count, char_count, sentence_count, # Raw baseline metrics
        avg_word_length, avg_sentence_length, type_token_ratio, capital_ratio, digit_ratio, syllable_density,
        comma_dns, semi_dns, colon_dns, question_dns, exclaim_dns, dash_dns, paren_dns, quote_dns,
        hash_dns, asterisk_dns, backtick_dns,
        flesch_reading_ease, gunning_fog, code_blocks, bullet_points, math_vars
    ]



def extract_linguistic_features(text):
    if pd.isna(text) or str(text).strip() == "":
        return [0.0] * 13

    raw_str = str(text).lower()

    # Extract only words using regex to avoid punctuation clinging to words
    words = re.findall(r'\b[a-z]+\b', raw_str)
    total_words = len(words)

    if total_words == 0:
        return [0.0] * 13

    scale = 100.0 / total_words

    # Category Densities
    hedging_dns = sum(words.count(w) for w in HEDGING_WORDS) * scale
    confid_dns = sum(words.count(w) for w in CONFIDENCE_WORDS) * scale
    transit_dns = sum(words.count(w) for w in TRANSITION_WORDS) * scale
    reasoning_dns = sum(words.count(w) for w in REASONING_WORDS) * scale
    formal_dns = sum(words.count(w) for w in FORMAL_WORDS) * scale

    # Pronoun Densities
    first_p_dns = sum(words.count(w) for w in FIRST_PERSON) * scale
    second_p_dns = sum(words.count(w) for w in SECOND_PERSON) * scale
    third_p_dns = sum(words.count(w) for w in THIRD_PERSON) * scale

    # Syntax Patterns
    contraction_count = len(re.findall(r"\b\w+n't\b|\b\w+'[a-z]+\b", raw_str))
    contraction_dns = contraction_count * scale

    negation_dns = sum(words.count(w) for w in ['not', 'no', 'never', 'neither', 'nor', 'cannot']) * scale
    question_w_dns = sum(words.count(w) for w in ['what', 'why', 'how', 'when', 'where', 'who', 'which']) * scale
    passive_proxy_dns = sum(words.count(w) for w in ['was', 'were', 'been', 'being', 'is', 'are', 'am']) * scale

    # Check for list structures (e.g., "1.", "First,", "Secondly,")
    list_marker_dns = len(re.findall(r'\b(?:firstly|secondly|thirdly|finally|lastly)\b', raw_str)) * scale

    return [
        hedging_dns, confid_dns, transit_dns, reasoning_dns, formal_dns,
        first_p_dns, second_p_dns, third_p_dns,
        contraction_dns, negation_dns, question_w_dns, passive_proxy_dns, list_marker_dns
    ]

print("Linguistic extraction function defined.")