feat: use ASR segments for CJK NLP splitting

core/_2_asr.py

@@ -1,6 +1,6 @@
 from core.utils import *
 from core.asr_backend.demucs_vl import demucs_audio
-from core.asr_backend.audio_preprocess import process_transcription, convert_video_to_audio, split_audio, save_results, normalize_audio_volume
+from core.asr_backend.audio_preprocess import process_transcription, convert_video_to_audio, split_audio, save_results, save_segments, normalize_audio_volume
 from core._1_ytdlp import find_video_files
 from core.utils.models import *
 
@@ -43,8 +43,9 @@ def transcribe():
         combined_result['segments'].extend(result['segments'])
 
     # 6. Process df
+    save_segments(combined_result)
     df = process_transcription(combined_result)
     save_results(df)
 
 if __name__ == "__main__":
-    transcribe()
+    transcribe()

core/asr_backend/audio_preprocess.py

@@ -177,5 +177,30 @@ def save_results(df: pd.DataFrame):
     df.to_excel(_2_CLEANED_CHUNKS, index=False)
     rprint(f"[green]📊 Excel file saved to {_2_CLEANED_CHUNKS}[/green]")
 
+def save_segments(result: Dict):
+    """Save ASR segment-level text for NLP splitting.
+
+    Word/character-level rows remain in cleaned_chunks.xlsx for precise timestamp
+    alignment. Segment text is a better input for spaCy/LLM splitting in CJK
+    languages where WhisperX alignment may be character-level.
+    """
+    os.makedirs('output/log', exist_ok=True)
+    rows = []
+    for segment in result.get('segments', []):
+        text = str(segment.get('text', '')).strip()
+        if not text and segment.get('words'):
+            text = ''.join(str(word.get('word', word.get('text', ''))) for word in segment['words']).strip()
+        if not text:
+            continue
+        rows.append({
+            'text': text,
+            'start': segment.get('start'),
+            'end': segment.get('end'),
+            'speaker_id': segment.get('speaker_id')
+        })
+
+    pd.DataFrame(rows, columns=['text', 'start', 'end', 'speaker_id']).to_excel(_2_ASR_SEGMENTS, index=False)
+    rprint(f"[green]📊 Segment file saved to {_2_ASR_SEGMENTS}[/green]")
+
 def save_language(language: str):
-    update_key("whisper.detected_language", language)
+    update_key("whisper.detected_language", language)

core/asr_backend/whisperX_local.py

@@ -121,6 +121,7 @@ def load_audio_segment(audio_file, start, end):
 
     # Save language
     update_key("whisper.language", result['language'])
+    update_key("whisper.detected_language", result['language'])
     if result['language'] == 'zh' and WHISPER_LANGUAGE != 'zh':
         raise ValueError("Please specify the transcription language as zh and try again!")
 
@@ -147,4 +148,4 @@ def load_audio_segment(audio_file, start, end):
                 word['start'] += start
             if 'end' in word:
                 word['end'] += start
-    return result
+    return result

core/spacy_utils/load_nlp_model.py

@@ -12,7 +12,8 @@ def get_spacy_model(language: str):
 
 @except_handler("Failed to load NLP Spacy model")
 def init_nlp():
-    language = "en" if load_key("whisper.language") == "en" else load_key("whisper.detected_language")
+    whisper_language = load_key("whisper.language")
+    language = load_key("whisper.detected_language") if whisper_language == "auto" else whisper_language
     model = get_spacy_model(language)
     rprint(f"[blue]⏳ Loading NLP Spacy model: <{model}> ...[/blue]")
     try:

core/spacy_utils/split_by_mark.py

@@ -3,49 +3,97 @@
 import warnings
 from core.spacy_utils.load_nlp_model import init_nlp, SPLIT_BY_MARK_FILE
 from core.utils.config_utils import load_key, get_joiner
+from core.utils.models import _2_ASR_SEGMENTS, _2_CLEANED_CHUNKS
 from rich import print as rprint
 
 warnings.filterwarnings("ignore", category=FutureWarning)
 
+MAX_NLP_INPUT_BYTES = 40000
+
+def _clean_text(value):
+    if pd.isna(value):
+        return ''
+    return str(value or '').strip().strip('"').strip()
+
+def _split_text_by_bytes(text, max_bytes=MAX_NLP_INPUT_BYTES):
+    parts = []
+    current = ''
+    for char in text:
+        candidate = current + char
+        if current and len(candidate.encode('utf-8')) > max_bytes:
+            parts.append(current)
+            current = char
+        else:
+            current = candidate
+    if current:
+        parts.append(current)
+    return parts
+
+def _merge_units_by_bytes(units, joiner, max_bytes=MAX_NLP_INPUT_BYTES):
+    batches = []
+    current = ''
+    for unit in units:
+        if not unit:
+            continue
+        candidate = unit if not current else current + joiner + unit
+        if current and len(candidate.encode('utf-8')) > max_bytes:
+            batches.append(current)
+            current = unit
+        else:
+            current = candidate
+    if current:
+        batches.append(current)
+    return batches
+
+def _load_nlp_inputs(joiner):
+    if os.path.exists(_2_ASR_SEGMENTS):
+        segments = pd.read_excel(_2_ASR_SEGMENTS)
+        texts = [_clean_text(text) for text in segments.get('text', [])]
+        texts = [text for text in texts if text]
+        if texts:
+            rprint(f"[blue]🔍 Using ASR segment text for NLP splitting: {len(texts)} segment(s)[/blue]")
+            return [part for text in texts for part in _split_text_by_bytes(text)]
+
+    rprint(f"[yellow]⚠️ {_2_ASR_SEGMENTS} not found or empty, rebuilding NLP text from {_2_CLEANED_CHUNKS}.[/yellow]")
+    chunks = pd.read_excel(_2_CLEANED_CHUNKS)
+    texts = [_clean_text(text) for text in chunks.text.to_list()]
+    return _merge_units_by_bytes(texts, joiner)
+
 def split_by_mark(nlp):
     whisper_language = load_key("whisper.language")
     language = load_key("whisper.detected_language") if whisper_language == 'auto' else whisper_language # consider force english case
     joiner = get_joiner(language)
     rprint(f"[blue]🔍 Using {language} language joiner: '{joiner}'[/blue]")
-    chunks = pd.read_excel("output/log/cleaned_chunks.xlsx")
-    chunks.text = chunks.text.apply(lambda x: x.strip('"').strip(""))
-
-    # join with joiner
-    input_text = joiner.join(chunks.text.to_list())
-
-    doc = nlp(input_text)
-    assert doc.has_annotation("SENT_START")
+    input_texts = _load_nlp_inputs(joiner)
 
     # skip - and ...
     sentences_by_mark = []
     current_sentence = []
 
     # iterate all sentences
-    for sent in doc.sents:
-        text = sent.text.strip()
-
-        # check if the current sentence ends with - or ...
-        if current_sentence and (
-            text.startswith('-') or 
-            text.startswith('...') or
-            current_sentence[-1].endswith('-') or
-            current_sentence[-1].endswith('...')
-        ):
-            current_sentence.append(text)
-        else:
-            if current_sentence:
-                sentences_by_mark.append(' '.join(current_sentence))
-                current_sentence = []
-            current_sentence.append(text)
+    for input_text in input_texts:
+        doc = nlp(input_text)
+        assert doc.has_annotation("SENT_START")
+        for sent in doc.sents:
+            text = sent.text.strip()
+
+            # check if the current sentence ends with - or ...
+            if current_sentence and (
+                text.startswith('-') or 
+                text.startswith('...') or
+                current_sentence[-1].endswith('-') or
+                current_sentence[-1].endswith('...')
+            ):
+                current_sentence.append(text)
+            else:
+                if current_sentence:
+                    sentences_by_mark.append(joiner.join(current_sentence))
+                    current_sentence = []
+                current_sentence.append(text)
 
     # add the last sentence
     if current_sentence:
-        sentences_by_mark.append(' '.join(current_sentence))
+        sentences_by_mark.append(joiner.join(current_sentence))
 
     with open(SPLIT_BY_MARK_FILE, "w", encoding="utf-8") as output_file:
         for i, sentence in enumerate(sentences_by_mark):

core/utils/models.py

@@ -3,6 +3,7 @@
 # ------------------------------------------
 
 _2_CLEANED_CHUNKS = "output/log/cleaned_chunks.xlsx"
+_2_ASR_SEGMENTS = "output/log/asr_segments.xlsx"
 _3_1_SPLIT_BY_NLP = "output/log/split_by_nlp.txt"
 _3_2_SPLIT_BY_MEANING = "output/log/split_by_meaning.txt"
 _4_1_TERMINOLOGY = "output/log/terminology.json"
@@ -31,6 +32,7 @@
 
 __all__ = [
     "_2_CLEANED_CHUNKS",
+    "_2_ASR_SEGMENTS",
     "_3_1_SPLIT_BY_NLP",
     "_3_2_SPLIT_BY_MEANING",
     "_4_1_TERMINOLOGY",