diff --git a/core/_2_asr.py b/core/_2_asr.py index f54e8b10..fad9d298 100644 --- a/core/_2_asr.py +++ b/core/_2_asr.py @@ -1,6 +1,6 @@ from core.utils import * from core.asr_backend.demucs_vl import demucs_audio -from core.asr_backend.audio_preprocess import process_transcription, convert_video_to_audio, split_audio, save_results, normalize_audio_volume +from core.asr_backend.audio_preprocess import process_transcription, convert_video_to_audio, split_audio, save_results, save_segments, normalize_audio_volume from core._1_ytdlp import find_video_files from core.utils.models import * @@ -43,8 +43,9 @@ def transcribe(): combined_result['segments'].extend(result['segments']) # 6. Process df + save_segments(combined_result) df = process_transcription(combined_result) save_results(df) if __name__ == "__main__": - transcribe() \ No newline at end of file + transcribe() diff --git a/core/asr_backend/audio_preprocess.py b/core/asr_backend/audio_preprocess.py index 0d0db2ff..a100fb60 100644 --- a/core/asr_backend/audio_preprocess.py +++ b/core/asr_backend/audio_preprocess.py @@ -177,5 +177,30 @@ def save_results(df: pd.DataFrame): df.to_excel(_2_CLEANED_CHUNKS, index=False) rprint(f"[green]📊 Excel file saved to {_2_CLEANED_CHUNKS}[/green]") +def save_segments(result: Dict): + """Save ASR segment-level text for NLP splitting. + + Word/character-level rows remain in cleaned_chunks.xlsx for precise timestamp + alignment. Segment text is a better input for spaCy/LLM splitting in CJK + languages where WhisperX alignment may be character-level. + """ + os.makedirs('output/log', exist_ok=True) + rows = [] + for segment in result.get('segments', []): + text = str(segment.get('text', '')).strip() + if not text and segment.get('words'): + text = ''.join(str(word.get('word', word.get('text', ''))) for word in segment['words']).strip() + if not text: + continue + rows.append({ + 'text': text, + 'start': segment.get('start'), + 'end': segment.get('end'), + 'speaker_id': segment.get('speaker_id') + }) + + pd.DataFrame(rows, columns=['text', 'start', 'end', 'speaker_id']).to_excel(_2_ASR_SEGMENTS, index=False) + rprint(f"[green]📊 Segment file saved to {_2_ASR_SEGMENTS}[/green]") + def save_language(language: str): - update_key("whisper.detected_language", language) \ No newline at end of file + update_key("whisper.detected_language", language) diff --git a/core/asr_backend/whisperX_local.py b/core/asr_backend/whisperX_local.py index da96c7b8..c64a01ca 100644 --- a/core/asr_backend/whisperX_local.py +++ b/core/asr_backend/whisperX_local.py @@ -121,6 +121,7 @@ def load_audio_segment(audio_file, start, end): # Save language update_key("whisper.language", result['language']) + update_key("whisper.detected_language", result['language']) if result['language'] == 'zh' and WHISPER_LANGUAGE != 'zh': raise ValueError("Please specify the transcription language as zh and try again!") @@ -147,4 +148,4 @@ def load_audio_segment(audio_file, start, end): word['start'] += start if 'end' in word: word['end'] += start - return result \ No newline at end of file + return result diff --git a/core/spacy_utils/load_nlp_model.py b/core/spacy_utils/load_nlp_model.py index c296a886..e0140e31 100644 --- a/core/spacy_utils/load_nlp_model.py +++ b/core/spacy_utils/load_nlp_model.py @@ -12,7 +12,8 @@ def get_spacy_model(language: str): @except_handler("Failed to load NLP Spacy model") def init_nlp(): - language = "en" if load_key("whisper.language") == "en" else load_key("whisper.detected_language") + whisper_language = load_key("whisper.language") + language = load_key("whisper.detected_language") if whisper_language == "auto" else whisper_language model = get_spacy_model(language) rprint(f"[blue]⏳ Loading NLP Spacy model: <{model}> ...[/blue]") try: diff --git a/core/spacy_utils/split_by_mark.py b/core/spacy_utils/split_by_mark.py index 3cd31275..8695ac50 100644 --- a/core/spacy_utils/split_by_mark.py +++ b/core/spacy_utils/split_by_mark.py @@ -3,49 +3,97 @@ import warnings from core.spacy_utils.load_nlp_model import init_nlp, SPLIT_BY_MARK_FILE from core.utils.config_utils import load_key, get_joiner +from core.utils.models import _2_ASR_SEGMENTS, _2_CLEANED_CHUNKS from rich import print as rprint warnings.filterwarnings("ignore", category=FutureWarning) +MAX_NLP_INPUT_BYTES = 40000 + +def _clean_text(value): + if pd.isna(value): + return '' + return str(value or '').strip().strip('"').strip() + +def _split_text_by_bytes(text, max_bytes=MAX_NLP_INPUT_BYTES): + parts = [] + current = '' + for char in text: + candidate = current + char + if current and len(candidate.encode('utf-8')) > max_bytes: + parts.append(current) + current = char + else: + current = candidate + if current: + parts.append(current) + return parts + +def _merge_units_by_bytes(units, joiner, max_bytes=MAX_NLP_INPUT_BYTES): + batches = [] + current = '' + for unit in units: + if not unit: + continue + candidate = unit if not current else current + joiner + unit + if current and len(candidate.encode('utf-8')) > max_bytes: + batches.append(current) + current = unit + else: + current = candidate + if current: + batches.append(current) + return batches + +def _load_nlp_inputs(joiner): + if os.path.exists(_2_ASR_SEGMENTS): + segments = pd.read_excel(_2_ASR_SEGMENTS) + texts = [_clean_text(text) for text in segments.get('text', [])] + texts = [text for text in texts if text] + if texts: + rprint(f"[blue]🔍 Using ASR segment text for NLP splitting: {len(texts)} segment(s)[/blue]") + return [part for text in texts for part in _split_text_by_bytes(text)] + + rprint(f"[yellow]⚠️ {_2_ASR_SEGMENTS} not found or empty, rebuilding NLP text from {_2_CLEANED_CHUNKS}.[/yellow]") + chunks = pd.read_excel(_2_CLEANED_CHUNKS) + texts = [_clean_text(text) for text in chunks.text.to_list()] + return _merge_units_by_bytes(texts, joiner) + def split_by_mark(nlp): whisper_language = load_key("whisper.language") language = load_key("whisper.detected_language") if whisper_language == 'auto' else whisper_language # consider force english case joiner = get_joiner(language) rprint(f"[blue]🔍 Using {language} language joiner: '{joiner}'[/blue]") - chunks = pd.read_excel("output/log/cleaned_chunks.xlsx") - chunks.text = chunks.text.apply(lambda x: x.strip('"').strip("")) - - # join with joiner - input_text = joiner.join(chunks.text.to_list()) - - doc = nlp(input_text) - assert doc.has_annotation("SENT_START") + input_texts = _load_nlp_inputs(joiner) # skip - and ... sentences_by_mark = [] current_sentence = [] # iterate all sentences - for sent in doc.sents: - text = sent.text.strip() - - # check if the current sentence ends with - or ... - if current_sentence and ( - text.startswith('-') or - text.startswith('...') or - current_sentence[-1].endswith('-') or - current_sentence[-1].endswith('...') - ): - current_sentence.append(text) - else: - if current_sentence: - sentences_by_mark.append(' '.join(current_sentence)) - current_sentence = [] - current_sentence.append(text) + for input_text in input_texts: + doc = nlp(input_text) + assert doc.has_annotation("SENT_START") + for sent in doc.sents: + text = sent.text.strip() + + # check if the current sentence ends with - or ... + if current_sentence and ( + text.startswith('-') or + text.startswith('...') or + current_sentence[-1].endswith('-') or + current_sentence[-1].endswith('...') + ): + current_sentence.append(text) + else: + if current_sentence: + sentences_by_mark.append(joiner.join(current_sentence)) + current_sentence = [] + current_sentence.append(text) # add the last sentence if current_sentence: - sentences_by_mark.append(' '.join(current_sentence)) + sentences_by_mark.append(joiner.join(current_sentence)) with open(SPLIT_BY_MARK_FILE, "w", encoding="utf-8") as output_file: for i, sentence in enumerate(sentences_by_mark): diff --git a/core/utils/models.py b/core/utils/models.py index 8e15c829..45b9f328 100644 --- a/core/utils/models.py +++ b/core/utils/models.py @@ -3,6 +3,7 @@ # ------------------------------------------ _2_CLEANED_CHUNKS = "output/log/cleaned_chunks.xlsx" +_2_ASR_SEGMENTS = "output/log/asr_segments.xlsx" _3_1_SPLIT_BY_NLP = "output/log/split_by_nlp.txt" _3_2_SPLIT_BY_MEANING = "output/log/split_by_meaning.txt" _4_1_TERMINOLOGY = "output/log/terminology.json" @@ -31,6 +32,7 @@ __all__ = [ "_2_CLEANED_CHUNKS", + "_2_ASR_SEGMENTS", "_3_1_SPLIT_BY_NLP", "_3_2_SPLIT_BY_MEANING", "_4_1_TERMINOLOGY",