import os import re import json from collections import Counter from difflib import SequenceMatcher import numpy as np import soundfile as sf import gradio as gr from accent_engine import AccentBridgeEngine, phonetic_similarity, normalize_audio_safe, STOPWORDS from slide_parser import ( extract_slides_from_file, extract_priority_keywords, extract_acronyms, get_distinctive_next_keywords, should_auto_advance, ) engine = AccentBridgeEngine(finetuned_dir="./saved_model") SAMPLE_DECK_TEXT = """Slide 1: Project Overview & API Architecture Today we present our baseline model on the right slide and explain three core API modules in our open source file system. --- Slide 2: Dataset & WER Evaluation Next we evaluate WER and keyword recall across Asian accents using unseen test speakers on GPU. --- Slide 3: Failure Cases & Future Strategy Finally we analyze where the algorithm fails on rapid overlapping speech and outline our future strategy.""" def load_judge_demo_clips() -> list: manifest_path = "./demo_samples/manifest.json" if not os.path.exists(manifest_path): return [] try: with open(manifest_path, "r", encoding="utf-8") as f: items = json.load(f) return [x for x in items if os.path.exists(x.get("file", ""))] except Exception: return [] def format_srt_time(seconds: float) -> str: hrs, mins, secs = int(seconds // 3600), int((seconds % 3600) // 60), int(seconds % 60) millis = int((seconds - int(seconds)) * 1000) return f"{hrs:02d}:{mins:02d}:{secs:02d},{millis:03d}" def export_srt_file(history: list) -> str: path = "./accentbridge_subtitles.srt" lines, cursor = [], 0.0 for idx, item in enumerate(history, start=1): dur = max(2.0, item.get("duration", 4.0)) lines.append(f"{idx}\n{format_srt_time(cursor)} --> {format_srt_time(cursor + dur)}\n{item['text']}\n") cursor += dur + 0.3 with open(path, "w", encoding="utf-8") as f: f.write("\n".join(lines) if lines else "1\n00:00:00,000 --> 00:00:02,000\nNo subtitles recorded yet.\n") return path def build_dropdown_choices(slides: list) -> list: return [f"หน้าที่ {i + 1}: {s['title'][:45]}" for i, s in enumerate(slides)] def render_slide_card(slides: list, idx: int) -> str: if not slides: return "📌 **ยังไม่ได้โหลดสไลด์** — กดปุ่ม **'🎯 โหลดสไลด์ตัวอย่าง 3 หน้า'** ด้านบนเพื่อทดลองทันที หรืออัปโหลดไฟล์ของคุณเองได้เลยค่ะ" idx = max(0, min(idx, len(slides) - 1)) s = slides[idx] ac_note = f" | 🔤 ตัวย่อบนสไลด์: {', '.join(a.upper() for a in s.get('acronyms', []))}" if s.get("acronyms") else "" return f"### 📑 สไลด์หน้าที่ {idx + 1} / {len(slides)}: {s['title']}{ac_note}\n\n{s['text'][:500]}" def render_word_diff_html(base_txt: str, ab_txt: str) -> str: if not base_txt and not ab_txt: return "
Visual Word Diff: รอเปรียบเทียบคำผิด (ขีดฆ่าสีแดง) กับคำที่แก้สำเร็จ (สีเขียว)...
" b_words, a_words = base_txt.split(), ab_txt.split() chunks = [] for tag, i1, i2, j1, j2 in SequenceMatcher(None, b_words, a_words).get_opcodes(): if tag == "equal": chunks.append(" ".join(a_words[j1:j2])) else: if i1 < i2: chunks.append(f"{' '.join(b_words[i1:i2])}") if j1 < j2: chunks.append(f"{' '.join(a_words[j1:j2])} ✨") return "
Visual Diff (Baseline → AccentBridge):
" + " ".join(chunks) + "
" def load_presentation(slide_file, rough_script): slides = extract_slides_from_file(slide_file) if not slides and rough_script and rough_script.strip(): blocks = [b.strip() for b in re.split(r"\n\s*---\s*\n|\n\s*\n", rough_script) if b.strip()] slides = [ {"index": i, "title": b.splitlines()[0][:65], "text": b, "keywords": extract_priority_keywords(b, 32), "acronyms": extract_acronyms(b)} for i, b in enumerate(blocks) ] if not slides: return [], 0, render_slide_card([], 0), "file, right, slide, three, write, open source, api, wer", gr.update(choices=[], value=None) choices = build_dropdown_choices(slides) return slides, 0, render_slide_card(slides, 0), ", ".join(slides[0]["keywords"]), gr.update(choices=choices, value=choices[0]) def load_sample_deck(): slides, idx, card, kws, dd_update = load_presentation(None, SAMPLE_DECK_TEXT) return SAMPLE_DECK_TEXT, slides, idx, card, kws, dd_update def pick_judge_clip(clip_idx: int): clips = load_judge_demo_clips() if clip_idx < len(clips): c = clips[clip_idx] wav_arr, sr = sf.read(c["file"], dtype="float32") int16_arr = (np.clip(wav_arr, -1.0, 1.0) * 32767).astype(np.int16) return (sr, int16_arr), c["keywords"] return None, "file, right, slide, three, model" def save_current_keywords(slides: list, idx: int, edited_keywords: str) -> list: if slides and 0 <= idx < len(slides): slides[idx]["keywords"] = engine.parse_hotwords(edited_keywords) return slides def change_slide_manual(slides: list, current_idx: int, target_idx: int, current_keywords: str): if not slides: return slides, 0, render_slide_card([], 0), current_keywords, gr.update() slides = save_current_keywords(slides, current_idx, current_keywords) new_idx = max(0, min(target_idx, len(slides) - 1)) choices = build_dropdown_choices(slides) return slides, new_idx, render_slide_card(slides, new_idx), ", ".join(slides[new_idx]["keywords"]), gr.update(choices=choices, value=choices[new_idx]) def jump_to_slide_dd(selected_label: str, slides: list, current_idx: int, current_keywords: str): if not slides or not selected_label: return slides, current_idx, render_slide_card(slides, current_idx), current_keywords choices = build_dropdown_choices(slides) target_idx = choices.index(selected_label) if selected_label in choices else current_idx slides = save_current_keywords(slides, current_idx, current_keywords) return slides, target_idx, render_slide_card(slides, target_idx), ", ".join(slides[target_idx]["keywords"]) def extra_sensitivity_adjust(text: str, rescued: list, hotwords: list, mode: str, word_confs: dict): if mode == "Conservative (แก้เฉพาะคำที่เสียงใกล้เคียงมาก)": filtered_rescued = [(o, n) for o, n in rescued if phonetic_similarity(o, n) >= 0.87] reverted = {n: o for o, n in rescued if (o, n) not in filtered_rescued} return " ".join([reverted.get(w, w) for w in text.split()]), filtered_rescued if mode == "Aggressive (ช่วยสำเนียงไม่ชัดเต็มที่)": words, single_hw, new_rescued = text.split(), [h for h in hotwords if " " not in h], list(rescued) already_fixed = {w_part for _, n in rescued for w_part in n.split()} for i, w in enumerate(words): if w in STOPWORDS or w in single_hw or w in already_fixed or word_confs.get(w, 0.65) >= 0.92: continue best_hw, best_sim = None, 0.0 for hw in single_hw: if abs(len(hw) - len(w)) <= 3: sim = phonetic_similarity(w, hw) if sim > best_sim: best_sim, best_hw = sim, hw if best_hw and best_sim >= 0.74: words[i] = best_hw new_rescued.append((w, best_hw)) return " ".join(words), new_rescued return text, rescued def render_live_banner(text: str, rescued_pairs: list, hotwords: list, word_confs: dict) -> str: rescued_targets = {part for _, hw in rescued_pairs for part in hw.split()} hw_set = {part for hw in hotwords for part in hw.split()} styled_words = [] for w in text.split(): conf = word_confs.get(w, 0.75) if w in rescued_targets: styled_words.append(f"{w} ✨") elif w in hw_set: styled_words.append(f"{w}") elif conf < 0.50 and w not in STOPWORDS: styled_words.append(f"{w}") else: styled_words.append(w) return ( "
" + (" ".join(styled_words) if styled_words else "🎙️ รอรับเสียงพูดของคุณ... (เขียว ✨ = แก้คำเพี้ยนด้วย CMU Phonemes | ฟ้า = คำตรงสไลด์)") + "
" ) def format_coaching_summary(history: list) -> str: all_pairs = [pair for h in history for pair in h.get("rescued", [])] if not all_pairs: return "ยังไม่มีคำที่ต้องช่วยแก้" counts = Counter(all_pairs) return "💡 สรุปคำศัพท์เฉพาะตัวที่ผู้พูดควรเน้นเสียง:\n" + "\n".join( [f"• คำว่า '{target}' (ได้ยินคล้าย '{heard}' — แก้ช่วย {cnt} ครั้ง)" for (heard, target), cnt in counts.most_common(6)] ) def process_speech(audio_tuple, custom_keywords, rough_script, slides, current_idx, auto_advance_on, sensitivity_mode, gate_on, history): if audio_tuple is None: return ( render_live_banner("", [], [], {}), render_word_diff_html("", ""), "", slides, current_idx, render_slide_card(slides, current_idx), custom_keywords, gr.update(), history, "ยังไม่มีประวัติการพูด", format_coaching_summary(history), None, ) slides = save_current_keywords(slides, current_idx, custom_keywords) sr, raw_arr = audio_tuple arr = normalize_audio_safe(raw_arr, sr) duration_sec = len(arr) / 16000.0 curr_slide = slides[current_idx] if (slides and 0 <= current_idx < len(slides)) else None next_slide = slides[current_idx + 1] if (slides and auto_advance_on and current_idx + 1 < len(slides)) else None active_context = curr_slide["text"] if curr_slide else (rough_script or "") slide_topic = curr_slide["title"] if curr_slide else "" lookahead_words = get_distinctive_next_keywords(curr_slide, next_slide, limit=8) if next_slide else [] if next_slide: active_context = active_context + "\n" + next_slide["text"] combined_hw_str = (custom_keywords + ", " + ", ".join(lookahead_words)).strip(", ") if lookahead_words else custom_keywords baseline_txt = engine.transcribe_baseline(arr) ab_txt, rescued = engine.transcribe_accentbridge( arr, hotwords_raw=combined_hw_str, context_text=active_context, slide_topic=slide_topic, use_confidence_gate=gate_on ) if not ab_txt: return ( render_live_banner("", [], [], {}), render_word_diff_html("", ""), "⚠️ ไม่พบเสียงพูดที่ชัดเจน", slides, current_idx, render_slide_card(slides, current_idx), custom_keywords, gr.update(), history, "\n".join([f"[Slide {h['slide']}] {h['text']}" for h in history]), format_coaching_summary(history), None, ) word_confs = dict(engine.last_word_confidences) active_hotwords = engine.parse_hotwords(combined_hw_str, context_text=active_context) ab_txt, rescued = extra_sensitivity_adjust(ab_txt, rescued, active_hotwords, sensitivity_mode, word_confs) new_idx, updated_keywords, advance_note, dd_update = current_idx, custom_keywords, "", gr.update() if curr_slide and next_slide and auto_advance_on: should_move, reason = should_auto_advance(ab_txt, curr_slide, next_slide, phonetic_similarity) if should_move: new_idx = current_idx + 1 updated_keywords = ", ".join(slides[new_idx]["keywords"]) choices = build_dropdown_choices(slides) dd_update = gr.update(choices=choices, value=choices[new_idx]) advance_note = reason used_slide_num = (new_idx + 1) if advance_note else (current_idx + 1 if curr_slide else 1) history = list(history) + [{"slide": used_slide_num, "text": ab_txt, "duration": duration_sec, "rescued": rescued}] srt_path = export_srt_file(history) banner_html = render_live_banner(ab_txt, rescued, active_hotwords, word_confs) diff_html = render_word_diff_html(baseline_txt, ab_txt) avg_conf = (sum(word_confs.values()) / len(word_confs) * 100.0) if word_confs else 75.0 logs = [f"🎙️ ความยาว {duration_sec:.1f}s | ความมั่นใจเฉลี่ย {avg_conf:.0f}%"] if rescued: logs.append("✅ แก้คำ/ตัวย่อสำเร็จ: " + ", ".join([f"'{o}' → '{n}'" for o, n in rescued])) if advance_note: logs.append(advance_note) return ( banner_html, diff_html, " | ".join(logs), slides, new_idx, render_slide_card(slides, new_idx), updated_keywords, dd_update, history, "\n".join([f"[Slide {h['slide']}] {h['text']}" for h in history]), format_coaching_summary(history), srt_path, ) def clear_transcript_history(): return [], "ล้างประวัติซับไตเติลแล้ว", "ล้างข้อมูลสรุปคำฝึกออกเสียงแล้ว", None status_msg = ( "✅ โหลดโมเดล Fine-tuned AccentBridge + พจนานุกรมเสียง CMU ARPAbet สำเร็จ" if engine.using_finetuned else "⚡ โหมดพร้อมทดสอบทันที (รัน python train_and_eval.py เพื่อสร้างไฟล์โมเดล Fine-tuned)" ) with gr.Blocks(title="AccentBridge — Phoneme & Slide-Synced Subtitle AI") as demo: slides_state, idx_state, history_state = gr.State([]), gr.State(0), gr.State([]) gr.Markdown(f"# 🎙️ AccentBridge: Phoneme-Aligned & Slide-Synced Subtitle AI\n**สถานะ:** {status_msg}") live_banner = gr.HTML(render_live_banner("", [], [], {})) with gr.Accordion("📄 ขั้นตอนที่ 1: โหลดไฟล์ Presentation หรือกดปุ่มโหลดสไลด์ตัวอย่าง 1 คลิก", open=True): sample_btn = gr.Button("🎯 โหลดสไลด์ตัวอย่าง 3 หน้าพร้อมทดลองทันที (1-Click Sample Deck)", variant="secondary") with gr.Row(): slide_upload = gr.File(file_types=[".pptx", ".pdf", ".docx", ".txt", ".md"], type="filepath", label="อัปโหลดไฟล์สไลด์ (.pptx, .pdf, .docx, .txt)") rough_script_box = gr.Textbox(lines=3, label="หรือวางบทพูดแบ่งหน้าด้วยเครื่องหมาย ---") load_btn = gr.Button("📂 แยกหน้าสไลด์จากไฟล์/ข้อความด้านบน") with gr.Group(): gr.Markdown("### 🖥️ ขั้นตอนที่ 2: แผงควบคุมหน้าสไลด์ (กดเปลี่ยนมือ + ข้ามหน้า Q&A + เลื่อนหน้าอัตโนมัติ)") slide_display_md = gr.Markdown(render_slide_card([], 0)) with gr.Row(): prev_btn = gr.Button("◀ หน้าก่อนหน้า (Prev)") slide_jump_dd = gr.Dropdown(choices=[], label="เลือกข้ามไปหน้าสไลด์ทันที (ช่วง Q&A)", interactive=True) next_btn = gr.Button("หน้าถัดไป (Next) ▶") with gr.Row(): auto_switch_cb = gr.Checkbox(value=True, label="🔄 เลื่อนหน้าตามเสียงพูดอัตโนมัติ") gate_cb = gr.Checkbox(value=True, label="🛡️ เปิด Acoustic Confidence Gate") sensitivity_radio = gr.Radio( choices=["Conservative (แก้เฉพาะคำที่เสียงใกล้เคียงมาก)", "Balanced (สมดุล แนะนำ)", "Aggressive (ช่วยสำเนียงไม่ชัดเต็มที่)"], value="Balanced (สมดุล แนะนำ)", label="🎚️ ระดับความเข้มข้นในการดักคำจากสไลด์", ) keywords_box = gr.Textbox(value="file, right, slide, three, write, open source, api, wer, baseline", label="🎯 คำและตัวย่อเฉพาะของหน้านี้", lines=2) sample_btn.click(fn=load_sample_deck, inputs=[], outputs=[rough_script_box, slides_state, idx_state, slide_display_md, keywords_box, slide_jump_dd]) load_btn.click(fn=load_presentation, inputs=[slide_upload, rough_script_box], outputs=[slides_state, idx_state, slide_display_md, keywords_box, slide_jump_dd]) prev_btn.click(fn=lambda s, i, k: change_slide_manual(s, i, i - 1, k), inputs=[slides_state, idx_state, keywords_box], outputs=[slides_state, idx_state, slide_display_md, keywords_box, slide_jump_dd]) next_btn.click(fn=lambda s, i, k: change_slide_manual(s, i, i + 1, k), inputs=[slides_state, idx_state, keywords_box], outputs=[slides_state, idx_state, slide_display_md, keywords_box, slide_jump_dd]) slide_jump_dd.select(fn=jump_to_slide_dd, inputs=[slide_jump_dd, slides_state, idx_state, keywords_box], outputs=[slides_state, idx_state, slide_display_md, keywords_box]) gr.Markdown("### 🎤 ขั้นตอนที่ 3: พูดใส่ไมค์ หรือกดปุ่มคลิปเสียงตัวอย่างเพื่อให้กรรมการทดสอบได้ใน 1 คลิก") with gr.Row(): audio_in = gr.Audio(sources=["microphone", "upload"], type="numpy", label="อัดเสียงพูด หรืออัปโหลดไฟล์เสียง") with gr.Column(): run_btn = gr.Button("🚀 แปลงเสียงเป็นซับไตเติล", variant="primary") diff_out = gr.HTML(render_word_diff_html("", "")) rescued_log = gr.Textbox(label="🔍 สถานะเสียง & บันทึกการแก้คำ/เปลี่ยนหน้า") judge_clips = load_judge_demo_clips() if judge_clips: with gr.Row(): clip1_btn = gr.Button(f"🎧 โหลดคลิปเสียงจริงตัวอย่างที่ 1 (สำเนียง {judge_clips[0]['l1']})") clip1_btn.click(fn=lambda: pick_judge_clip(0), inputs=[], outputs=[audio_in, keywords_box]) if len(judge_clips) > 1: clip2_btn = gr.Button(f"🎧 โหลดคลิปเสียงจริงตัวอย่างที่ 2 (สำเนียง {judge_clips[1]['l1']})") clip2_btn.click(fn=lambda: pick_judge_clip(1), inputs=[], outputs=[audio_in, keywords_box]) with gr.Accordion("📝 ประวัติซับไตเติลสะสม, คำแนะนำฝึกออกเสียง & ดาวน์โหลดไฟล์ .SRT", open=True): history_box = gr.Textbox(lines=4, label="บทบันทึกซับไตเติลเรียงตามหน้าสไลด์", interactive=False) coaching_box = gr.Textbox(lines=3, label="🎓 สรุปคำที่ผู้พูดออกเสียงเพี้ยนบ่อย", interactive=False) with gr.Row(): srt_file_out = gr.File(label="📥 ดาวน์โหลดไฟล์ซับไตเติล (.srt)") clear_btn = gr.Button("🗑️ ล้างประวัติซับไตเติล") speech_inputs = [audio_in, keywords_box, rough_script_box, slides_state, idx_state, auto_switch_cb, sensitivity_radio, gate_cb, history_state] speech_outputs = [live_banner, diff_out, rescued_log, slides_state, idx_state, slide_display_md, keywords_box, slide_jump_dd, history_state, history_box, coaching_box, srt_file_out] audio_in.stop_recording(fn=process_speech, inputs=speech_inputs, outputs=speech_outputs) run_btn.click(fn=process_speech, inputs=speech_inputs, outputs=speech_outputs) clear_btn.click(fn=clear_transcript_history, inputs=[], outputs=[history_state, history_box, coaching_box, srt_file_out]) if __name__ == "__main__": demo.launch(share=("SPACE_ID" not in os.environ))