import os import tempfile import torch import torchaudio from fastapi import FastAPI, WebSocket, WebSocketDisconnect from fastapi.middleware.cors import CORSMiddleware from faster_whisper import WhisperModel from df.enhance import enhance, init_df, load_audio, save_audio from pyannote.audio import Pipeline as DiarizationPipeline from pipeline.postprocess import process app = FastAPI() app.add_middleware( CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"], ) print("Loading Whisper model...", flush=True) model = WhisperModel("large-v3", device="cpu", compute_type="int8") print("Whisper model ready!") print("Loading DeepFilterNet model...", flush=True) df_model, df_state, _ = init_df() print("DeepFilterNet model ready!") USE_DEEPFILTERNET = False # غیرفعال - شواهد نشون داد کمکی نمی‌کنه و بعضی صداها رو خراب می‌کنه print("Loading word-alignment model...", flush=True) align_bundle = torchaudio.pipelines.MMS_FA align_model = align_bundle.get_model().to("cpu") align_labels = align_bundle.get_labels() align_dictionary = {c: i for i, c in enumerate(align_labels)} print("Word-alignment model ready!") print("Loading speaker-diarization model...", flush=True) HF_TOKEN = os.environ.get("HF_TOKEN") diarize_pipeline = DiarizationPipeline.from_pretrained( "pyannote/speaker-diarization-3.1", use_auth_token=HF_TOKEN ) print("Speaker-diarization model ready!") def realign_word_timestamps(waveform_16k, words, whisper_start, whisper_end): """ برای یه جمله، به‌جای زمان‌بندی حدسی خودِ Whisper، با یه مدل تخصصی جدا (wav2vec2 از طریق torchaudio) دقیقاً چک می‌کنه هر کلمه از کجای صدا گفته شده. اگه به هر دلیلی هم‌ترازی جواب نداد، همون زمان‌بندی قبلی Whisper دست‌نخورده می‌مونه. """ try: margin = 1 start_sample = max(0, int((whisper_start - margin) * 16000)) end_sample = int((whisper_end + margin) * 16000) clip = waveform_16k[:, start_sample:end_sample] clean_words = [] for w in words: token = "".join(ch for ch in w["word"].lower() if ch in align_dictionary) if token: clean_words.append((w, token)) if not clean_words: return words transcript = "".join(tok for _, tok in clean_words) tokens = [align_dictionary[c] for c in transcript] with torch.inference_mode(): emission, _ = align_model(clip) targets = torch.tensor([tokens], dtype=torch.int32) input_lengths = torch.tensor([emission.shape[1]]) target_lengths = torch.tensor([len(tokens)]) aligned_tokens, scores = torchaudio.functional.forced_align( emission, targets, input_lengths, target_lengths, blank=0 ) spans = torchaudio.functional.merge_tokens(aligned_tokens[0], scores[0]) ratio = clip.shape[1] / emission.shape[1] / 16000 new_words = [] span_idx = 0 for w, tok in clean_words: n_chars = len(tok) matched = [s for s in spans[span_idx:] if s.token != 0][:n_chars] if len(matched) < n_chars: new_words.append(w) continue w_start = matched[0].start * ratio + (whisper_start - margin) w_end = matched[-1].end * ratio + (whisper_start - margin) new_words.append({"word": w["word"], "start": round(max(0, w_start), 3), "end": round(max(0, w_end), 3)}) span_idx += n_chars return new_words if len(new_words) == len(words) else words except Exception as e: print(f" [align] skipped due to: {type(e).__name__}: {e}") return words def get_speaker_at(diarization, time_sec): """ برای یه لحظه‌ی مشخص از صدا، می‌گه اون لحظه مال کدوم گوینده‌ست (SPEAKER_00, SPEAKER_01, ...). اگه هیچ گوینده‌ای پیدا نشد، None برمی‌گردونه. """ for turn, _, speaker in diarization.itertracks(yield_label=True): if turn.start <= time_sec <= turn.end: return speaker return None @app.get("/health") async def health(): return {"status": "ok", "message": "SubFlow backend is running"} @app.websocket("/transcribe") async def transcribe(websocket: WebSocket): await websocket.accept() tmp_path = None enhanced_path = None try: print("Step 1: waiting for bytes") audio_bytes = await websocket.receive_bytes() print(f"Step 2: received {len(audio_bytes)} bytes") await websocket.send_json({"status": "receiving"}) with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp: tmp.write(audio_bytes) tmp_path = tmp.name print(f"Step 3: saved to {tmp_path}") audio_to_transcribe = tmp_path if USE_DEEPFILTERNET: await websocket.send_json({"status": "cleaning"}) print("Step 3.5: cleaning audio with DeepFilterNet") audio_data, _ = load_audio(tmp_path, sr=df_state.sr()) enhanced_audio = enhance(df_model, df_state, audio_data, atten_lim_db=20) enhanced_path = tmp_path.replace(".wav", "_enhanced.wav") save_audio(enhanced_path, enhanced_audio, df_state.sr()) audio_to_transcribe = enhanced_path print(f"Step 3.6: cleaned audio saved to {enhanced_path}") else: print("Step 3.5: skipping DeepFilterNet (disabled for A/B test)") await websocket.send_json({"status": "transcribing"}) print("Step 4: starting transcription") segments_raw, info = model.transcribe( audio_to_transcribe, language="en", word_timestamps=True, vad_filter=True, vad_parameters=dict( min_silence_duration_ms=300, speech_pad_ms=200, threshold=0.3, ), ) print("Step 5: transcription done") await websocket.send_json({"status": "aligning"}) print("Step 5.5: loading audio for word-alignment") align_waveform, sr = torchaudio.load(audio_to_transcribe) if sr != 16000: align_waveform = torchaudio.functional.resample(align_waveform, sr, 16000) if align_waveform.shape[0] > 1: align_waveform = align_waveform.mean(dim=0, keepdim=True) print("Step 5.6: running speaker diarization") diarization = diarize_pipeline(audio_to_transcribe) segments = [] for seg in segments_raw: words = list(seg.words) if not words: continue word_dicts = [{"word": w.word, "start": w.start, "end": w.end} for w in words] realigned = realign_word_timestamps( align_waveform, word_dicts, word_dicts[0]["start"], word_dicts[-1]["end"] ) mid_time = (realigned[0]["start"] + realigned[-1]["end"]) / 2 speaker = get_speaker_at(diarization, mid_time) segments.append({ "text": seg.text.strip(), "start": realigned[0]["start"], "end": realigned[-1]["end"], "words": realigned, "speaker": speaker, }) print(f"Step 6: got {len(segments)} segments") for i, seg in enumerate(segments): print(f" seg[{i}]: '{seg['text']}' | speaker={seg['speaker']} | dur={seg['end']-seg['start']:.2f}s") await websocket.send_json({"status": "processing"}) srt_content = process(segments) print(f"Step 7: SRT built") await websocket.send_json({ "status": "done", "srt": srt_content, "language": info.language, "duration": info.duration, }) print("Step 8: done sent!") import asyncio await asyncio.sleep(1) except WebSocketDisconnect: print("WebSocket disconnected by client") except Exception as e: print(f"ERROR: {type(e).__name__}: {e}") import traceback traceback.print_exc() try: await websocket.send_json({"status": "error", "message": str(e)}) except Exception: pass finally: if tmp_path and os.path.exists(tmp_path): os.remove(tmp_path) if enhanced_path and os.path.exists(enhanced_path): os.remove(enhanced_path) try: await websocket.close() except Exception: pass