Subflow / app.py
AliJabbarpour's picture
Update app.py
0295fc8 verified
Raw History Blame
8.77 kB
import os
import tempfile
import torch
import torchaudio
from fastapi import FastAPI, WebSocket, WebSocketDisconnect
from fastapi.middleware.cors import CORSMiddleware
from faster_whisper import WhisperModel
from df.enhance import enhance, init_df, load_audio, save_audio
from pyannote.audio import Pipeline as DiarizationPipeline
from pipeline.postprocess import process
app = FastAPI()
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_methods=["*"],
allow_headers=["*"],
)
print("Loading Whisper model...", flush=True)
model = WhisperModel("large-v3", device="cpu", compute_type="int8")
print("Whisper model ready!")
print("Loading DeepFilterNet model...", flush=True)
df_model, df_state, _ = init_df()
print("DeepFilterNet model ready!")
USE_DEEPFILTERNET = False # غیرفعال - شواهد نشون داد کمکی نمی‌کنه و بعضی صداها رو خراب می‌کنه
print("Loading word-alignment model...", flush=True)
align_bundle = torchaudio.pipelines.MMS_FA
align_model = align_bundle.get_model().to("cpu")
align_labels = align_bundle.get_labels()
align_dictionary = {c: i for i, c in enumerate(align_labels)}
print("Word-alignment model ready!")
print("Loading speaker-diarization model...", flush=True)
HF_TOKEN = os.environ.get("HF_TOKEN")
diarize_pipeline = DiarizationPipeline.from_pretrained(
"pyannote/speaker-diarization-3.1", use_auth_token=HF_TOKEN
)
print("Speaker-diarization model ready!")
def realign_word_timestamps(waveform_16k, words, whisper_start, whisper_end):
"""
برای یه جمله، به‌جای زمان‌بندی حدسی خودِ Whisper، با یه مدل تخصصی جدا
(wav2vec2 از طریق torchaudio) دقیقاً چک می‌کنه هر کلمه از کجای صدا گفته شده.
اگه به هر دلیلی هم‌ترازی جواب نداد، همون زمان‌بندی قبلی Whisper دست‌نخورده می‌مونه.
"""
try:
margin = 1
start_sample = max(0, int((whisper_start - margin) * 16000))
end_sample = int((whisper_end + margin) * 16000)
clip = waveform_16k[:, start_sample:end_sample]
clean_words = []
for w in words:
token = "".join(ch for ch in w["word"].lower() if ch in align_dictionary)
if token:
clean_words.append((w, token))
if not clean_words:
return words
transcript = "".join(tok for _, tok in clean_words)
tokens = [align_dictionary[c] for c in transcript]
with torch.inference_mode():
emission, _ = align_model(clip)
targets = torch.tensor([tokens], dtype=torch.int32)
input_lengths = torch.tensor([emission.shape[1]])
target_lengths = torch.tensor([len(tokens)])
aligned_tokens, scores = torchaudio.functional.forced_align(
emission, targets, input_lengths, target_lengths, blank=0
)
spans = torchaudio.functional.merge_tokens(aligned_tokens[0], scores[0])
ratio = clip.shape[1] / emission.shape[1] / 16000
new_words = []
span_idx = 0
for w, tok in clean_words:
n_chars = len(tok)
matched = [s for s in spans[span_idx:] if s.token != 0][:n_chars]
if len(matched) < n_chars:
new_words.append(w)
continue
w_start = matched[0].start * ratio + (whisper_start - margin)
w_end = matched[-1].end * ratio + (whisper_start - margin)
new_words.append({"word": w["word"], "start": round(max(0, w_start), 3), "end": round(max(0, w_end), 3)})
span_idx += n_chars
return new_words if len(new_words) == len(words) else words
except Exception as e:
print(f" [align] skipped due to: {type(e).__name__}: {e}")
return words
def get_speaker_at(diarization, time_sec):
"""
برای یه لحظه‌ی مشخص از صدا، می‌گه اون لحظه مال کدوم گوینده‌ست (SPEAKER_00, SPEAKER_01, ...).
اگه هیچ گوینده‌ای پیدا نشد، None برمی‌گردونه.
"""
for turn, _, speaker in diarization.itertracks(yield_label=True):
if turn.start <= time_sec <= turn.end:
return speaker
return None
@app.get("/health")
async def health():
return {"status": "ok", "message": "SubFlow backend is running"}
@app.websocket("/transcribe")
async def transcribe(websocket: WebSocket):
await websocket.accept()
tmp_path = None
enhanced_path = None
try:
print("Step 1: waiting for bytes")
audio_bytes = await websocket.receive_bytes()
print(f"Step 2: received {len(audio_bytes)} bytes")
await websocket.send_json({"status": "receiving"})
with tempfile.NamedTemporaryFile(suffix=".wav", delete=False) as tmp:
tmp.write(audio_bytes)
tmp_path = tmp.name
print(f"Step 3: saved to {tmp_path}")
audio_to_transcribe = tmp_path
if USE_DEEPFILTERNET:
await websocket.send_json({"status": "cleaning"})
print("Step 3.5: cleaning audio with DeepFilterNet")
audio_data, _ = load_audio(tmp_path, sr=df_state.sr())
enhanced_audio = enhance(df_model, df_state, audio_data, atten_lim_db=20)
enhanced_path = tmp_path.replace(".wav", "_enhanced.wav")
save_audio(enhanced_path, enhanced_audio, df_state.sr())
audio_to_transcribe = enhanced_path
print(f"Step 3.6: cleaned audio saved to {enhanced_path}")
else:
print("Step 3.5: skipping DeepFilterNet (disabled for A/B test)")
await websocket.send_json({"status": "transcribing"})
print("Step 4: starting transcription")
segments_raw, info = model.transcribe(
audio_to_transcribe,
language="en",
word_timestamps=True,
vad_filter=True,
vad_parameters=dict(
min_silence_duration_ms=300,
speech_pad_ms=200,
threshold=0.3,
),
)
print("Step 5: transcription done")
await websocket.send_json({"status": "aligning"})
print("Step 5.5: loading audio for word-alignment")
align_waveform, sr = torchaudio.load(audio_to_transcribe)
if sr != 16000:
align_waveform = torchaudio.functional.resample(align_waveform, sr, 16000)
if align_waveform.shape[0] > 1:
align_waveform = align_waveform.mean(dim=0, keepdim=True)
print("Step 5.6: running speaker diarization")
diarization = diarize_pipeline(audio_to_transcribe)
segments = []
for seg in segments_raw:
words = list(seg.words)
if not words:
continue
word_dicts = [{"word": w.word, "start": w.start, "end": w.end} for w in words]
realigned = realign_word_timestamps(
align_waveform, word_dicts, word_dicts[0]["start"], word_dicts[-1]["end"]
)
mid_time = (realigned[0]["start"] + realigned[-1]["end"]) / 2
speaker = get_speaker_at(diarization, mid_time)
segments.append({
"text": seg.text.strip(),
"start": realigned[0]["start"],
"end": realigned[-1]["end"],
"words": realigned,
"speaker": speaker,
})
print(f"Step 6: got {len(segments)} segments")
for i, seg in enumerate(segments):
print(f" seg[{i}]: '{seg['text']}' | speaker={seg['speaker']} | dur={seg['end']-seg['start']:.2f}s")
await websocket.send_json({"status": "processing"})
srt_content = process(segments)
print(f"Step 7: SRT built")
await websocket.send_json({
"status": "done",
"srt": srt_content,
"language": info.language,
"duration": info.duration,
})
print("Step 8: done sent!")
import asyncio
await asyncio.sleep(1)
except WebSocketDisconnect:
print("WebSocket disconnected by client")
except Exception as e:
print(f"ERROR: {type(e).__name__}: {e}")
import traceback
traceback.print_exc()
try:
await websocket.send_json({"status": "error", "message": str(e)})
except Exception:
pass
finally:
if tmp_path and os.path.exists(tmp_path):
os.remove(tmp_path)
if enhanced_path and os.path.exists(enhanced_path):
os.remove(enhanced_path)
try:
await websocket.close()
except Exception:
pass