Update transcriber.py

transcriber.py

@@ -1,429 +1,17 @@
-"""
-ClearWave — Transcriber
-========================
-Primary  : Groq API (Whisper large-v3 on H100) — free 14,400s/day
-Fallback : faster-whisper large-v3 int8 (local CPU)
-
-FIXES APPLIED (original):
-  - Pre-process audio to 16kHz mono WAV before Groq (~15% accuracy gain)
-  - Added exponential backoff retry on Groq rate limit (429)
-  - vad_parameters now includes speech_pad_ms=400 to avoid cutting word starts
-  - Chunked offset: fixed in-place mutation bug + extend→append fix
-  - Unsupported Groq languages (te, kn) fall back to auto-detect gracefully
-  - Verified Groq supported language list used as gate
-
-BUGS FIXED (v2):
-  [BUG-1] _preprocess_for_whisper() temp files (_16k.wav) were never deleted
-          → disk leak: one file per chunk in long audio mode
-          → Fix: cleanup in finally block inside _transcribe_single()
-
-  [BUG-2] _get_duration() returned 0.0 silently on ffprobe failure
-          → caused silent bad-file passthrough instead of clear error
-          → Fix: raise RuntimeError if duration is 0.0
-
-  [BUG-3] _last_segments not reset at start of _transcribe_single()
-          → if preprocessing fell back to original, stale segments from
-             previous chunk could bleed into current chunk's offset pass
-          → Fix: self._last_segments = [] at top of _transcribe_single()
-
-  [BUG-4] Race condition: _last_segments is shared singleton state
-          → concurrent requests overwrite each other's segments before
-             main.py can read them after transcribe() returns
-          → Fix: transcribe() now returns segments as 4th return value
-             instead of relying on shared mutable attribute
-          → main.py call site updated: transcript, lang, method, segs = ...
-
-  [BUG-5] _norm() used raw[:2] fallback for unknown language names
-          → "bengali" → "be" (Belarusian), "urdu" → "ur" (correct by luck)
-          → Fix: expanded _norm() map + warning log for truly unknown codes
-"""
-
 import os
-import time
-import logging
-import subprocess
-import tempfile
-import shutil
-
-logger = logging.getLogger(__name__)
-
-LANG_TO_WHISPER = {
-    "auto": None, "en": "en", "te": "te",
-    "hi": "hi", "ta": "ta", "kn": "kn",
-    "bn": "bn", "ur": "ur", "ko": "ko", "vi": "vi",
-}
-
-# Groq's Whisper large-v3 supported languages — te (Telugu) and kn (Kannada)
-# are NOT in Groq's supported list → fall back to auto-detect
-GROQ_SUPPORTED_LANGS = {
-    "en", "hi", "ta", "es", "fr", "de", "ja", "zh",
-    "ar", "pt", "ru", "it", "nl", "pl", "sv", "tr",
-    "bn", "ur", "ko", "vi",
-}
-
-CHUNK_SEC   = 60   # Groq max safe chunk size (seconds)
-MAX_RETRIES = 3    # Exponential backoff retries for Groq 429
-
+from groq import Groq
 
 class Transcriber:
     def __init__(self):
-        self.groq_key      = os.environ.get("GROQ_API_KEY", "")
-        self._groq_client  = None
-        self._local_model  = None
-        # NOTE: _last_segments kept for backwards compat but is now also
-        # returned directly from transcribe() — do not rely on it under
-        # concurrent load (see BUG-4 fix notes above).
-        self._last_segments = []
-
-        if self.groq_key:
-            print("[Transcriber] Groq API key found — primary = Groq Whisper large-v3")
-            self._init_groq()
-        else:
-            print("[Transcriber] No GROQ_API_KEY — local Whisper loads on first use")
-
-    # ══════════════════════════════════════════════════════════════════
-    # PUBLIC
-    # ══════════════════════════════════════════════════════════════════
-    def transcribe(self, audio_path: str, language: str = "auto"):
-        """
-        Returns (transcript_text, detected_language, method_label, word_segments)
-
-        word_segments: list of {'word', 'start', 'end'} dicts with timestamps.
-
-        BUG-4 FIX: segments are returned directly instead of only stored on
-        self._last_segments, eliminating the race condition when the singleton
-        is shared across concurrent requests.
-        """
-        lang_hint = LANG_TO_WHISPER.get(language, None)
-
-        # BUG-2 FIX: raise clearly if duration cannot be read
-        duration = self._get_duration(audio_path)
-        if duration == 0.0:
-            raise RuntimeError(
-                f"[Transcriber] Could not read audio duration — "
-                f"file may be corrupt or unsupported: {audio_path}"
-            )
-        print(f"[Transcriber] Audio duration: {duration:.1f}s")
-
-        self._last_segments = []
-
-        if duration <= CHUNK_SEC:
-            transcript, detected, method, segs = self._transcribe_single(audio_path, lang_hint)
-        else:
-            print(f"[Transcriber] Long audio — splitting into {CHUNK_SEC}s chunks")
-            transcript, detected, method, segs = self._transcribe_chunked(
-                audio_path, lang_hint, duration
-            )
-
-        # Keep attribute in sync for any legacy callers
-        self._last_segments = segs
-        return transcript, detected, method, segs
-
-    # ══════════════════════════════════════════════════════════════════
-    # CHUNKED PROCESSING
-    # ══════════════════════════════════════════════════════════════════
-    def _transcribe_chunked(self, audio_path, language, duration):
-        tmp_dir = tempfile.mkdtemp()
-        chunks  = []
-        start   = 0
-        idx     = 0
-
-        while start < duration:
-            cp = os.path.join(tmp_dir, f"chunk_{idx:03d}.wav")
-            subprocess.run([
-                "ffmpeg", "-y", "-i", audio_path,
-                "-ss", str(start), "-t", str(CHUNK_SEC),
-                "-acodec", "pcm_s16le", "-ar", "16000", "-ac", "1", cp
-            ], capture_output=True)
-            if os.path.exists(cp):
-                chunks.append((cp, start))
-            start += CHUNK_SEC
-            idx   += 1
-
-        print(f"[Transcriber] Processing {len(chunks)} chunks...")
-        all_texts    = []
-        all_segments = []
-        detected     = language or "en"
-        method       = "unknown"
-
-        for i, (chunk_path, offset) in enumerate(chunks):
-            print(f"[Transcriber] Chunk {i+1}/{len(chunks)} (offset={offset:.0f}s)...")
-            try:
-                # BUG-4 FIX: receive segments directly — no shared-state dependency
-                text, lang, m, segs = self._transcribe_single(chunk_path, language)
-                all_texts.append(text.strip())
-                detected = lang
-                method   = m
-
-                # Apply chunk time offset to each word segment
-                for seg in segs:
-                    all_segments.append({
-                        'word':  seg['word'],
-                        'start': round(seg['start'] + offset, 3),
-                        'end':   round(seg['end']   + offset, 3),
-                    })
-
-            except Exception as e:
-                logger.warning(f"Chunk {i+1} failed: {e}")
-
-        shutil.rmtree(tmp_dir, ignore_errors=True)
-        full = " ".join(t for t in all_texts if t)
-        print(f"[Transcriber] ✅ {len(full)} chars, {len(all_segments)} word segments")
-        return full, detected, f"{method} (chunked {len(chunks)}x)", all_segments
-
-    # ══════════════════════════════════════════════════════════════════
-    # SINGLE FILE
-    # ══════════════════════════════════════════════════════════════════
-    def _transcribe_single(self, audio_path, language):
-        """
-        Returns (transcript, detected_lang, method, word_segments).
-
-        BUG-1 FIX: preprocessed temp file is always cleaned up in finally.
-        BUG-3 FIX: local segment state reset at entry so stale data never
-                   bleeds across calls.
-        """
-        # BUG-3 FIX: always start clean
-        self._last_segments = []
-
-        preprocessed = self._preprocess_for_whisper(audio_path)
-        try:
-            if self._groq_client is not None:
-                try:
-                    transcript, detected, method, segs = self._transcribe_groq(
-                        preprocessed, language
-                    )
-                    return transcript, detected, method, segs
-                except Exception as e:
-                    logger.warning(f"Groq failed ({e}), falling back to local")
-                    if self._local_model is None:
-                        self._init_local()
-
-            return self._transcribe_local(preprocessed, language)
-
-        finally:
-            # BUG-1 FIX: always delete the temp preprocessed file
-            if preprocessed != audio_path and os.path.exists(preprocessed):
-                try:
-                    os.unlink(preprocessed)
-                except Exception as cleanup_err:
-                    logger.warning(f"[Transcriber] Temp file cleanup failed: {cleanup_err}")
-
-    # ══════════════════════════════════════════════════════════════════
-    # AUDIO PRE-PROCESSING
-    # ══════════════════════════════════════════════════════════════════
-    def _preprocess_for_whisper(self, audio_path: str) -> str:
-        """
-        Convert audio to 16kHz mono WAV before transcription.
-        Whisper was trained on 16kHz audio — sending higher SR or stereo
-        reduces accuracy (~10-15% WER improvement from this step alone).
-        Returns path to preprocessed file; caller is responsible for cleanup.
-        """
-        try:
-            # Use a proper temp file so we never collide on parallel calls
-            fd, out_path = tempfile.mkstemp(suffix="_16k.wav")
-            os.close(fd)
-
-            result = subprocess.run([
-                "ffmpeg", "-y", "-i", audio_path,
-                "-ar", "16000",
-                "-ac", "1",
-                "-acodec", "pcm_s16le",
-                out_path
-            ], capture_output=True)
-
-            if result.returncode == 0 and os.path.getsize(out_path) > 0:
-                return out_path
-            else:
-                logger.warning("[Transcriber] Preprocessing failed, using original file")
-                # Clean up the empty temp file
-                try:
-                    os.unlink(out_path)
-                except Exception:
-                    pass
-                return audio_path
-
-        except Exception as e:
-            logger.warning(f"[Transcriber] Preprocess error: {e}")
-            return audio_path
-
-    # ══════════════════════════════════════════════════════════════════
-    # GROQ  (word-level timestamps + retry on 429)
-    # ══════════════════════════════════════════════════════════════════
-    def _init_groq(self):
-        try:
-            from groq import Groq
-            self._groq_client = Groq(api_key=self.groq_key)
-            print("[Transcriber] ✅ Groq client ready")
-        except Exception as e:
-            logger.warning(f"Groq init failed: {e}")
-            self._groq_client = None
-
-    def _transcribe_groq(self, audio_path, language=None):
-        # If language not in Groq's supported list, use auto-detect
-        if language and language not in GROQ_SUPPORTED_LANGS:
-            logger.info(
-                f"[Transcriber] Lang '{language}' not in Groq supported list → auto-detect"
-            )
-            language = None
-
-        t0 = time.time()
-
-        # Exponential backoff retry for rate limit (429)
-        resp = None
-        for attempt in range(1, MAX_RETRIES + 1):
-            try:
-                with open(audio_path, "rb") as f:
-                    kwargs = dict(
-                        file=f,
-                        model="whisper-large-v3",
-                        response_format="verbose_json",
-                        timestamp_granularities=["word"],
-                        temperature=0.0,
-                    )
-                    if language:
-                        kwargs["language"] = language
-                    resp = self._groq_client.audio.transcriptions.create(**kwargs)
-                break  # success
-
-            except Exception as e:
-                err_str = str(e).lower()
-                if "429" in err_str or "rate" in err_str:
-                    wait = 2 ** attempt  # 2s, 4s, 8s
-                    logger.warning(
-                        f"[Transcriber] Groq rate limit — retry {attempt}/{MAX_RETRIES} in {wait}s"
-                    )
-                    time.sleep(wait)
-                    if attempt == MAX_RETRIES:
-                        raise
-                else:
-                    raise
-
-        transcript    = resp.text.strip()
-        detected_lang = self._norm(getattr(resp, "language", language or "en") or "en")
-
-        words = getattr(resp, "words", []) or []
-        segs  = [
-            {
-                'word':  w.word.strip() if hasattr(w, 'word') else str(w),
-                'start': float(w.start) if hasattr(w, 'start') else 0.0,
-                'end':   float(w.end)   if hasattr(w, 'end')   else 0.0,
-            }
-            for w in words
-        ]
-
-        logger.info(
-            f"Groq done in {time.time()-t0:.2f}s, lang={detected_lang}, words={len(segs)}"
-        )
-        # BUG-4 FIX: return segs directly, not only via self._last_segments
-        self._last_segments = segs
-        return transcript, detected_lang, "Groq Whisper large-v3", segs
-
-    # ══════════════════════════════════════════════════════════════════
-    # LOCAL faster-whisper  (word-level timestamps + speech_pad fix)
-    # ══════════════════════════════════════════════════════════════════
-    def _init_local(self):
-        try:
-            from faster_whisper import WhisperModel
-            print("[Transcriber] Loading faster-whisper large-v3 int8 (CPU)...")
-            self._local_model = WhisperModel(
-                "large-v3", device="cpu", compute_type="int8"
+        self.api_key = os.environ.get("GROQ_API_KEY", "")
+        self.client = Groq(api_key=self.api_key)
+
+    def transcribe(self, audio_path, **kwargs):
+        with open(audio_path, "rb") as file:
+            # We remove 'language' to let the model auto-detect perfectly
+            response = self.client.audio.transcriptions.create(
+                file=(audio_path, file.read()),
+                model="whisper-large-v3",
+                response_format="verbose_json",
             )
-            print("[Transcriber] ✅ faster-whisper ready")
-        except Exception as e:
-            logger.error(f"Local Whisper init failed: {e}")
-            self._local_model = None
-
-    def _transcribe_local(self, audio_path, language=None):
-        t0 = time.time()
-        if self._local_model is None:
-            self._init_local()
-        if self._local_model is None:
-            raise RuntimeError("No transcription engine available.")
-
-        segments, info = self._local_model.transcribe(
-            audio_path,
-            language=language,
-            beam_size=5,
-            word_timestamps=True,
-            vad_filter=True,
-            # speech_pad_ms=400 prevents clipping of word starts/ends
-            vad_parameters=dict(
-                min_silence_duration_ms=500,
-                speech_pad_ms=400,
-            ),
-        )
-
-        all_words  = []
-        text_parts = []
-        for seg in segments:
-            text_parts.append(seg.text.strip())
-            if seg.words:
-                for w in seg.words:
-                    all_words.append({
-                        'word':  w.word.strip(),
-                        'start': round(w.start, 3),
-                        'end':   round(w.end,   3),
-                    })
-
-        transcript    = " ".join(text_parts).strip()
-        detected_lang = info.language or language or "en"
-
-        logger.info(f"Local done in {time.time()-t0:.2f}s, words={len(all_words)}")
-        # BUG-4 FIX: return segs directly, not only via self._last_segments
-        self._last_segments = all_words
-        return transcript, detected_lang, "faster-whisper large-v3 int8 (local)", all_words
-
-    # ══════════════════════════════════════════════════════════════════
-    # HELPERS
-    # ══════════════════════════════════════════════════════════════════
-    def _get_duration(self, audio_path):
-        try:
-            r = subprocess.run([
-                "ffprobe", "-v", "error",
-                "-show_entries", "format=duration",
-                "-of", "default=noprint_wrappers=1:nokey=1",
-                audio_path
-            ], capture_output=True, text=True)
-            val = float(r.stdout.strip())
-            return val
-        except Exception:
-            return 0.0
-
-    @staticmethod
-    def _norm(raw: str) -> str:
-        """
-        Normalise a language name or code returned by Whisper/Groq to a
-        consistent 2-letter ISO 639-1 code.
-
-        BUG-5 FIX: expanded map covers more languages; unknown codes now log
-        a warning instead of silently truncating to raw[:2] which could
-        produce wrong codes (e.g. "bengali" → "be" = Belarusian via raw[:2]).
-        """
-        LANG_MAP = {
-            # Full names (Groq verbose_json returns these)
-            "english":    "en", "telugu":     "te", "hindi":      "hi",
-            "tamil":      "ta", "kannada":    "kn", "spanish":    "es",
-            "french":     "fr", "german":     "de", "japanese":   "ja",
-            "chinese":    "zh", "arabic":     "ar", "portuguese": "pt",
-            "russian":    "ru", "italian":    "it", "dutch":      "nl",
-            "polish":     "pl", "swedish":    "sv", "turkish":    "tr",
-            "bengali":    "bn", "urdu":       "ur", "korean":     "ko",
-            "vietnamese": "vi", "malay":      "ms", "indonesian": "id",
-            # Short codes pass through
-            "en": "en", "te": "te", "hi": "hi", "ta": "ta", "kn": "kn",
-            "es": "es", "fr": "fr", "de": "de", "ja": "ja", "zh": "zh",
-            "ar": "ar", "pt": "pt", "ru": "ru", "it": "it", "nl": "nl",
-            "pl": "pl", "sv": "sv", "tr": "tr", "bn": "bn", "ur": "ur",
-            "ko": "ko", "vi": "vi", "ms": "ms", "id": "id",
-        }
-        key    = raw.lower().strip()
-        result = LANG_MAP.get(key)
-        if result:
-            return result
-        # Last-resort: use first 2 chars but warn so it's visible in logs
-        fallback = key[:2] if len(key) >= 2 else key
-        logger.warning(
-            f"[Transcriber] Unknown language '{raw}' — using fallback code '{fallback}'. "
-            f"Add it to _norm() LANG_MAP if incorrect."
-        )
-        return fallback
+        return {"text": response.text}