Spaces:
Sleeping
Sleeping
Update src/translate.py
Browse files- src/translate.py +38 -103
src/translate.py
CHANGED
|
@@ -1,115 +1,50 @@
|
|
| 1 |
# src/translate.py
|
| 2 |
-
|
| 3 |
-
import
|
| 4 |
-
import re
|
| 5 |
-
from typing import Tuple, Optional
|
| 6 |
-
|
| 7 |
from transformers import pipeline
|
|
|
|
| 8 |
|
| 9 |
-
|
| 10 |
-
ENG = "eng_Latn"
|
| 11 |
-
HIN = "hin_Deva"
|
| 12 |
|
| 13 |
-
#
|
| 14 |
-
|
|
|
|
| 15 |
|
| 16 |
-
|
| 17 |
-
|
| 18 |
-
"INDICTRANS2_EN2INDIC",
|
| 19 |
-
"ai4bharat/indictrans2-en-indic-dist-320M",
|
| 20 |
-
)
|
| 21 |
-
MODEL_INDIC2EN = os.getenv(
|
| 22 |
-
"INDICTRANS2_INDIC2EN",
|
| 23 |
-
"ai4bharat/indictrans2-indic-en-dist-320M",
|
| 24 |
-
)
|
| 25 |
|
| 26 |
-
|
| 27 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 28 |
|
| 29 |
-
|
| 30 |
-
|
| 31 |
-
|
|
|
|
| 32 |
|
| 33 |
-
#
|
| 34 |
-
|
|
|
|
|
|
|
| 35 |
|
| 36 |
def detect_lang(text: str) -> str:
|
| 37 |
-
|
| 38 |
-
if
|
| 39 |
-
return
|
| 40 |
return ENG
|
| 41 |
|
| 42 |
-
def
|
| 43 |
-
|
| 44 |
-
if
|
| 45 |
-
return
|
| 46 |
-
|
| 47 |
-
|
| 48 |
-
|
| 49 |
-
|
| 50 |
-
|
| 51 |
-
|
| 52 |
-
|
| 53 |
-
)
|
| 54 |
-
return _EN2INDIC_PIPE
|
| 55 |
-
|
| 56 |
-
def _load_indic2en():
|
| 57 |
-
global _INDIC2EN_PIPE
|
| 58 |
-
if _INDIC2EN_PIPE is not None:
|
| 59 |
-
return _INDIC2EN_PIPE
|
| 60 |
-
if not ENABLE_TRANSLATION:
|
| 61 |
-
return None
|
| 62 |
-
_INDIC2EN_PIPE = pipeline(
|
| 63 |
-
task="translation",
|
| 64 |
-
model=MODEL_INDIC2EN,
|
| 65 |
-
trust_remote_code=True,
|
| 66 |
-
cache_dir=HF_CACHE,
|
| 67 |
-
)
|
| 68 |
-
return _INDIC2EN_PIPE
|
| 69 |
-
|
| 70 |
-
def to_english(text: str, src_lang: str = HIN) -> str:
|
| 71 |
-
"""Translate Indic → English; graceful no-op on failure/disabled."""
|
| 72 |
-
if not text or not ENABLE_TRANSLATION:
|
| 73 |
-
return text
|
| 74 |
-
try:
|
| 75 |
-
pipe = _load_indic2en()
|
| 76 |
-
if pipe is None:
|
| 77 |
-
return text
|
| 78 |
-
out = pipe(text, src_lang=src_lang, tgt_lang=ENG)
|
| 79 |
-
return out[0]["translation_text"]
|
| 80 |
-
except Exception:
|
| 81 |
-
return text
|
| 82 |
-
|
| 83 |
-
def from_english(text: str, tgt_lang: str = HIN) -> str:
|
| 84 |
-
"""Translate English → target Indic; graceful no-op on failure/disabled."""
|
| 85 |
-
if not text or not ENABLE_TRANSLATION:
|
| 86 |
-
return text
|
| 87 |
-
try:
|
| 88 |
-
pipe = _load_en2indic()
|
| 89 |
-
if pipe is None:
|
| 90 |
-
return text
|
| 91 |
-
out = pipe(text, src_lang=ENG, tgt_lang=tgt_lang)
|
| 92 |
-
return out[0]["translation_text"]
|
| 93 |
-
except Exception:
|
| 94 |
-
return text
|
| 95 |
-
|
| 96 |
-
# -------- High-level helpers for chat.py --------
|
| 97 |
-
|
| 98 |
-
def normalize_user_input(user_text: str) -> Tuple[str, str]:
|
| 99 |
-
"""
|
| 100 |
-
Returns (english_text, src_lang_code).
|
| 101 |
-
If user typed Hindi, we translate to English for downstream intent+SQL.
|
| 102 |
-
"""
|
| 103 |
-
lang = detect_lang(user_text)
|
| 104 |
-
if lang == HIN:
|
| 105 |
-
return to_english(user_text, src_lang=HIN), HIN
|
| 106 |
-
return user_text, ENG
|
| 107 |
-
|
| 108 |
-
def translate_for_display(bot_reply_en: str, user_src_lang: str) -> str:
|
| 109 |
-
"""
|
| 110 |
-
If the user typed Hindi originally, translate the English bot reply back to Hindi.
|
| 111 |
-
Otherwise return English reply as-is.
|
| 112 |
-
"""
|
| 113 |
-
if user_src_lang == HIN:
|
| 114 |
-
return from_english(bot_reply_en, tgt_lang=HIN)
|
| 115 |
-
return bot_reply_en
|
|
|
|
| 1 |
# src/translate.py
|
| 2 |
+
import os, logging
|
| 3 |
+
from typing import Optional
|
|
|
|
|
|
|
|
|
|
| 4 |
from transformers import pipeline
|
| 5 |
+
from src.translate_init import ensure_models_local
|
| 6 |
|
| 7 |
+
log = logging.getLogger(__name__)
|
|
|
|
|
|
|
| 8 |
|
| 9 |
+
# Public labels your app already uses
|
| 10 |
+
ENG = "eng_Latn"
|
| 11 |
+
HINDI = "hin_Deva"
|
| 12 |
|
| 13 |
+
_local_hi2en: Optional[object] = None
|
| 14 |
+
_local_en2hi: Optional[object] = None
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
+
def _ensure_pipes():
|
| 17 |
+
global _local_hi2en, _local_en2hi
|
| 18 |
+
if _local_hi2en and _local_en2hi:
|
| 19 |
+
return
|
| 20 |
+
# Make sure models exist locally (download once)
|
| 21 |
+
cache_dir = os.getenv("HF_CACHE_DIR", "/data/hf-cache")
|
| 22 |
+
local_indic_en, local_en_indic = ensure_models_local(cache_dir=cache_dir)
|
| 23 |
|
| 24 |
+
log.info("translate: loading IndicTrans2 distilled from local cache...")
|
| 25 |
+
# IMPORTANT: trust_remote_code=True is needed for IndicTrans2 repos
|
| 26 |
+
_local_hi2en = pipeline("translation", model=local_indic_en, trust_remote_code=True)
|
| 27 |
+
_local_en2hi = pipeline("translation", model=local_en_indic, trust_remote_code=True)
|
| 28 |
|
| 29 |
+
# quick smoke test
|
| 30 |
+
_ = _local_hi2en("नमस्ते")
|
| 31 |
+
_ = _local_en2hi("hello")
|
| 32 |
+
log.info("translate: IndicTrans2 distilled is ready (local).")
|
| 33 |
|
| 34 |
def detect_lang(text: str) -> str:
|
| 35 |
+
# trivially: if devanagari present → Hindi, else English
|
| 36 |
+
if any('\u0900' <= ch <= '\u097F' for ch in text):
|
| 37 |
+
return HINDI
|
| 38 |
return ENG
|
| 39 |
|
| 40 |
+
def ensure_english(text: str) -> str:
|
| 41 |
+
_ensure_pipes()
|
| 42 |
+
if detect_lang(text) == HINDI:
|
| 43 |
+
return _local_hi2en(text)[0]["translation_text"]
|
| 44 |
+
return text
|
| 45 |
+
|
| 46 |
+
def en_to_lang(text: str, tgt_lang: str = HINDI) -> str:
|
| 47 |
+
_ensure_pipes()
|
| 48 |
+
if tgt_lang == HINDI and detect_lang(text) != HINDI:
|
| 49 |
+
return _local_en2hi(text)[0]["translation_text"]
|
| 50 |
+
return text
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|