chatbot / src /translate.py
SuhasGholkar's picture
Update src/translate.py
2f96148 verified
Raw History Blame Contribute Delete
3.58 kB
# src/translate.py
import logging
from transformers import pipeline
log = logging.getLogger("translate")
# Language codes
ENG = "eng_Latn"
HINDI = "hin_Deva"
# Globals
_local_hi2en = None
_local_en2hi = None
GEN_KW = dict(max_new_tokens=128, num_beams=4) # safe on CPU, avoids truncation
def _ensure_pipes():
"""Lazy init of IndicTrans2 pipelines (distilled)."""
global _local_hi2en, _local_en2hi
log.info("_ensure_pipes: start")
if _local_hi2en is None or _local_en2hi is None:
log.info("Loading IndicTrans2 distilled pipelines...")
try:
_local_hi2en = pipeline(
"translation",
model="ai4bharat/indictrans2-indic-en-dist-200M",
trust_remote_code=True
)
_local_en2hi = pipeline(
"translation",
model="ai4bharat/indictrans2-en-indic-dist-200M",
trust_remote_code=True
)
log.info("IndicTrans2 pipelines ready")
except Exception as e:
log.exception("Could not load IndicTrans2 pipelines: %s", e)
raise
log.info("_ensure_pipes: End")
def detect_lang(text: str) -> str:
"""Super-light language detector (hacky)."""
# if contains Devanagari → assume Hindi
log.info("Inside: detect_lang")
if any("\u0900" <= ch <= "\u097F" for ch in text):
return HINDI
return ENG
def ensure_english(text: str) -> str:
"""
Return English text. If input is Hindi, translate hi->en using IndicTrans2.
Prefer the 'prefix' format; fall back to kwargs if needed.
"""
_ensure_pipes()
if not isinstance(text, str):
text = str(text or "")
text = " ".join(text.split())
if detect_lang(text) == HINDI:
log.info("ensure_english: translating HI->EN")
prefixed = f"{HINDI} {ENG} {text}"
try:
log.info("ensure_english:calling _local_hi2en ")
out = _local_hi2en(prefixed, **GEN_KW)
log.info(f" ensure_english-> _local_hi2en -> returning value: {out[0]['translation_text']}")
return out[0]["translation_text"]
except Exception as e :
log.exception("Exception occured in _local_hi2en. Exception is : %s", e)
out = _local_hi2en(text, src_lang=HINDI, tgt_lang=ENG, **GEN_KW)
return out[0]["translation_text"]
return text
def en_to_lang(text: str, tgt_lang: str = HINDI) -> str:
"""
Translate English -> target Indic. Prefer the 'prefix' format; fallback to kwargs.
"""
try:
_ensure_pipes()
except Exception as e:
log.exception("en_to_lang --> Exception occured in _ensure_pipes. Exception is : %s", e)
log.info("en_to_lang: post _ensure_pipes")
if not isinstance(text, str):
text = str(text or "")
text = " ".join(text.split())
log.info(f"en_to_lang: Proceeding to conversion Tgt_lang = {tgt_lang} ")
if tgt_lang == HINDI and detect_lang(text) != HINDI:
log.info("en_to_lang: translating EN->HI")
prefixed = f"{ENG} {HINDI} {text}"
try:
out = _local_en2hi(prefixed, **GEN_KW)
log.info(f"en_to_lang returning value: {out[0]['translation_text']}")
return out[0]["translation_text"]
except Exception as e:
log.exception("Exception occured in en_to_lang. Exception is : %s", e)
out = _local_en2hi(text, src_lang=ENG, tgt_lang=HINDI, **GEN_KW)
return out[0]["translation_text"]
return text