SuhasGholkar commited on
Commit
c73808e
·
verified ·
1 Parent(s): 4b8e988

Update src/translate.py

Browse files
Files changed (1) hide show
  1. src/translate.py +38 -103
src/translate.py CHANGED
@@ -1,115 +1,50 @@
1
  # src/translate.py
2
- from __future__ import annotations
3
- import os
4
- import re
5
- from typing import Tuple, Optional
6
-
7
  from transformers import pipeline
 
8
 
9
- # ---- Language codes used by IndicTrans2 ----
10
- ENG = "eng_Latn"
11
- HIN = "hin_Deva"
12
 
13
- # Toggle translation via env if needed
14
- ENABLE_TRANSLATION = os.getenv("ENABLE_TRANSLATION", "1") == "1"
 
15
 
16
- # Distilled model IDs (CPU-friendly)
17
- MODEL_EN2INDIC = os.getenv(
18
- "INDICTRANS2_EN2INDIC",
19
- "ai4bharat/indictrans2-en-indic-dist-320M",
20
- )
21
- MODEL_INDIC2EN = os.getenv(
22
- "INDICTRANS2_INDIC2EN",
23
- "ai4bharat/indictrans2-indic-en-dist-320M",
24
- )
25
 
26
- # Cache dir (optional; helps on HF Spaces if you have persistent storage)
27
- HF_CACHE = os.getenv("HF_HOME") or os.getenv("TRANSFORMERS_CACHE")
 
 
 
 
 
28
 
29
- # Globals: lazy-loaded once
30
- _EN2INDIC_PIPE = None # translation pipeline EN->Indic
31
- _INDIC2EN_PIPE = None # translation pipeline Indic->EN
 
32
 
33
- # Very light language detection: Devanagari = Hindi
34
- _RE_DEVANAGARI = re.compile(r"[\u0900-\u097F]")
 
 
35
 
36
  def detect_lang(text: str) -> str:
37
- """Return ENG or HIN based on presence of Devanagari script."""
38
- if text and _RE_DEVANAGARI.search(text):
39
- return HIN
40
  return ENG
41
 
42
- def _load_en2indic():
43
- global _EN2INDIC_PIPE
44
- if _EN2INDIC_PIPE is not None:
45
- return _EN2INDIC_PIPE
46
- if not ENABLE_TRANSLATION:
47
- return None
48
- _EN2INDIC_PIPE = pipeline(
49
- task="translation",
50
- model=MODEL_EN2INDIC,
51
- trust_remote_code=True,
52
- cache_dir=HF_CACHE,
53
- )
54
- return _EN2INDIC_PIPE
55
-
56
- def _load_indic2en():
57
- global _INDIC2EN_PIPE
58
- if _INDIC2EN_PIPE is not None:
59
- return _INDIC2EN_PIPE
60
- if not ENABLE_TRANSLATION:
61
- return None
62
- _INDIC2EN_PIPE = pipeline(
63
- task="translation",
64
- model=MODEL_INDIC2EN,
65
- trust_remote_code=True,
66
- cache_dir=HF_CACHE,
67
- )
68
- return _INDIC2EN_PIPE
69
-
70
- def to_english(text: str, src_lang: str = HIN) -> str:
71
- """Translate Indic → English; graceful no-op on failure/disabled."""
72
- if not text or not ENABLE_TRANSLATION:
73
- return text
74
- try:
75
- pipe = _load_indic2en()
76
- if pipe is None:
77
- return text
78
- out = pipe(text, src_lang=src_lang, tgt_lang=ENG)
79
- return out[0]["translation_text"]
80
- except Exception:
81
- return text
82
-
83
- def from_english(text: str, tgt_lang: str = HIN) -> str:
84
- """Translate English → target Indic; graceful no-op on failure/disabled."""
85
- if not text or not ENABLE_TRANSLATION:
86
- return text
87
- try:
88
- pipe = _load_en2indic()
89
- if pipe is None:
90
- return text
91
- out = pipe(text, src_lang=ENG, tgt_lang=tgt_lang)
92
- return out[0]["translation_text"]
93
- except Exception:
94
- return text
95
-
96
- # -------- High-level helpers for chat.py --------
97
-
98
- def normalize_user_input(user_text: str) -> Tuple[str, str]:
99
- """
100
- Returns (english_text, src_lang_code).
101
- If user typed Hindi, we translate to English for downstream intent+SQL.
102
- """
103
- lang = detect_lang(user_text)
104
- if lang == HIN:
105
- return to_english(user_text, src_lang=HIN), HIN
106
- return user_text, ENG
107
-
108
- def translate_for_display(bot_reply_en: str, user_src_lang: str) -> str:
109
- """
110
- If the user typed Hindi originally, translate the English bot reply back to Hindi.
111
- Otherwise return English reply as-is.
112
- """
113
- if user_src_lang == HIN:
114
- return from_english(bot_reply_en, tgt_lang=HIN)
115
- return bot_reply_en
 
1
  # src/translate.py
2
+ import os, logging
3
+ from typing import Optional
 
 
 
4
  from transformers import pipeline
5
+ from src.translate_init import ensure_models_local
6
 
7
+ log = logging.getLogger(__name__)
 
 
8
 
9
+ # Public labels your app already uses
10
+ ENG = "eng_Latn"
11
+ HINDI = "hin_Deva"
12
 
13
+ _local_hi2en: Optional[object] = None
14
+ _local_en2hi: Optional[object] = None
 
 
 
 
 
 
 
15
 
16
+ def _ensure_pipes():
17
+ global _local_hi2en, _local_en2hi
18
+ if _local_hi2en and _local_en2hi:
19
+ return
20
+ # Make sure models exist locally (download once)
21
+ cache_dir = os.getenv("HF_CACHE_DIR", "/data/hf-cache")
22
+ local_indic_en, local_en_indic = ensure_models_local(cache_dir=cache_dir)
23
 
24
+ log.info("translate: loading IndicTrans2 distilled from local cache...")
25
+ # IMPORTANT: trust_remote_code=True is needed for IndicTrans2 repos
26
+ _local_hi2en = pipeline("translation", model=local_indic_en, trust_remote_code=True)
27
+ _local_en2hi = pipeline("translation", model=local_en_indic, trust_remote_code=True)
28
 
29
+ # quick smoke test
30
+ _ = _local_hi2en("नमस्ते")
31
+ _ = _local_en2hi("hello")
32
+ log.info("translate: IndicTrans2 distilled is ready (local).")
33
 
34
  def detect_lang(text: str) -> str:
35
+ # trivially: if devanagari present → Hindi, else English
36
+ if any('\u0900' <= ch <= '\u097F' for ch in text):
37
+ return HINDI
38
  return ENG
39
 
40
+ def ensure_english(text: str) -> str:
41
+ _ensure_pipes()
42
+ if detect_lang(text) == HINDI:
43
+ return _local_hi2en(text)[0]["translation_text"]
44
+ return text
45
+
46
+ def en_to_lang(text: str, tgt_lang: str = HINDI) -> str:
47
+ _ensure_pipes()
48
+ if tgt_lang == HINDI and detect_lang(text) != HINDI:
49
+ return _local_en2hi(text)[0]["translation_text"]
50
+ return text