Koshur_lughat / server.py
Sameer0313's picture
Restore server entrypoint after cache cleanup
0f25643 verified
Raw History Blame Contribute Delete
10.1 kB
import re
import unicodedata
import spaces # Must be imported before torch for ZeroGPU.
import torch
import gradio as gr
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
SRC_KASHMIRI = "kas_Arab"
SRC_ENGLISH = "eng_Latn"
MODEL_REPO = "facebook/nllb-200-distilled-600M"
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
DTYPE = torch.float16 if DEVICE == "cuda" else torch.float32
# These are conservative input repairs used by the public dictionary as well.
VARIANTS = str.maketrans({"أ": "ا", "إ": "ا", "ٱ": "ا", "ك": "ک", "ي": "ی", "ى": "ی", "ئ": "ی", "ه": "ہ", "ة": "ہ"})
def normalize_kashmiri(text: str) -> str:
"""Normalize common Arabic/OCR code-point variants without changing words."""
return unicodedata.normalize("NFC", str(text or "")).replace("\u200b", "").replace("\ufeff", "").translate(VARIANTS)
# Longest sequences must be replaced first.
TRANSLIT = [
("پھ", "pʰ"), ("تھ", "tʰ"), ("ژھ", "ʦʰ"), ("چھ", "ʧʰ"), ("ٹھ", "ʈʰ"), ("کھ", "kʰ"),
("نْ", "ñ"), ("یٖ", "yˌ"), ("وٗ", "w˔"), ("وٚ", "wᵉ"), ("ۄا", "ɔã"),
("پ", "p"), ("ب", "b"), ("ت", "t"), ("ط", "ṫ"), ("ژ", "ʦ"), ("چ", "ʧ"),
("د", "d"), ("ج", "ʤ"), ("ٹ", "ʈ"), ("ڈ", "ɖ"), ("ک", "k"), ("ق", "ḱ"),
("خ", "x"), ("گ", "g"), ("غ", "ġ"), ("ع", "ʔ"), ("ف", "f"), ("س", "s"),
("ث", "ṡ"), ("ص", "ś"), ("ش", "ʃ"), ("ز", "ż"), ("ذ", "z"), ("ض", "ź"),
("ظ", "ẑ"), ("ہ", "h"), ("ھ", "ʰ"), ("ح", "ħ"), ("م", "m"), ("ن", "n"),
("و", "w"), ("ؤ", "u͑"), ("ر", "r"), ("ڑ", "ɽ"), ("ل", "l"), ("ی", "y"),
("ؠ", "ʲ"), ("ں", "ñ"), ("ِ", "i"), ("ٕ", "ɨ"), ("ٟ", "ɨ̄"), ("ُ", "u"),
("ے", "ē"), ("ٔ", "ᵊ"), ("ٲ", "ə̄"), ("ۓ", "ɛ͑"), ("ۄ", "ɔ"), ("َ", "a"),
("ا", "ạ̄"), ("آ", "ã"), ("ْ", "͞"), ("ّ", "˖"), ("؟", "?"), ("۔", "."),
]
NEGATIVE_KASHMIRI = {"نہ", "نہٕ", "نَہ", "نَہٕ", "نا", "مَہ", "مت"}
QUESTION_KASHMIRI = {"کیا", "کوٚس", "کُس", "کَتھ", "کوت", "کٔتھ", "کِتھ"}
POSTPOSITIONS = {"منٛز", "پؠٹھ", "سٕتۍ", "سٕنٛد", "سٕنٛز", "خٲطرٕ", "تہٕ"}
AUXILIARIES = {"چھ", "چُھ", "چھِ", "چھُو", "آس", "آسِ", "آسُن", "آسٕن", "اتھ", "اوس"}
UNKNOWN_DIGRAPHS = [("tch", "چ"), ("sch", "ش"), ("sh", "ش"), ("ch", "چ"), ("kh", "خ"), ("gh", "غ"), ("ph", "ف"), ("th", "تھ"), ("dh", "دھ"), ("ng", "نگ"), ("qu", "کو"), ("ck", "ک"), ("ee", "ی"), ("oo", "و"), ("ou", "او"), ("ow", "او"), ("ai", "ے"), ("ay", "ے"), ("ea", "ی"), ("ei", "ی"), ("ie", "ی"), ("oa", "و"), ("oi", "و"), ("oy", "و")]
UNKNOWN_LETTERS = {
"a": "ا", "b": "ب", "c": "ک", "d": "د", "e": "ے", "f": "ف",
"g": "گ", "h": "ہ", "i": "ی", "j": "ج", "k": "ک", "l": "ل",
"m": "م", "n": "ن", "o": "و", "p": "پ", "q": "ق", "r": "ر",
"s": "س", "t": "ت", "u": "و", "v": "و", "w": "و", "x": "کس",
"y": "ی", "z": "ز",
}
def transliterate_unknown_english(value: str) -> str:
"""Render an English token in Kashmiri script when the model leaves it unknown."""
def convert(match):
word = match.group(0).lower()
for source, target in UNKNOWN_DIGRAPHS:
word = word.replace(source, target)
return ''.join(UNKNOWN_LETTERS.get(character, character) for character in word)
return re.sub(r"[A-Za-z][A-Za-z'-]*", convert, str(value or ""))
def grammar_review(text: str, direction: str) -> str:
if direction == "Kashmiri → English":
words = re.findall(r"[\u0600-\u06ff\u0750-\u077f\u08a0-\u08ff][\u0600-\u06ff\u0750-\u077f\u08a0-\u08ff\u064b-\u065f\u200c\u200d'’ʼ-]*", text)
question = text.rstrip().endswith(("؟", "?")) or any(word.rstrip("،؛؟۔,.!?") in QUESTION_KASHMIRI for word in words)
negative = any(word.rstrip("،؛؟۔,.!?") in NEGATIVE_KASHMIRI for word in words)
postposition = any(word.rstrip("،؛؟۔,.!?\u064b-\u065f") in POSTPOSITIONS for word in words)
auxiliary = any(word.rstrip("،؛؟۔,.!?\u064b-\u065f") in AUXILIARIES for word in words)
order = "likely verb-final / SOV" if len(words) >= 3 else "short clause"
return "Kashmiri grammar: " + " · ".join([order, "negative polarity" if negative else "affirmative polarity", "question" if question else "statement", "check case/postposition" if postposition else "check case", "check copula/TAM agreement" if auxiliary else "check TAM and verb agreement"]) + "."
words = re.findall(r"[A-Za-z][A-Za-z'-]*", text.lower())
question = text.rstrip().endswith("?") or bool(re.match(r"\s*(who|what|when|where|why|how|is|are|am|was|were|do|does|did|can|could|will|would|should)\b", text, re.I))
negative = any(word in {"not", "no", "never", "neither"} or word.endswith("n't") for word in words)
tense = "future" if any(word in {"will", "shall"} for word in words) else "past" if any(word in {"was", "were", "did", "had"} for word in words) else "present/unspecified"
return "English grammar: " + " · ".join(["question" if question else "statement", "negative polarity" if negative else "affirmative polarity", f"{tense} meaning", "review Kashmiri case/agreement"]) + "."
def repair_output(output: str, source: str, direction: str) -> str:
value = normalize_kashmiri(output) if direction == "English → Kashmiri" else str(output or "").strip()
question = source.rstrip().endswith("?") if direction == "English → Kashmiri" else source.rstrip().endswith(("؟", "?"))
if question:
value = re.sub(r"[.!؟?۔]+\s*$", "", value).strip() + ("؟" if direction == "English → Kashmiri" else "?")
return value
def fallback_unknown_words(source: str, output: str) -> str:
words = list(dict.fromkeys(re.findall(r"\b[A-Za-z][A-Za-z'-]{2,}\b", source)))
if len(words) == 1 and (not output or output.casefold() == words[0].casefold()):
return transliterate_unknown_english(words[0])
for word in words:
native = transliterate_unknown_english(word)
if re.search(rf"\b{re.escape(word)}\b", output, re.I):
output = re.sub(rf"\b{re.escape(word)}\b", native, output, count=1, flags=re.I)
return output
def transliterate_kashmiri(text: str) -> str:
"""Return the orthography-based Romanized reading shown beside output."""
value = normalize_kashmiri(text)
for native, roman in TRANSLIT:
value = value.replace(native, roman)
return re.sub(r"\s+([,;:!?])", r"\1", value).strip()
def load_model(repo: str):
tokenizer = AutoTokenizer.from_pretrained(repo, src_lang=SRC_KASHMIRI)
model = AutoModelForSeq2SeqLM.from_pretrained(repo, torch_dtype=DTYPE).to(DEVICE)
model.eval()
return tokenizer, model
# NLLB-200 is public and supports both kas_Arab and eng_Latn in one model.
tokenizer, model = load_model(MODEL_REPO)
def run_model(text: str, source: str, target: str) -> str:
tokenizer.src_lang = source
inputs = tokenizer([text], truncation=True, padding="longest", return_tensors="pt", return_attention_mask=True).to(DEVICE)
forced_bos = tokenizer.convert_tokens_to_ids(target)
with torch.no_grad():
generated = model.generate(**inputs, forced_bos_token_id=forced_bos, use_cache=True, max_length=256, num_beams=3, num_return_sequences=1)
return tokenizer.batch_decode(generated, skip_special_tokens=True, clean_up_tokenization_spaces=True)[0].strip()
@spaces.GPU(duration=60)
def translate(text: str, direction: str):
"""Translate one sentence with real normalization, grammar review and output repair."""
if not text or not text.strip():
return "", "", "", ""
if len(text) > 1000:
raise gr.Error("Please enter no more than 1,000 characters.")
if direction == "Kashmiri → English":
source_text = normalize_kashmiri(text.strip())
if not re.search(r"[\u0600-\u06ff\u0750-\u077f\u08a0-\u08ff]", source_text):
raise gr.Error("Enter Kashmiri in Perso-Arabic script for this direction.")
output = repair_output(run_model(source_text, SRC_KASHMIRI, SRC_ENGLISH), source_text, direction)
return output, source_text, transliterate_kashmiri(source_text), grammar_review(source_text, direction)
source_text = text.strip()
output = repair_output(run_model(source_text, SRC_ENGLISH, SRC_KASHMIRI), source_text, direction)
output = fallback_unknown_words(source_text, output)
return output, "", transliterate_kashmiri(output), grammar_review(source_text, direction)
with gr.Blocks(title="Koshur Lughat · Server translator") as demo:
gr.Markdown(
"# Koshur Lughat\n"
"## Server-side Kashmiri ↔ English translation\n"
"The model runs on this Hugging Face Space. Visitors send text to the server; they do not download the translation model."
)
direction = gr.Radio(["Kashmiri → English", "English → Kashmiri"], value="Kashmiri → English", label="Direction")
source = gr.Textbox(lines=5, label="Input sentence", placeholder="اَتہِ کٲشُر جُملہٕ لِکھِو…")
translate_button = gr.Button("Translate", variant="primary")
result = gr.Textbox(lines=5, label="Translation", interactive=False)
normalized = gr.Textbox(lines=3, label="Normalized Kashmiri input", interactive=False)
romanized = gr.Textbox(lines=3, label="Romanized reading", interactive=False)
grammar = gr.Textbox(lines=3, label="Grammar review", interactive=False)
translate_button.click(translate, inputs=[source, direction], outputs=[result, normalized, romanized, grammar])
gr.Examples(
examples=[
["مےٚ پٔر اَکھ کِتاب", "Kashmiri → English"],
["The weather is good today.", "English → Kashmiri"],
],
inputs=[source, direction],
)
gr.Markdown(
"**Accuracy note:** This is an automatic NLLB-200 model. Kashmiri dialect, spelling and agreement still need fluent-speaker review for important text."
)
demo.launch(mcp_server=True)
# live-entrypoint-20261003