import re import unicodedata import spaces # Must be imported before torch for ZeroGPU. import torch import gradio as gr from transformers import AutoModelForSeq2SeqLM, AutoTokenizer SRC_KASHMIRI = "kas_Arab" SRC_ENGLISH = "eng_Latn" MODEL_REPO = "facebook/nllb-200-distilled-600M" DEVICE = "cuda" if torch.cuda.is_available() else "cpu" DTYPE = torch.float16 if DEVICE == "cuda" else torch.float32 # These are conservative input repairs used by the public dictionary as well. VARIANTS = str.maketrans({"أ": "ا", "إ": "ا", "ٱ": "ا", "ك": "ک", "ي": "ی", "ى": "ی", "ئ": "ی", "ه": "ہ", "ة": "ہ"}) def normalize_kashmiri(text: str) -> str: """Normalize common Arabic/OCR code-point variants without changing words.""" return unicodedata.normalize("NFC", str(text or "")).replace("\u200b", "").replace("\ufeff", "").translate(VARIANTS) # Longest sequences must be replaced first. TRANSLIT = [ ("پھ", "pʰ"), ("تھ", "tʰ"), ("ژھ", "ʦʰ"), ("چھ", "ʧʰ"), ("ٹھ", "ʈʰ"), ("کھ", "kʰ"), ("نْ", "ñ"), ("یٖ", "yˌ"), ("وٗ", "w˔"), ("وٚ", "wᵉ"), ("ۄا", "ɔã"), ("پ", "p"), ("ب", "b"), ("ت", "t"), ("ط", "ṫ"), ("ژ", "ʦ"), ("چ", "ʧ"), ("د", "d"), ("ج", "ʤ"), ("ٹ", "ʈ"), ("ڈ", "ɖ"), ("ک", "k"), ("ق", "ḱ"), ("خ", "x"), ("گ", "g"), ("غ", "ġ"), ("ع", "ʔ"), ("ف", "f"), ("س", "s"), ("ث", "ṡ"), ("ص", "ś"), ("ش", "ʃ"), ("ز", "ż"), ("ذ", "z"), ("ض", "ź"), ("ظ", "ẑ"), ("ہ", "h"), ("ھ", "ʰ"), ("ح", "ħ"), ("م", "m"), ("ن", "n"), ("و", "w"), ("ؤ", "u͑"), ("ر", "r"), ("ڑ", "ɽ"), ("ل", "l"), ("ی", "y"), ("ؠ", "ʲ"), ("ں", "ñ"), ("ِ", "i"), ("ٕ", "ɨ"), ("ٟ", "ɨ̄"), ("ُ", "u"), ("ے", "ē"), ("ٔ", "ᵊ"), ("ٲ", "ə̄"), ("ۓ", "ɛ͑"), ("ۄ", "ɔ"), ("َ", "a"), ("ا", "ạ̄"), ("آ", "ã"), ("ْ", "͞"), ("ّ", "˖"), ("؟", "?"), ("۔", "."), ] NEGATIVE_KASHMIRI = {"نہ", "نہٕ", "نَہ", "نَہٕ", "نا", "مَہ", "مت"} QUESTION_KASHMIRI = {"کیا", "کوٚس", "کُس", "کَتھ", "کوت", "کٔتھ", "کِتھ"} POSTPOSITIONS = {"منٛز", "پؠٹھ", "سٕتۍ", "سٕنٛد", "سٕنٛز", "خٲطرٕ", "تہٕ"} AUXILIARIES = {"چھ", "چُھ", "چھِ", "چھُو", "آس", "آسِ", "آسُن", "آسٕن", "اتھ", "اوس"} UNKNOWN_DIGRAPHS = [("tch", "چ"), ("sch", "ش"), ("sh", "ش"), ("ch", "چ"), ("kh", "خ"), ("gh", "غ"), ("ph", "ف"), ("th", "تھ"), ("dh", "دھ"), ("ng", "نگ"), ("qu", "کو"), ("ck", "ک"), ("ee", "ی"), ("oo", "و"), ("ou", "او"), ("ow", "او"), ("ai", "ے"), ("ay", "ے"), ("ea", "ی"), ("ei", "ی"), ("ie", "ی"), ("oa", "و"), ("oi", "و"), ("oy", "و")] UNKNOWN_LETTERS = { "a": "ا", "b": "ب", "c": "ک", "d": "د", "e": "ے", "f": "ف", "g": "گ", "h": "ہ", "i": "ی", "j": "ج", "k": "ک", "l": "ل", "m": "م", "n": "ن", "o": "و", "p": "پ", "q": "ق", "r": "ر", "s": "س", "t": "ت", "u": "و", "v": "و", "w": "و", "x": "کس", "y": "ی", "z": "ز", } def transliterate_unknown_english(value: str) -> str: """Render an English token in Kashmiri script when the model leaves it unknown.""" def convert(match): word = match.group(0).lower() for source, target in UNKNOWN_DIGRAPHS: word = word.replace(source, target) return ''.join(UNKNOWN_LETTERS.get(character, character) for character in word) return re.sub(r"[A-Za-z][A-Za-z'-]*", convert, str(value or "")) def grammar_review(text: str, direction: str) -> str: if direction == "Kashmiri → English": words = re.findall(r"[\u0600-\u06ff\u0750-\u077f\u08a0-\u08ff][\u0600-\u06ff\u0750-\u077f\u08a0-\u08ff\u064b-\u065f\u200c\u200d'’ʼ-]*", text) question = text.rstrip().endswith(("؟", "?")) or any(word.rstrip("،؛؟۔,.!?") in QUESTION_KASHMIRI for word in words) negative = any(word.rstrip("،؛؟۔,.!?") in NEGATIVE_KASHMIRI for word in words) postposition = any(word.rstrip("،؛؟۔,.!?\u064b-\u065f") in POSTPOSITIONS for word in words) auxiliary = any(word.rstrip("،؛؟۔,.!?\u064b-\u065f") in AUXILIARIES for word in words) order = "likely verb-final / SOV" if len(words) >= 3 else "short clause" return "Kashmiri grammar: " + " · ".join([order, "negative polarity" if negative else "affirmative polarity", "question" if question else "statement", "check case/postposition" if postposition else "check case", "check copula/TAM agreement" if auxiliary else "check TAM and verb agreement"]) + "." words = re.findall(r"[A-Za-z][A-Za-z'-]*", text.lower()) question = text.rstrip().endswith("?") or bool(re.match(r"\s*(who|what|when|where|why|how|is|are|am|was|were|do|does|did|can|could|will|would|should)\b", text, re.I)) negative = any(word in {"not", "no", "never", "neither"} or word.endswith("n't") for word in words) tense = "future" if any(word in {"will", "shall"} for word in words) else "past" if any(word in {"was", "were", "did", "had"} for word in words) else "present/unspecified" return "English grammar: " + " · ".join(["question" if question else "statement", "negative polarity" if negative else "affirmative polarity", f"{tense} meaning", "review Kashmiri case/agreement"]) + "." def repair_output(output: str, source: str, direction: str) -> str: value = normalize_kashmiri(output) if direction == "English → Kashmiri" else str(output or "").strip() question = source.rstrip().endswith("?") if direction == "English → Kashmiri" else source.rstrip().endswith(("؟", "?")) if question: value = re.sub(r"[.!؟?۔]+\s*$", "", value).strip() + ("؟" if direction == "English → Kashmiri" else "?") return value def fallback_unknown_words(source: str, output: str) -> str: words = list(dict.fromkeys(re.findall(r"\b[A-Za-z][A-Za-z'-]{2,}\b", source))) if len(words) == 1 and (not output or output.casefold() == words[0].casefold()): return transliterate_unknown_english(words[0]) for word in words: native = transliterate_unknown_english(word) if re.search(rf"\b{re.escape(word)}\b", output, re.I): output = re.sub(rf"\b{re.escape(word)}\b", native, output, count=1, flags=re.I) return output def transliterate_kashmiri(text: str) -> str: """Return the orthography-based Romanized reading shown beside output.""" value = normalize_kashmiri(text) for native, roman in TRANSLIT: value = value.replace(native, roman) return re.sub(r"\s+([,;:!?])", r"\1", value).strip() def load_model(repo: str): tokenizer = AutoTokenizer.from_pretrained(repo, src_lang=SRC_KASHMIRI) model = AutoModelForSeq2SeqLM.from_pretrained(repo, torch_dtype=DTYPE).to(DEVICE) model.eval() return tokenizer, model # NLLB-200 is public and supports both kas_Arab and eng_Latn in one model. tokenizer, model = load_model(MODEL_REPO) def run_model(text: str, source: str, target: str) -> str: tokenizer.src_lang = source inputs = tokenizer([text], truncation=True, padding="longest", return_tensors="pt", return_attention_mask=True).to(DEVICE) forced_bos = tokenizer.convert_tokens_to_ids(target) with torch.no_grad(): generated = model.generate(**inputs, forced_bos_token_id=forced_bos, use_cache=True, max_length=256, num_beams=3, num_return_sequences=1) return tokenizer.batch_decode(generated, skip_special_tokens=True, clean_up_tokenization_spaces=True)[0].strip() @spaces.GPU(duration=60) def translate(text: str, direction: str): """Translate one sentence with real normalization, grammar review and output repair.""" if not text or not text.strip(): return "", "", "", "" if len(text) > 1000: raise gr.Error("Please enter no more than 1,000 characters.") if direction == "Kashmiri → English": source_text = normalize_kashmiri(text.strip()) if not re.search(r"[\u0600-\u06ff\u0750-\u077f\u08a0-\u08ff]", source_text): raise gr.Error("Enter Kashmiri in Perso-Arabic script for this direction.") output = repair_output(run_model(source_text, SRC_KASHMIRI, SRC_ENGLISH), source_text, direction) return output, source_text, transliterate_kashmiri(source_text), grammar_review(source_text, direction) source_text = text.strip() output = repair_output(run_model(source_text, SRC_ENGLISH, SRC_KASHMIRI), source_text, direction) output = fallback_unknown_words(source_text, output) return output, "", transliterate_kashmiri(output), grammar_review(source_text, direction) with gr.Blocks(title="Koshur Lughat · Server translator") as demo: gr.Markdown( "# Koshur Lughat\n" "## Server-side Kashmiri ↔ English translation\n" "The model runs on this Hugging Face Space. Visitors send text to the server; they do not download the translation model." ) direction = gr.Radio(["Kashmiri → English", "English → Kashmiri"], value="Kashmiri → English", label="Direction") source = gr.Textbox(lines=5, label="Input sentence", placeholder="اَتہِ کٲشُر جُملہٕ لِکھِو…") translate_button = gr.Button("Translate", variant="primary") result = gr.Textbox(lines=5, label="Translation", interactive=False) normalized = gr.Textbox(lines=3, label="Normalized Kashmiri input", interactive=False) romanized = gr.Textbox(lines=3, label="Romanized reading", interactive=False) grammar = gr.Textbox(lines=3, label="Grammar review", interactive=False) translate_button.click(translate, inputs=[source, direction], outputs=[result, normalized, romanized, grammar]) gr.Examples( examples=[ ["مےٚ پٔر اَکھ کِتاب", "Kashmiri → English"], ["The weather is good today.", "English → Kashmiri"], ], inputs=[source, direction], ) gr.Markdown( "**Accuracy note:** This is an automatic NLLB-200 model. Kashmiri dialect, spelling and agreement still need fluent-speaker review for important text." ) demo.launch(mcp_server=True) # live-entrypoint-20261003