Download server.py from Sameer0313/Koshur_lughat: direct link, hf CLI and curl.
- Browser
- Download file 10.1 kB
-
https://huggingface.co/Sameer0313/Koshur_lughat/resolve/main/server.py
- Command line
-
hf download hf://Sameer0313/Koshur_lughat/server.py
-
curl -L -o server.py https://huggingface.co/Sameer0313/Koshur_lughat/resolve/main/server.py
10.1 kB
| import re | |
| import unicodedata | |
| import spaces # Must be imported before torch for ZeroGPU. | |
| import torch | |
| import gradio as gr | |
| from transformers import AutoModelForSeq2SeqLM, AutoTokenizer | |
| SRC_KASHMIRI = "kas_Arab" | |
| SRC_ENGLISH = "eng_Latn" | |
| MODEL_REPO = "facebook/nllb-200-distilled-600M" | |
| DEVICE = "cuda" if torch.cuda.is_available() else "cpu" | |
| DTYPE = torch.float16 if DEVICE == "cuda" else torch.float32 | |
| # These are conservative input repairs used by the public dictionary as well. | |
| VARIANTS = str.maketrans({"أ": "ا", "إ": "ا", "ٱ": "ا", "ك": "ک", "ي": "ی", "ى": "ی", "ئ": "ی", "ه": "ہ", "ة": "ہ"}) | |
| def normalize_kashmiri(text: str) -> str: | |
| """Normalize common Arabic/OCR code-point variants without changing words.""" | |
| return unicodedata.normalize("NFC", str(text or "")).replace("\u200b", "").replace("\ufeff", "").translate(VARIANTS) | |
| # Longest sequences must be replaced first. | |
| TRANSLIT = [ | |
| ("پھ", "pʰ"), ("تھ", "tʰ"), ("ژھ", "ʦʰ"), ("چھ", "ʧʰ"), ("ٹھ", "ʈʰ"), ("کھ", "kʰ"), | |
| ("نْ", "ñ"), ("یٖ", "yˌ"), ("وٗ", "w˔"), ("وٚ", "wᵉ"), ("ۄا", "ɔã"), | |
| ("پ", "p"), ("ب", "b"), ("ت", "t"), ("ط", "ṫ"), ("ژ", "ʦ"), ("چ", "ʧ"), | |
| ("د", "d"), ("ج", "ʤ"), ("ٹ", "ʈ"), ("ڈ", "ɖ"), ("ک", "k"), ("ق", "ḱ"), | |
| ("خ", "x"), ("گ", "g"), ("غ", "ġ"), ("ع", "ʔ"), ("ف", "f"), ("س", "s"), | |
| ("ث", "ṡ"), ("ص", "ś"), ("ش", "ʃ"), ("ز", "ż"), ("ذ", "z"), ("ض", "ź"), | |
| ("ظ", "ẑ"), ("ہ", "h"), ("ھ", "ʰ"), ("ح", "ħ"), ("م", "m"), ("ن", "n"), | |
| ("و", "w"), ("ؤ", "u͑"), ("ر", "r"), ("ڑ", "ɽ"), ("ل", "l"), ("ی", "y"), | |
| ("ؠ", "ʲ"), ("ں", "ñ"), ("ِ", "i"), ("ٕ", "ɨ"), ("ٟ", "ɨ̄"), ("ُ", "u"), | |
| ("ے", "ē"), ("ٔ", "ᵊ"), ("ٲ", "ə̄"), ("ۓ", "ɛ͑"), ("ۄ", "ɔ"), ("َ", "a"), | |
| ("ا", "ạ̄"), ("آ", "ã"), ("ْ", "͞"), ("ّ", "˖"), ("؟", "?"), ("۔", "."), | |
| ] | |
| NEGATIVE_KASHMIRI = {"نہ", "نہٕ", "نَہ", "نَہٕ", "نا", "مَہ", "مت"} | |
| QUESTION_KASHMIRI = {"کیا", "کوٚس", "کُس", "کَتھ", "کوت", "کٔتھ", "کِتھ"} | |
| POSTPOSITIONS = {"منٛز", "پؠٹھ", "سٕتۍ", "سٕنٛد", "سٕنٛز", "خٲطرٕ", "تہٕ"} | |
| AUXILIARIES = {"چھ", "چُھ", "چھِ", "چھُو", "آس", "آسِ", "آسُن", "آسٕن", "اتھ", "اوس"} | |
| UNKNOWN_DIGRAPHS = [("tch", "چ"), ("sch", "ش"), ("sh", "ش"), ("ch", "چ"), ("kh", "خ"), ("gh", "غ"), ("ph", "ف"), ("th", "تھ"), ("dh", "دھ"), ("ng", "نگ"), ("qu", "کو"), ("ck", "ک"), ("ee", "ی"), ("oo", "و"), ("ou", "او"), ("ow", "او"), ("ai", "ے"), ("ay", "ے"), ("ea", "ی"), ("ei", "ی"), ("ie", "ی"), ("oa", "و"), ("oi", "و"), ("oy", "و")] | |
| UNKNOWN_LETTERS = { | |
| "a": "ا", "b": "ب", "c": "ک", "d": "د", "e": "ے", "f": "ف", | |
| "g": "گ", "h": "ہ", "i": "ی", "j": "ج", "k": "ک", "l": "ل", | |
| "m": "م", "n": "ن", "o": "و", "p": "پ", "q": "ق", "r": "ر", | |
| "s": "س", "t": "ت", "u": "و", "v": "و", "w": "و", "x": "کس", | |
| "y": "ی", "z": "ز", | |
| } | |
| def transliterate_unknown_english(value: str) -> str: | |
| """Render an English token in Kashmiri script when the model leaves it unknown.""" | |
| def convert(match): | |
| word = match.group(0).lower() | |
| for source, target in UNKNOWN_DIGRAPHS: | |
| word = word.replace(source, target) | |
| return ''.join(UNKNOWN_LETTERS.get(character, character) for character in word) | |
| return re.sub(r"[A-Za-z][A-Za-z'-]*", convert, str(value or "")) | |
| def grammar_review(text: str, direction: str) -> str: | |
| if direction == "Kashmiri → English": | |
| words = re.findall(r"[\u0600-\u06ff\u0750-\u077f\u08a0-\u08ff][\u0600-\u06ff\u0750-\u077f\u08a0-\u08ff\u064b-\u065f\u200c\u200d'’ʼ-]*", text) | |
| question = text.rstrip().endswith(("؟", "?")) or any(word.rstrip("،؛؟۔,.!?") in QUESTION_KASHMIRI for word in words) | |
| negative = any(word.rstrip("،؛؟۔,.!?") in NEGATIVE_KASHMIRI for word in words) | |
| postposition = any(word.rstrip("،؛؟۔,.!?\u064b-\u065f") in POSTPOSITIONS for word in words) | |
| auxiliary = any(word.rstrip("،؛؟۔,.!?\u064b-\u065f") in AUXILIARIES for word in words) | |
| order = "likely verb-final / SOV" if len(words) >= 3 else "short clause" | |
| return "Kashmiri grammar: " + " · ".join([order, "negative polarity" if negative else "affirmative polarity", "question" if question else "statement", "check case/postposition" if postposition else "check case", "check copula/TAM agreement" if auxiliary else "check TAM and verb agreement"]) + "." | |
| words = re.findall(r"[A-Za-z][A-Za-z'-]*", text.lower()) | |
| question = text.rstrip().endswith("?") or bool(re.match(r"\s*(who|what|when|where|why|how|is|are|am|was|were|do|does|did|can|could|will|would|should)\b", text, re.I)) | |
| negative = any(word in {"not", "no", "never", "neither"} or word.endswith("n't") for word in words) | |
| tense = "future" if any(word in {"will", "shall"} for word in words) else "past" if any(word in {"was", "were", "did", "had"} for word in words) else "present/unspecified" | |
| return "English grammar: " + " · ".join(["question" if question else "statement", "negative polarity" if negative else "affirmative polarity", f"{tense} meaning", "review Kashmiri case/agreement"]) + "." | |
| def repair_output(output: str, source: str, direction: str) -> str: | |
| value = normalize_kashmiri(output) if direction == "English → Kashmiri" else str(output or "").strip() | |
| question = source.rstrip().endswith("?") if direction == "English → Kashmiri" else source.rstrip().endswith(("؟", "?")) | |
| if question: | |
| value = re.sub(r"[.!؟?۔]+\s*$", "", value).strip() + ("؟" if direction == "English → Kashmiri" else "?") | |
| return value | |
| def fallback_unknown_words(source: str, output: str) -> str: | |
| words = list(dict.fromkeys(re.findall(r"\b[A-Za-z][A-Za-z'-]{2,}\b", source))) | |
| if len(words) == 1 and (not output or output.casefold() == words[0].casefold()): | |
| return transliterate_unknown_english(words[0]) | |
| for word in words: | |
| native = transliterate_unknown_english(word) | |
| if re.search(rf"\b{re.escape(word)}\b", output, re.I): | |
| output = re.sub(rf"\b{re.escape(word)}\b", native, output, count=1, flags=re.I) | |
| return output | |
| def transliterate_kashmiri(text: str) -> str: | |
| """Return the orthography-based Romanized reading shown beside output.""" | |
| value = normalize_kashmiri(text) | |
| for native, roman in TRANSLIT: | |
| value = value.replace(native, roman) | |
| return re.sub(r"\s+([,;:!?])", r"\1", value).strip() | |
| def load_model(repo: str): | |
| tokenizer = AutoTokenizer.from_pretrained(repo, src_lang=SRC_KASHMIRI) | |
| model = AutoModelForSeq2SeqLM.from_pretrained(repo, torch_dtype=DTYPE).to(DEVICE) | |
| model.eval() | |
| return tokenizer, model | |
| # NLLB-200 is public and supports both kas_Arab and eng_Latn in one model. | |
| tokenizer, model = load_model(MODEL_REPO) | |
| def run_model(text: str, source: str, target: str) -> str: | |
| tokenizer.src_lang = source | |
| inputs = tokenizer([text], truncation=True, padding="longest", return_tensors="pt", return_attention_mask=True).to(DEVICE) | |
| forced_bos = tokenizer.convert_tokens_to_ids(target) | |
| with torch.no_grad(): | |
| generated = model.generate(**inputs, forced_bos_token_id=forced_bos, use_cache=True, max_length=256, num_beams=3, num_return_sequences=1) | |
| return tokenizer.batch_decode(generated, skip_special_tokens=True, clean_up_tokenization_spaces=True)[0].strip() | |
| def translate(text: str, direction: str): | |
| """Translate one sentence with real normalization, grammar review and output repair.""" | |
| if not text or not text.strip(): | |
| return "", "", "", "" | |
| if len(text) > 1000: | |
| raise gr.Error("Please enter no more than 1,000 characters.") | |
| if direction == "Kashmiri → English": | |
| source_text = normalize_kashmiri(text.strip()) | |
| if not re.search(r"[\u0600-\u06ff\u0750-\u077f\u08a0-\u08ff]", source_text): | |
| raise gr.Error("Enter Kashmiri in Perso-Arabic script for this direction.") | |
| output = repair_output(run_model(source_text, SRC_KASHMIRI, SRC_ENGLISH), source_text, direction) | |
| return output, source_text, transliterate_kashmiri(source_text), grammar_review(source_text, direction) | |
| source_text = text.strip() | |
| output = repair_output(run_model(source_text, SRC_ENGLISH, SRC_KASHMIRI), source_text, direction) | |
| output = fallback_unknown_words(source_text, output) | |
| return output, "", transliterate_kashmiri(output), grammar_review(source_text, direction) | |
| with gr.Blocks(title="Koshur Lughat · Server translator") as demo: | |
| gr.Markdown( | |
| "# Koshur Lughat\n" | |
| "## Server-side Kashmiri ↔ English translation\n" | |
| "The model runs on this Hugging Face Space. Visitors send text to the server; they do not download the translation model." | |
| ) | |
| direction = gr.Radio(["Kashmiri → English", "English → Kashmiri"], value="Kashmiri → English", label="Direction") | |
| source = gr.Textbox(lines=5, label="Input sentence", placeholder="اَتہِ کٲشُر جُملہٕ لِکھِو…") | |
| translate_button = gr.Button("Translate", variant="primary") | |
| result = gr.Textbox(lines=5, label="Translation", interactive=False) | |
| normalized = gr.Textbox(lines=3, label="Normalized Kashmiri input", interactive=False) | |
| romanized = gr.Textbox(lines=3, label="Romanized reading", interactive=False) | |
| grammar = gr.Textbox(lines=3, label="Grammar review", interactive=False) | |
| translate_button.click(translate, inputs=[source, direction], outputs=[result, normalized, romanized, grammar]) | |
| gr.Examples( | |
| examples=[ | |
| ["مےٚ پٔر اَکھ کِتاب", "Kashmiri → English"], | |
| ["The weather is good today.", "English → Kashmiri"], | |
| ], | |
| inputs=[source, direction], | |
| ) | |
| gr.Markdown( | |
| "**Accuracy note:** This is an automatic NLLB-200 model. Kashmiri dialect, spelling and agreement still need fluent-speaker review for important text." | |
| ) | |
| demo.launch(mcp_server=True) | |
| # live-entrypoint-20261003 | |