Download app.py from entropy25/mt_moe: direct link, hf CLI and curl.
- Browser
- Download file 10.8 kB
-
https://huggingface.co/spaces/entropy25/mt_moe/resolve/main/app.py
- Command line
-
hf download hf://spaces/entropy25/mt_moe/app.py
-
curl -L -o app.py https://huggingface.co/spaces/entropy25/mt_moe/resolve/main/app.py
10.8 kB
| import os | |
| import threading | |
| import time | |
| import gradio as gr | |
| import torch | |
| import torch.nn as nn | |
| from huggingface_hub import hf_hub_download | |
| from lingua import Language, LanguageDetectorBuilder | |
| from peft import PeftModel | |
| from transformers import AutoModelForSeq2SeqLM, AutoTokenizer | |
| BASE_MODEL = os.getenv("BASE_MODEL", "facebook/nllb-200-distilled-600M") | |
| ROUTER_REPO = os.getenv("ROUTER_REPO", "entropy25/moe_router") | |
| EXPERT_LANGS = ["en", "de", "fr", "nl"] | |
| EXPERT_REPOS = { | |
| "en": os.getenv("ADAPTER_EN", "entropy25/moe_en"), | |
| "de": os.getenv("ADAPTER_DE", "entropy25/moe_de"), | |
| "fr": os.getenv("ADAPTER_FR", "entropy25/moe_fr"), | |
| "nl": os.getenv("ADAPTER_NL", "entropy25/moe_nl"), | |
| } | |
| LANG_CODES = { | |
| "en": "eng_Latn", | |
| "de": "deu_Latn", | |
| "fr": "fra_Latn", | |
| "nl": "nld_Latn", | |
| "no": "nob_Latn", | |
| } | |
| LANG_LABELS = { | |
| "en": "English", | |
| "de": "German", | |
| "fr": "French", | |
| "nl": "Dutch", | |
| } | |
| LABEL_TO_LANG = {label: lang for lang, label in LANG_LABELS.items()} | |
| AUTO_MODE = "Auto-detect language (recommended)" | |
| ROUTER_MODE = "Learned MoE router (experimental)" | |
| MAX_LENGTH = 256 | |
| NUM_BEAMS = 3 | |
| ROUTER_HIDDEN_DIM = 256 | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| model_dtype = torch.float16 if device == "cuda" else torch.float32 | |
| inference_lock = threading.Lock() | |
| class GatedRouter(nn.Module): | |
| """Router architecture used by the training code.""" | |
| def __init__(self, input_dim, hidden_dim, num_experts): | |
| super().__init__() | |
| self.network = nn.Sequential( | |
| nn.Linear(input_dim, hidden_dim), | |
| nn.ReLU(), | |
| nn.Dropout(0.1), | |
| nn.Linear(hidden_dim, num_experts), | |
| ) | |
| def forward(self, hidden_states): | |
| # This intentionally mirrors the trained router, including its | |
| # unmasked sequence mean. | |
| pooled = hidden_states.mean(dim=1) | |
| pooled = pooled.to(next(self.network.parameters()).dtype) | |
| logits = self.network(pooled) | |
| return torch.softmax(logits, dim=-1), logits | |
| LINGUA_TO_LANG = { | |
| Language.ENGLISH: "en", | |
| Language.GERMAN: "de", | |
| Language.FRENCH: "fr", | |
| Language.DUTCH: "nl", | |
| } | |
| language_detector = LanguageDetectorBuilder.from_languages( | |
| *LINGUA_TO_LANG.keys() | |
| ).with_preloaded_language_models().build() | |
| print("Loading tokenizer and backbone...") | |
| tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL) | |
| backbone = AutoModelForSeq2SeqLM.from_pretrained( | |
| BASE_MODEL, | |
| torch_dtype=model_dtype, | |
| low_cpu_mem_usage=True, | |
| ).to(device) | |
| backbone.eval() | |
| print("Loading expert LoRA adapters...") | |
| first_lang = EXPERT_LANGS[0] | |
| peft_model = PeftModel.from_pretrained( | |
| backbone, | |
| EXPERT_REPOS[first_lang], | |
| adapter_name=first_lang, | |
| ) | |
| for lang in EXPERT_LANGS[1:]: | |
| peft_model.load_adapter(EXPERT_REPOS[lang], adapter_name=lang) | |
| peft_model.eval() | |
| print("Loading router...") | |
| router_path = hf_hub_download(repo_id=ROUTER_REPO, filename="router.pt") | |
| router = GatedRouter( | |
| input_dim=backbone.config.hidden_size, | |
| hidden_dim=ROUTER_HIDDEN_DIM, | |
| num_experts=len(EXPERT_LANGS), | |
| ).to(device) | |
| router.load_state_dict(torch.load(router_path, map_location=device, weights_only=True)) | |
| router.eval() | |
| print(f"All models loaded on {device}.") | |
| EXAMPLES = { | |
| "en": "Mud weight adjusted to 1.82 specific gravity at 3,247 meters depth.", | |
| "de": "Das Schlammgewicht wurde bei 3.247 Metern Tiefe auf ein spezifisches Gewicht von 1,82 angepasst.", | |
| "fr": "Le poids de la boue a été ajusté à une densité de 1,82 à 3 247 mètres de profondeur.", | |
| "nl": "Het slikgewicht werd aangepast naar een soortelijk gewicht van 1,82 op 3.247 meter diepte.", | |
| } | |
| def detect_source_language(text): | |
| confidence_values = language_detector.compute_language_confidence_values(text) | |
| if not confidence_values: | |
| raise gr.Error("The source language could not be detected. Please select it manually.") | |
| best = confidence_values[0] | |
| second = confidence_values[1] | |
| lang = LINGUA_TO_LANG[best.language] | |
| return lang, best.value, best.value - second.value | |
| def tokenize(text, source_lang): | |
| tokenizer.src_lang = LANG_CODES[source_lang] | |
| inputs = tokenizer( | |
| text, | |
| return_tensors="pt", | |
| truncation=True, | |
| max_length=MAX_LENGTH, | |
| ) | |
| return {name: tensor.to(device) for name, tensor in inputs.items()} | |
| def select_with_router(inputs): | |
| # Adapter selection is global PEFT state. Reset it before routing so the | |
| # decision cannot depend on whichever expert handled the previous request. | |
| peft_model.set_adapter(first_lang) | |
| encoder_outputs = backbone.get_encoder()( | |
| input_ids=inputs["input_ids"], | |
| attention_mask=inputs["attention_mask"], | |
| return_dict=True, | |
| ) | |
| weights, logits = router(encoder_outputs.last_hidden_state.float()) | |
| expert_idx = weights.argmax(dim=-1).item() | |
| return EXPERT_LANGS[expert_idx], weights[0], logits[0] | |
| def generate_translation(inputs, expert_lang): | |
| peft_model.set_adapter(expert_lang) | |
| output = peft_model.generate( | |
| **inputs, | |
| forced_bos_token_id=tokenizer.convert_tokens_to_ids(LANG_CODES["no"]), | |
| max_length=MAX_LENGTH, | |
| num_beams=NUM_BEAMS, | |
| early_stopping=True, | |
| ) | |
| return tokenizer.batch_decode(output, skip_special_tokens=True)[0] | |
| def format_router_values(values): | |
| return ", ".join( | |
| f"{LANG_LABELS[lang]}: {values[index].item():.1%}" | |
| for index, lang in enumerate(EXPERT_LANGS) | |
| ) | |
| def route_and_translate(text, source_mode): | |
| text = text.strip() | |
| if not text: | |
| return "", "Enter a sentence to translate." | |
| with inference_lock: | |
| start = time.time() | |
| detection_note = None | |
| router_note = None | |
| if source_mode in (AUTO_MODE, ROUTER_MODE): | |
| source_lang, confidence, margin = detect_source_language(text) | |
| detection_note = ( | |
| f"Detected source: {LANG_LABELS[source_lang]} " | |
| f"({confidence:.1%} confidence)" | |
| ) | |
| if margin < 0.15: | |
| detection_note += " - low confidence; use manual selection if needed" | |
| else: | |
| source_lang = LABEL_TO_LANG[source_mode] | |
| inputs = tokenize(text, source_lang) | |
| if source_mode == ROUTER_MODE: | |
| expert_lang, weights, _ = select_with_router(inputs) | |
| router_note = ( | |
| f"Learned router selected: {LANG_LABELS[expert_lang]} expert\n" | |
| f"Router weights: {format_router_values(weights)}" | |
| ) | |
| else: | |
| expert_lang = source_lang | |
| translation = generate_translation(inputs, expert_lang) | |
| elapsed = time.time() - start | |
| lines = [] | |
| if detection_note: | |
| lines.append(detection_note) | |
| if router_note: | |
| lines.append(router_note) | |
| else: | |
| selection = "Detected" if source_mode == AUTO_MODE else "Manually selected" | |
| lines.append(f"{selection} expert: {LANG_LABELS[expert_lang]}") | |
| lines.append(f"Inference time: {elapsed:.2f}s on {device}") | |
| return translation, "\n".join(lines) | |
| def load_example(lang_label): | |
| return EXAMPLES[LABEL_TO_LANG.get(lang_label, "en")] | |
| custom_css = """ | |
| .gradio-container { | |
| max-width: 1040px !important; | |
| font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif !important; | |
| } | |
| .app-header { padding: 20px 0 4px; text-align: center; } | |
| .app-header h1 { color: #17326d; font-size: 30px; margin-bottom: 6px; } | |
| .app-header p { color: #667085; margin: 0; } | |
| .translate-box { | |
| background: white !important; | |
| border: 1px solid #dce4f2 !important; | |
| border-radius: 8px !important; | |
| min-height: 230px; | |
| } | |
| .text-area textarea { | |
| border: none !important; | |
| font-size: 17px !important; | |
| line-height: 1.65 !important; | |
| padding: 20px !important; | |
| min-height: 210px !important; | |
| } | |
| .translate-btn { min-height: 46px !important; font-weight: 600 !important; } | |
| .status-box textarea { | |
| color: #475467 !important; | |
| font-family: ui-monospace, SFMono-Regular, Menlo, monospace !important; | |
| font-size: 13px !important; | |
| line-height: 1.55 !important; | |
| } | |
| """ | |
| with gr.Blocks(css=custom_css, theme=gr.themes.Default()) as demo: | |
| gr.HTML( | |
| "<div class='app-header'>" | |
| "<h1>Petroleum MoE Translator</h1>" | |
| "<p>English, German, French, or Dutch to Norwegian</p>" | |
| "</div>" | |
| ) | |
| source_mode = gr.Dropdown( | |
| choices=[AUTO_MODE, ROUTER_MODE] + [LANG_LABELS[lang] for lang in EXPERT_LANGS], | |
| value=AUTO_MODE, | |
| label="Selection mode", | |
| info="Automatic mode detects the source language and uses its matching expert.", | |
| ) | |
| with gr.Row(equal_height=True): | |
| with gr.Group(elem_classes="translate-box"): | |
| input_text = gr.Textbox( | |
| label="Source text", | |
| placeholder="Enter petroleum-domain text", | |
| lines=8, | |
| elem_classes="text-area", | |
| ) | |
| with gr.Group(elem_classes="translate-box"): | |
| output_text = gr.Textbox( | |
| label="Norwegian translation", | |
| lines=8, | |
| interactive=False, | |
| elem_classes="text-area", | |
| ) | |
| translate_btn = gr.Button( | |
| "Translate to Norwegian", | |
| variant="primary", | |
| elem_classes="translate-btn", | |
| ) | |
| info_display = gr.Textbox( | |
| label="Routing details", | |
| lines=3, | |
| interactive=False, | |
| elem_classes="status-box", | |
| ) | |
| with gr.Accordion("Example sentences", open=True): | |
| with gr.Row(): | |
| for label in [LANG_LABELS[lang] for lang in EXPERT_LANGS]: | |
| gr.Button(label, size="sm").click( | |
| lambda selected=label: load_example(selected), | |
| outputs=input_text, | |
| ) | |
| with gr.Accordion("About the two automatic modes", open=False): | |
| gr.Markdown( | |
| "**Auto-detect language** is the reliable deployment mode: it detects the " | |
| "source language and activates the corresponding expert.\n\n" | |
| "**Learned MoE router** reproduces the research setting after language " | |
| "detection. It may select a different expert because it was trained to " | |
| "optimise translation, not to serve as a standalone language identifier." | |
| ) | |
| translate_btn.click( | |
| fn=route_and_translate, | |
| inputs=[input_text, source_mode], | |
| outputs=[output_text, info_display], | |
| concurrency_limit=1, | |
| ) | |
| input_text.submit( | |
| fn=route_and_translate, | |
| inputs=[input_text, source_mode], | |
| outputs=[output_text, info_display], | |
| concurrency_limit=1, | |
| ) | |
| demo.queue(default_concurrency_limit=1).launch() | |