import os import threading import time import gradio as gr import torch import torch.nn as nn from huggingface_hub import hf_hub_download from lingua import Language, LanguageDetectorBuilder from peft import PeftModel from transformers import AutoModelForSeq2SeqLM, AutoTokenizer BASE_MODEL = os.getenv("BASE_MODEL", "facebook/nllb-200-distilled-600M") ROUTER_REPO = os.getenv("ROUTER_REPO", "entropy25/moe_router") EXPERT_LANGS = ["en", "de", "fr", "nl"] EXPERT_REPOS = { "en": os.getenv("ADAPTER_EN", "entropy25/moe_en"), "de": os.getenv("ADAPTER_DE", "entropy25/moe_de"), "fr": os.getenv("ADAPTER_FR", "entropy25/moe_fr"), "nl": os.getenv("ADAPTER_NL", "entropy25/moe_nl"), } LANG_CODES = { "en": "eng_Latn", "de": "deu_Latn", "fr": "fra_Latn", "nl": "nld_Latn", "no": "nob_Latn", } LANG_LABELS = { "en": "English", "de": "German", "fr": "French", "nl": "Dutch", } LABEL_TO_LANG = {label: lang for lang, label in LANG_LABELS.items()} AUTO_MODE = "Auto-detect language (recommended)" ROUTER_MODE = "Learned MoE router (experimental)" MAX_LENGTH = 256 NUM_BEAMS = 3 ROUTER_HIDDEN_DIM = 256 device = "cuda" if torch.cuda.is_available() else "cpu" model_dtype = torch.float16 if device == "cuda" else torch.float32 inference_lock = threading.Lock() class GatedRouter(nn.Module): """Router architecture used by the training code.""" def __init__(self, input_dim, hidden_dim, num_experts): super().__init__() self.network = nn.Sequential( nn.Linear(input_dim, hidden_dim), nn.ReLU(), nn.Dropout(0.1), nn.Linear(hidden_dim, num_experts), ) def forward(self, hidden_states): # This intentionally mirrors the trained router, including its # unmasked sequence mean. pooled = hidden_states.mean(dim=1) pooled = pooled.to(next(self.network.parameters()).dtype) logits = self.network(pooled) return torch.softmax(logits, dim=-1), logits LINGUA_TO_LANG = { Language.ENGLISH: "en", Language.GERMAN: "de", Language.FRENCH: "fr", Language.DUTCH: "nl", } language_detector = LanguageDetectorBuilder.from_languages( *LINGUA_TO_LANG.keys() ).with_preloaded_language_models().build() print("Loading tokenizer and backbone...") tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL) backbone = AutoModelForSeq2SeqLM.from_pretrained( BASE_MODEL, torch_dtype=model_dtype, low_cpu_mem_usage=True, ).to(device) backbone.eval() print("Loading expert LoRA adapters...") first_lang = EXPERT_LANGS[0] peft_model = PeftModel.from_pretrained( backbone, EXPERT_REPOS[first_lang], adapter_name=first_lang, ) for lang in EXPERT_LANGS[1:]: peft_model.load_adapter(EXPERT_REPOS[lang], adapter_name=lang) peft_model.eval() print("Loading router...") router_path = hf_hub_download(repo_id=ROUTER_REPO, filename="router.pt") router = GatedRouter( input_dim=backbone.config.hidden_size, hidden_dim=ROUTER_HIDDEN_DIM, num_experts=len(EXPERT_LANGS), ).to(device) router.load_state_dict(torch.load(router_path, map_location=device, weights_only=True)) router.eval() print(f"All models loaded on {device}.") EXAMPLES = { "en": "Mud weight adjusted to 1.82 specific gravity at 3,247 meters depth.", "de": "Das Schlammgewicht wurde bei 3.247 Metern Tiefe auf ein spezifisches Gewicht von 1,82 angepasst.", "fr": "Le poids de la boue a été ajusté à une densité de 1,82 à 3 247 mètres de profondeur.", "nl": "Het slikgewicht werd aangepast naar een soortelijk gewicht van 1,82 op 3.247 meter diepte.", } def detect_source_language(text): confidence_values = language_detector.compute_language_confidence_values(text) if not confidence_values: raise gr.Error("The source language could not be detected. Please select it manually.") best = confidence_values[0] second = confidence_values[1] lang = LINGUA_TO_LANG[best.language] return lang, best.value, best.value - second.value def tokenize(text, source_lang): tokenizer.src_lang = LANG_CODES[source_lang] inputs = tokenizer( text, return_tensors="pt", truncation=True, max_length=MAX_LENGTH, ) return {name: tensor.to(device) for name, tensor in inputs.items()} @torch.inference_mode() def select_with_router(inputs): # Adapter selection is global PEFT state. Reset it before routing so the # decision cannot depend on whichever expert handled the previous request. peft_model.set_adapter(first_lang) encoder_outputs = backbone.get_encoder()( input_ids=inputs["input_ids"], attention_mask=inputs["attention_mask"], return_dict=True, ) weights, logits = router(encoder_outputs.last_hidden_state.float()) expert_idx = weights.argmax(dim=-1).item() return EXPERT_LANGS[expert_idx], weights[0], logits[0] @torch.inference_mode() def generate_translation(inputs, expert_lang): peft_model.set_adapter(expert_lang) output = peft_model.generate( **inputs, forced_bos_token_id=tokenizer.convert_tokens_to_ids(LANG_CODES["no"]), max_length=MAX_LENGTH, num_beams=NUM_BEAMS, early_stopping=True, ) return tokenizer.batch_decode(output, skip_special_tokens=True)[0] def format_router_values(values): return ", ".join( f"{LANG_LABELS[lang]}: {values[index].item():.1%}" for index, lang in enumerate(EXPERT_LANGS) ) def route_and_translate(text, source_mode): text = text.strip() if not text: return "", "Enter a sentence to translate." with inference_lock: start = time.time() detection_note = None router_note = None if source_mode in (AUTO_MODE, ROUTER_MODE): source_lang, confidence, margin = detect_source_language(text) detection_note = ( f"Detected source: {LANG_LABELS[source_lang]} " f"({confidence:.1%} confidence)" ) if margin < 0.15: detection_note += " - low confidence; use manual selection if needed" else: source_lang = LABEL_TO_LANG[source_mode] inputs = tokenize(text, source_lang) if source_mode == ROUTER_MODE: expert_lang, weights, _ = select_with_router(inputs) router_note = ( f"Learned router selected: {LANG_LABELS[expert_lang]} expert\n" f"Router weights: {format_router_values(weights)}" ) else: expert_lang = source_lang translation = generate_translation(inputs, expert_lang) elapsed = time.time() - start lines = [] if detection_note: lines.append(detection_note) if router_note: lines.append(router_note) else: selection = "Detected" if source_mode == AUTO_MODE else "Manually selected" lines.append(f"{selection} expert: {LANG_LABELS[expert_lang]}") lines.append(f"Inference time: {elapsed:.2f}s on {device}") return translation, "\n".join(lines) def load_example(lang_label): return EXAMPLES[LABEL_TO_LANG.get(lang_label, "en")] custom_css = """ .gradio-container { max-width: 1040px !important; font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif !important; } .app-header { padding: 20px 0 4px; text-align: center; } .app-header h1 { color: #17326d; font-size: 30px; margin-bottom: 6px; } .app-header p { color: #667085; margin: 0; } .translate-box { background: white !important; border: 1px solid #dce4f2 !important; border-radius: 8px !important; min-height: 230px; } .text-area textarea { border: none !important; font-size: 17px !important; line-height: 1.65 !important; padding: 20px !important; min-height: 210px !important; } .translate-btn { min-height: 46px !important; font-weight: 600 !important; } .status-box textarea { color: #475467 !important; font-family: ui-monospace, SFMono-Regular, Menlo, monospace !important; font-size: 13px !important; line-height: 1.55 !important; } """ with gr.Blocks(css=custom_css, theme=gr.themes.Default()) as demo: gr.HTML( "
English, German, French, or Dutch to Norwegian
" "