mt_moe / app.py
Entropyobserver
Fix language detection and routing workflow
f8e84c2
Raw History Blame Contribute Delete
10.8 kB
import os
import threading
import time
import gradio as gr
import torch
import torch.nn as nn
from huggingface_hub import hf_hub_download
from lingua import Language, LanguageDetectorBuilder
from peft import PeftModel
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
BASE_MODEL = os.getenv("BASE_MODEL", "facebook/nllb-200-distilled-600M")
ROUTER_REPO = os.getenv("ROUTER_REPO", "entropy25/moe_router")
EXPERT_LANGS = ["en", "de", "fr", "nl"]
EXPERT_REPOS = {
"en": os.getenv("ADAPTER_EN", "entropy25/moe_en"),
"de": os.getenv("ADAPTER_DE", "entropy25/moe_de"),
"fr": os.getenv("ADAPTER_FR", "entropy25/moe_fr"),
"nl": os.getenv("ADAPTER_NL", "entropy25/moe_nl"),
}
LANG_CODES = {
"en": "eng_Latn",
"de": "deu_Latn",
"fr": "fra_Latn",
"nl": "nld_Latn",
"no": "nob_Latn",
}
LANG_LABELS = {
"en": "English",
"de": "German",
"fr": "French",
"nl": "Dutch",
}
LABEL_TO_LANG = {label: lang for lang, label in LANG_LABELS.items()}
AUTO_MODE = "Auto-detect language (recommended)"
ROUTER_MODE = "Learned MoE router (experimental)"
MAX_LENGTH = 256
NUM_BEAMS = 3
ROUTER_HIDDEN_DIM = 256
device = "cuda" if torch.cuda.is_available() else "cpu"
model_dtype = torch.float16 if device == "cuda" else torch.float32
inference_lock = threading.Lock()
class GatedRouter(nn.Module):
"""Router architecture used by the training code."""
def __init__(self, input_dim, hidden_dim, num_experts):
super().__init__()
self.network = nn.Sequential(
nn.Linear(input_dim, hidden_dim),
nn.ReLU(),
nn.Dropout(0.1),
nn.Linear(hidden_dim, num_experts),
)
def forward(self, hidden_states):
# This intentionally mirrors the trained router, including its
# unmasked sequence mean.
pooled = hidden_states.mean(dim=1)
pooled = pooled.to(next(self.network.parameters()).dtype)
logits = self.network(pooled)
return torch.softmax(logits, dim=-1), logits
LINGUA_TO_LANG = {
Language.ENGLISH: "en",
Language.GERMAN: "de",
Language.FRENCH: "fr",
Language.DUTCH: "nl",
}
language_detector = LanguageDetectorBuilder.from_languages(
*LINGUA_TO_LANG.keys()
).with_preloaded_language_models().build()
print("Loading tokenizer and backbone...")
tokenizer = AutoTokenizer.from_pretrained(BASE_MODEL)
backbone = AutoModelForSeq2SeqLM.from_pretrained(
BASE_MODEL,
torch_dtype=model_dtype,
low_cpu_mem_usage=True,
).to(device)
backbone.eval()
print("Loading expert LoRA adapters...")
first_lang = EXPERT_LANGS[0]
peft_model = PeftModel.from_pretrained(
backbone,
EXPERT_REPOS[first_lang],
adapter_name=first_lang,
)
for lang in EXPERT_LANGS[1:]:
peft_model.load_adapter(EXPERT_REPOS[lang], adapter_name=lang)
peft_model.eval()
print("Loading router...")
router_path = hf_hub_download(repo_id=ROUTER_REPO, filename="router.pt")
router = GatedRouter(
input_dim=backbone.config.hidden_size,
hidden_dim=ROUTER_HIDDEN_DIM,
num_experts=len(EXPERT_LANGS),
).to(device)
router.load_state_dict(torch.load(router_path, map_location=device, weights_only=True))
router.eval()
print(f"All models loaded on {device}.")
EXAMPLES = {
"en": "Mud weight adjusted to 1.82 specific gravity at 3,247 meters depth.",
"de": "Das Schlammgewicht wurde bei 3.247 Metern Tiefe auf ein spezifisches Gewicht von 1,82 angepasst.",
"fr": "Le poids de la boue a été ajusté à une densité de 1,82 à 3 247 mètres de profondeur.",
"nl": "Het slikgewicht werd aangepast naar een soortelijk gewicht van 1,82 op 3.247 meter diepte.",
}
def detect_source_language(text):
confidence_values = language_detector.compute_language_confidence_values(text)
if not confidence_values:
raise gr.Error("The source language could not be detected. Please select it manually.")
best = confidence_values[0]
second = confidence_values[1]
lang = LINGUA_TO_LANG[best.language]
return lang, best.value, best.value - second.value
def tokenize(text, source_lang):
tokenizer.src_lang = LANG_CODES[source_lang]
inputs = tokenizer(
text,
return_tensors="pt",
truncation=True,
max_length=MAX_LENGTH,
)
return {name: tensor.to(device) for name, tensor in inputs.items()}
@torch.inference_mode()
def select_with_router(inputs):
# Adapter selection is global PEFT state. Reset it before routing so the
# decision cannot depend on whichever expert handled the previous request.
peft_model.set_adapter(first_lang)
encoder_outputs = backbone.get_encoder()(
input_ids=inputs["input_ids"],
attention_mask=inputs["attention_mask"],
return_dict=True,
)
weights, logits = router(encoder_outputs.last_hidden_state.float())
expert_idx = weights.argmax(dim=-1).item()
return EXPERT_LANGS[expert_idx], weights[0], logits[0]
@torch.inference_mode()
def generate_translation(inputs, expert_lang):
peft_model.set_adapter(expert_lang)
output = peft_model.generate(
**inputs,
forced_bos_token_id=tokenizer.convert_tokens_to_ids(LANG_CODES["no"]),
max_length=MAX_LENGTH,
num_beams=NUM_BEAMS,
early_stopping=True,
)
return tokenizer.batch_decode(output, skip_special_tokens=True)[0]
def format_router_values(values):
return ", ".join(
f"{LANG_LABELS[lang]}: {values[index].item():.1%}"
for index, lang in enumerate(EXPERT_LANGS)
)
def route_and_translate(text, source_mode):
text = text.strip()
if not text:
return "", "Enter a sentence to translate."
with inference_lock:
start = time.time()
detection_note = None
router_note = None
if source_mode in (AUTO_MODE, ROUTER_MODE):
source_lang, confidence, margin = detect_source_language(text)
detection_note = (
f"Detected source: {LANG_LABELS[source_lang]} "
f"({confidence:.1%} confidence)"
)
if margin < 0.15:
detection_note += " - low confidence; use manual selection if needed"
else:
source_lang = LABEL_TO_LANG[source_mode]
inputs = tokenize(text, source_lang)
if source_mode == ROUTER_MODE:
expert_lang, weights, _ = select_with_router(inputs)
router_note = (
f"Learned router selected: {LANG_LABELS[expert_lang]} expert\n"
f"Router weights: {format_router_values(weights)}"
)
else:
expert_lang = source_lang
translation = generate_translation(inputs, expert_lang)
elapsed = time.time() - start
lines = []
if detection_note:
lines.append(detection_note)
if router_note:
lines.append(router_note)
else:
selection = "Detected" if source_mode == AUTO_MODE else "Manually selected"
lines.append(f"{selection} expert: {LANG_LABELS[expert_lang]}")
lines.append(f"Inference time: {elapsed:.2f}s on {device}")
return translation, "\n".join(lines)
def load_example(lang_label):
return EXAMPLES[LABEL_TO_LANG.get(lang_label, "en")]
custom_css = """
.gradio-container {
max-width: 1040px !important;
font-family: -apple-system, BlinkMacSystemFont, 'Segoe UI', sans-serif !important;
}
.app-header { padding: 20px 0 4px; text-align: center; }
.app-header h1 { color: #17326d; font-size: 30px; margin-bottom: 6px; }
.app-header p { color: #667085; margin: 0; }
.translate-box {
background: white !important;
border: 1px solid #dce4f2 !important;
border-radius: 8px !important;
min-height: 230px;
}
.text-area textarea {
border: none !important;
font-size: 17px !important;
line-height: 1.65 !important;
padding: 20px !important;
min-height: 210px !important;
}
.translate-btn { min-height: 46px !important; font-weight: 600 !important; }
.status-box textarea {
color: #475467 !important;
font-family: ui-monospace, SFMono-Regular, Menlo, monospace !important;
font-size: 13px !important;
line-height: 1.55 !important;
}
"""
with gr.Blocks(css=custom_css, theme=gr.themes.Default()) as demo:
gr.HTML(
"<div class='app-header'>"
"<h1>Petroleum MoE Translator</h1>"
"<p>English, German, French, or Dutch to Norwegian</p>"
"</div>"
)
source_mode = gr.Dropdown(
choices=[AUTO_MODE, ROUTER_MODE] + [LANG_LABELS[lang] for lang in EXPERT_LANGS],
value=AUTO_MODE,
label="Selection mode",
info="Automatic mode detects the source language and uses its matching expert.",
)
with gr.Row(equal_height=True):
with gr.Group(elem_classes="translate-box"):
input_text = gr.Textbox(
label="Source text",
placeholder="Enter petroleum-domain text",
lines=8,
elem_classes="text-area",
)
with gr.Group(elem_classes="translate-box"):
output_text = gr.Textbox(
label="Norwegian translation",
lines=8,
interactive=False,
elem_classes="text-area",
)
translate_btn = gr.Button(
"Translate to Norwegian",
variant="primary",
elem_classes="translate-btn",
)
info_display = gr.Textbox(
label="Routing details",
lines=3,
interactive=False,
elem_classes="status-box",
)
with gr.Accordion("Example sentences", open=True):
with gr.Row():
for label in [LANG_LABELS[lang] for lang in EXPERT_LANGS]:
gr.Button(label, size="sm").click(
lambda selected=label: load_example(selected),
outputs=input_text,
)
with gr.Accordion("About the two automatic modes", open=False):
gr.Markdown(
"**Auto-detect language** is the reliable deployment mode: it detects the "
"source language and activates the corresponding expert.\n\n"
"**Learned MoE router** reproduces the research setting after language "
"detection. It may select a different expert because it was trained to "
"optimise translation, not to serve as a standalone language identifier."
)
translate_btn.click(
fn=route_and_translate,
inputs=[input_text, source_mode],
outputs=[output_text, info_display],
concurrency_limit=1,
)
input_text.submit(
fn=route_and_translate,
inputs=[input_text, source_mode],
outputs=[output_text, info_display],
concurrency_limit=1,
)
demo.queue(default_concurrency_limit=1).launch()