"""Gradio app serving google/translategemma-4b-it as a token-protected translate API. Space secrets / variables: HF_TOKEN read token from an account that accepted the TranslateGemma / Gemma terms TRANSLATE_TOKEN long random string; the bot sends it as the `token` argument MODEL_ID optional, defaults to google/translategemma-4b-it """ try: import spaces gpu = spaces.GPU(duration=60) except ImportError: def gpu(fn): return fn import hmac import os import threading import gradio as gr import torch from transformers import AutoModelForImageTextToText, AutoTokenizer MODEL_ID = os.environ.get("MODEL_ID", "google/translategemma-4b-it") API_TOKEN = os.environ.get("TRANSLATE_TOKEN") HF_TOKEN = os.environ.get("HF_TOKEN") MAX_CHARS = 2000 MAX_NEW_TOKENS_CAP = 512 _lock = threading.Lock() # Load on CPU at startup (ZeroGPU reports CUDA as "virtually" available). tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, token=HF_TOKEN) model = AutoModelForImageTextToText.from_pretrained( MODEL_ID, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True, token=HF_TOKEN, ) model.eval() DEVICE = "cuda" if torch.cuda.is_available() else "cpu" model.to(DEVICE) def build_prompt(text, source_lang, target_lang): messages = [{ "role": "user", "content": [{ "type": "text", "source_lang_code": source_lang, "target_lang_code": target_lang, "text": text, }], }] # Raises if a language code isn't in the template's supported map. return tokenizer.apply_chat_template( messages, tokenize=False, add_generation_prompt=True) @gpu def _generate(prompt, max_new): device = "cuda" if torch.cuda.is_available() else "cpu" inputs = tokenizer(prompt, return_tensors="pt", add_special_tokens=False).to(device) with torch.no_grad(): out = model.generate(**inputs, max_new_tokens=max_new, do_sample=False) return tokenizer.decode( out[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True).strip() def _check_token(token): if not API_TOKEN: return if not hmac.compare_digest((token or "").encode(), API_TOKEN.encode()): raise gr.Error("unauthorized") def translate(text, source_lang, target_lang, max_new_tokens, token): _check_token(token) text = (text or "").strip() if not text: raise gr.Error("text is required") source_lang = (source_lang or "").strip() target_lang = (target_lang or "").strip() if not source_lang or not target_lang: raise gr.Error("source_lang and target_lang are required") try: prompt = build_prompt(text[:MAX_CHARS], source_lang, target_lang) except Exception: raise gr.Error(f"unsupported language code: {source_lang} -> {target_lang}") max_new = min(int(max_new_tokens or 256), MAX_NEW_TOKENS_CAP) with _lock: return _generate(prompt, max_new) with gr.Blocks(title="TranslateGemma API") as demo: gr.Markdown("### TranslateGemma\nAPI endpoint: `/translate`. Enter the access token to use the form.") text = gr.Textbox(label="text", lines=4) source_lang = gr.Textbox(label="source_lang (e.g. en, sn, ny)", value="en") target_lang = gr.Textbox(label="target_lang (e.g. en, sn, ny)", value="sn") max_new_tokens = gr.Slider(32, MAX_NEW_TOKENS_CAP, value=256, step=1, label="max_new_tokens") token = gr.Textbox(label="token", type="password") out = gr.Textbox(label="translation", lines=6) gr.Button("Translate").click( translate, [text, source_lang, target_lang, max_new_tokens, token], out, api_name="translate") demo.queue(default_concurrency_limit=1, max_size=20) if __name__ == "__main__": demo.launch(server_name="0.0.0.0", server_port=int(os.environ.get("PORT", 7860)))