TranslateGemma / app.py
iamfae's picture
Create app.py
3d36979 verified
Raw History Blame Contribute Delete
3.84 kB
"""Gradio app serving google/translategemma-4b-it as a token-protected translate API.
Space secrets / variables:
HF_TOKEN read token from an account that accepted the TranslateGemma / Gemma terms
TRANSLATE_TOKEN long random string; the bot sends it as the `token` argument
MODEL_ID optional, defaults to google/translategemma-4b-it
"""
try:
import spaces
gpu = spaces.GPU(duration=60)
except ImportError:
def gpu(fn):
return fn
import hmac
import os
import threading
import gradio as gr
import torch
from transformers import AutoModelForImageTextToText, AutoTokenizer
MODEL_ID = os.environ.get("MODEL_ID", "google/translategemma-4b-it")
API_TOKEN = os.environ.get("TRANSLATE_TOKEN")
HF_TOKEN = os.environ.get("HF_TOKEN")
MAX_CHARS = 2000
MAX_NEW_TOKENS_CAP = 512
_lock = threading.Lock()
# Load on CPU at startup (ZeroGPU reports CUDA as "virtually" available).
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, token=HF_TOKEN)
model = AutoModelForImageTextToText.from_pretrained(
MODEL_ID,
torch_dtype=torch.bfloat16,
low_cpu_mem_usage=True,
token=HF_TOKEN,
)
model.eval()
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
model.to(DEVICE)
def build_prompt(text, source_lang, target_lang):
messages = [{
"role": "user",
"content": [{
"type": "text",
"source_lang_code": source_lang,
"target_lang_code": target_lang,
"text": text,
}],
}]
# Raises if a language code isn't in the template's supported map.
return tokenizer.apply_chat_template(
messages, tokenize=False, add_generation_prompt=True)
@gpu
def _generate(prompt, max_new):
device = "cuda" if torch.cuda.is_available() else "cpu"
inputs = tokenizer(prompt, return_tensors="pt", add_special_tokens=False).to(device)
with torch.no_grad():
out = model.generate(**inputs, max_new_tokens=max_new, do_sample=False)
return tokenizer.decode(
out[0][inputs["input_ids"].shape[1]:], skip_special_tokens=True).strip()
def _check_token(token):
if not API_TOKEN:
return
if not hmac.compare_digest((token or "").encode(), API_TOKEN.encode()):
raise gr.Error("unauthorized")
def translate(text, source_lang, target_lang, max_new_tokens, token):
_check_token(token)
text = (text or "").strip()
if not text:
raise gr.Error("text is required")
source_lang = (source_lang or "").strip()
target_lang = (target_lang or "").strip()
if not source_lang or not target_lang:
raise gr.Error("source_lang and target_lang are required")
try:
prompt = build_prompt(text[:MAX_CHARS], source_lang, target_lang)
except Exception:
raise gr.Error(f"unsupported language code: {source_lang} -> {target_lang}")
max_new = min(int(max_new_tokens or 256), MAX_NEW_TOKENS_CAP)
with _lock:
return _generate(prompt, max_new)
with gr.Blocks(title="TranslateGemma API") as demo:
gr.Markdown("### TranslateGemma\nAPI endpoint: `/translate`. Enter the access token to use the form.")
text = gr.Textbox(label="text", lines=4)
source_lang = gr.Textbox(label="source_lang (e.g. en, sn, ny)", value="en")
target_lang = gr.Textbox(label="target_lang (e.g. en, sn, ny)", value="sn")
max_new_tokens = gr.Slider(32, MAX_NEW_TOKENS_CAP, value=256, step=1, label="max_new_tokens")
token = gr.Textbox(label="token", type="password")
out = gr.Textbox(label="translation", lines=6)
gr.Button("Translate").click(
translate, [text, source_lang, target_lang, max_new_tokens, token], out,
api_name="translate")
demo.queue(default_concurrency_limit=1, max_size=20)
if __name__ == "__main__":
demo.launch(server_name="0.0.0.0", server_port=int(os.environ.get("PORT", 7860)))