Create app.py

app.py

@@ -0,0 +1,79 @@
+import gradio as gr
+import spaces
+import torch
+from transformers import AutoModelForCausalLM, AutoTokenizer
+
+MODEL_ID = "Scalai/scal-lite-60b-code"
+
+print("Loading tokenizer...")
+tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
+
+print("Loading model (this happens once at Space startup)...")
+model = AutoModelForCausalLM.from_pretrained(
+    MODEL_ID,
+    torch_dtype="auto",
+    device_map="auto",
+    trust_remote_code=True,
+)
+model.eval()
+
+
+@spaces.GPU(duration=90)  # max seconds this call is allowed to hold a GPU
+def generate(prompt, system_prompt, max_new_tokens, temperature, top_p):
+    messages = []
+    if system_prompt.strip():
+        messages.append({"role": "system", "content": system_prompt.strip()})
+    messages.append({"role": "user", "content": prompt})
+
+    inputs = tokenizer.apply_chat_template(
+        messages,
+        add_generation_prompt=True,
+        return_tensors="pt",
+    ).to(model.device)
+
+    attention_mask = torch.ones_like(inputs)
+
+    with torch.no_grad():
+        outputs = model.generate(
+            inputs,
+            attention_mask=attention_mask,
+            max_new_tokens=int(max_new_tokens),
+            do_sample=temperature > 0,
+            temperature=max(temperature, 0.01),
+            top_p=top_p,
+        )
+
+    reply = tokenizer.decode(outputs[0][inputs.shape[1]:], skip_special_tokens=True)
+    return reply
+
+
+with gr.Blocks(title="ScaLite-60B-Coder") as demo:
+    gr.Markdown(
+        "# ScaLite-60B-Coder (ZeroGPU)\n"
+        "Bilingual (EN/ES) code + math model on a shared ZeroGPU slice. "
+        "First response after idle time will be slower (cold start)."
+    )
+    with gr.Row():
+        with gr.Column():
+            system_prompt = gr.Textbox(
+                label="System prompt",
+                value="You are a logical, bilingual AI assistant. Think step-by-step.",
+                lines=2,
+            )
+            prompt = gr.Textbox(label="Prompt", placeholder="Write a Python function that...", lines=6)
+            with gr.Row():
+                max_new_tokens = gr.Slider(64, 3500, value=1024, step=64, label="Max new tokens")
+                temperature = gr.Slider(0.0, 1.5, value=0.6, step=0.05, label="Temperature")
+                top_p = gr.Slider(0.1, 1.0, value=0.95, step=0.05, label="Top-p")
+            run_btn = gr.Button("Generate", variant="primary")
+        with gr.Column():
+            output = gr.Textbox(label="Response", lines=20)
+
+    run_btn.click(
+        fn=generate,
+        inputs=[prompt, system_prompt, max_new_tokens, temperature, top_p],
+        outputs=output,
+    )
+
+if __name__ == "__main__":
+    demo.queue().launch()