chaungo commited on
Commit
ca77ebd
·
verified ·
1 Parent(s): f19fd71

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +49 -0
app.py ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import spaces
2
+ import torch
3
+ from transformers import AutoModelForCausalLM, AutoTokenizer
4
+ import gradio as gr
5
+
6
+ # Model ID (e.g., Mistral NeMo or Devstral/Codestral variant)
7
+ MODEL_ID = "mistralai/Mistral-Nemo-Instruct-2407"
8
+
9
+ print("Loading tokenizer...")
10
+ tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
11
+
12
+ print("Loading model structure (offloaded to CPU memory initially)...")
13
+ # Note: Loading at module level works because ZeroGPU enables a CUDA emulation mode outside the decorator
14
+ model = AutoModelForCausalLM.from_pretrained(
15
+ MODEL_ID,
16
+ torch_dtype=torch.bfloat16,
17
+ device_map="auto"
18
+ )
19
+
20
+ # Decorate your inference function with @spaces.GPU
21
+ # Use size="xlarge" if you are running a heavier model requiring >48GB VRAM
22
+ @spaces.GPU(duration=120)
23
+ def generate_code(prompt, history):
24
+ system_prompt = "You are an expert agentic coding assistant. Write clean, efficient code and handle tool logic accurately."
25
+
26
+ full_prompt = f"<s>[INST] {system_prompt}\n\nUser request: {prompt} [/INST]"
27
+
28
+ inputs = tokenizer(full_prompt, return_tensors="pt").to("cuda")
29
+
30
+ outputs = model.generate(
31
+ **inputs,
32
+ max_new_tokens=1536,
33
+ temperature=0.1,
34
+ do_sample=True
35
+ )
36
+
37
+ response = tokenizer.decode(outputs[0], skip_special_tokens=True)
38
+ # Clean up prompt echo from response if needed
39
+ return response
40
+
41
+ # Build Gradio UI
42
+ demo = gr.ChatInterface(
43
+ fn=generate_code,
44
+ title="Mistral Coding Agent (ZeroGPU)",
45
+ description="An autonomous coding assistant powered by Mistral and dynamic ZeroGPU allocation."
46
+ )
47
+
48
+ if __name__ == "__main__":
49
+ demo.launch()