Spaces:
Running on Zero
Running on Zero
Download app.py from chaungo/devstral-code: direct link, hf CLI and curl.
- Browser
- Download file 1.62 kB
-
https://huggingface.co/spaces/chaungo/devstral-code/resolve/ca77ebda83c514b05198206e998bb82e6380355b/app.py
- Command line
-
hf download hf://spaces/chaungo/devstral-code@ca77ebda83c514b05198206e998bb82e6380355b/app.py
-
curl -L -o app.py https://huggingface.co/spaces/chaungo/devstral-code/resolve/ca77ebda83c514b05198206e998bb82e6380355b/app.py
1.62 kB
| import spaces | |
| import torch | |
| from transformers import AutoModelForCausalLM, AutoTokenizer | |
| import gradio as gr | |
| # Model ID (e.g., Mistral NeMo or Devstral/Codestral variant) | |
| MODEL_ID = "mistralai/Mistral-Nemo-Instruct-2407" | |
| print("Loading tokenizer...") | |
| tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) | |
| print("Loading model structure (offloaded to CPU memory initially)...") | |
| # Note: Loading at module level works because ZeroGPU enables a CUDA emulation mode outside the decorator | |
| model = AutoModelForCausalLM.from_pretrained( | |
| MODEL_ID, | |
| torch_dtype=torch.bfloat16, | |
| device_map="auto" | |
| ) | |
| # Decorate your inference function with @spaces.GPU | |
| # Use size="xlarge" if you are running a heavier model requiring >48GB VRAM | |
| def generate_code(prompt, history): | |
| system_prompt = "You are an expert agentic coding assistant. Write clean, efficient code and handle tool logic accurately." | |
| full_prompt = f"<s>[INST] {system_prompt}\n\nUser request: {prompt} [/INST]" | |
| inputs = tokenizer(full_prompt, return_tensors="pt").to("cuda") | |
| outputs = model.generate( | |
| **inputs, | |
| max_new_tokens=1536, | |
| temperature=0.1, | |
| do_sample=True | |
| ) | |
| response = tokenizer.decode(outputs[0], skip_special_tokens=True) | |
| # Clean up prompt echo from response if needed | |
| return response | |
| # Build Gradio UI | |
| demo = gr.ChatInterface( | |
| fn=generate_code, | |
| title="Mistral Coding Agent (ZeroGPU)", | |
| description="An autonomous coding assistant powered by Mistral and dynamic ZeroGPU allocation." | |
| ) | |
| if __name__ == "__main__": | |
| demo.launch() |