Spaces:
Running on Zero
Running on Zero
Download app.py from chaungo/devstral-code: direct link, hf CLI and curl.
- Browser
- Download file 1.75 kB
-
https://huggingface.co/spaces/chaungo/devstral-code/resolve/main/app.py
- Command line
-
hf download hf://spaces/chaungo/devstral-code/app.py
-
curl -L -o app.py https://huggingface.co/spaces/chaungo/devstral-code/resolve/main/app.py
1.75 kB
| import spaces | |
| import torch | |
| from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig | |
| import gradio as gr | |
| # Latest model ID for Mistral Small 4 (or your preferred current variant) | |
| MODEL_ID = "mistralai/Devstral-Small-2507" | |
| print("Loading tokenizer...") | |
| tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) | |
| print("Configuring 4-bit quantization to fit MoE weights into ZeroGPU VRAM...") | |
| bnb_config = BitsAndBytesConfig( | |
| load_in_4bit=True, | |
| bnb_4bit_compute_dtype=torch.bfloat16, | |
| bnb_4bit_quant_type="nf4" | |
| ) | |
| print("Loading model...") | |
| model = AutoModelForCausalLM.from_pretrained( | |
| MODEL_ID, | |
| quantization_config=bnb_config, | |
| device_map="auto" | |
| ) | |
| def generate_code(prompt, history): | |
| system_message = "You are an elite, modern agentic coding assistant powered by Mistral. Write clean code, handle logic efficiently, and parse tool-calling instructions accurately." | |
| # Standard instruction formatting for current Mistral architectures | |
| full_prompt = f"<s>[INST] {system_message}\n\nUser request: {prompt} [/INST]" | |
| inputs = tokenizer(full_prompt, return_tensors="pt").to("cuda") | |
| outputs = model.generate( | |
| **inputs, | |
| max_new_tokens=2048, | |
| temperature=0.2, | |
| do_sample=True | |
| ) | |
| response = tokenizer.decode(outputs[0], skip_special_tokens=True) | |
| if "[/INST]" in response: | |
| response = response.split("[/INST]")[-1].strip() | |
| return response | |
| demo = gr.ChatInterface( | |
| fn=generate_code, | |
| title="Mistral Small 4 Agentic Coding Space", | |
| description="Running the latest Mistral Small architecture dynamically on ZeroGPU." | |
| ) | |
| if __name__ == "__main__": | |
| demo.launch() |