import spaces import torch from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig import gradio as gr # Latest model ID for Mistral Small 4 (or your preferred current variant) MODEL_ID = "mistralai/Devstral-Small-2507" print("Loading tokenizer...") tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) print("Configuring 4-bit quantization to fit MoE weights into ZeroGPU VRAM...") bnb_config = BitsAndBytesConfig( load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16, bnb_4bit_quant_type="nf4" ) print("Loading model...") model = AutoModelForCausalLM.from_pretrained( MODEL_ID, quantization_config=bnb_config, device_map="auto" ) @spaces.GPU(size="xlarge", duration=120) def generate_code(prompt, history): system_message = "You are an elite, modern agentic coding assistant powered by Mistral. Write clean code, handle logic efficiently, and parse tool-calling instructions accurately." # Standard instruction formatting for current Mistral architectures full_prompt = f"[INST] {system_message}\n\nUser request: {prompt} [/INST]" inputs = tokenizer(full_prompt, return_tensors="pt").to("cuda") outputs = model.generate( **inputs, max_new_tokens=2048, temperature=0.2, do_sample=True ) response = tokenizer.decode(outputs[0], skip_special_tokens=True) if "[/INST]" in response: response = response.split("[/INST]")[-1].strip() return response demo = gr.ChatInterface( fn=generate_code, title="Mistral Small 4 Agentic Coding Space", description="Running the latest Mistral Small architecture dynamically on ZeroGPU." ) if __name__ == "__main__": demo.launch()