import spaces import torch from transformers import AutoModelForCausalLM, AutoTokenizer import gradio as gr # Model ID (e.g., Mistral NeMo or Devstral/Codestral variant) MODEL_ID = "mistralai/Mistral-Nemo-Instruct-2407" print("Loading tokenizer...") tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) print("Loading model structure (offloaded to CPU memory initially)...") # Note: Loading at module level works because ZeroGPU enables a CUDA emulation mode outside the decorator model = AutoModelForCausalLM.from_pretrained( MODEL_ID, torch_dtype=torch.bfloat16, device_map="auto" ) # Decorate your inference function with @spaces.GPU # Use size="xlarge" if you are running a heavier model requiring >48GB VRAM @spaces.GPU(duration=120) def generate_code(prompt, history): system_prompt = "You are an expert agentic coding assistant. Write clean, efficient code and handle tool logic accurately." full_prompt = f"[INST] {system_prompt}\n\nUser request: {prompt} [/INST]" inputs = tokenizer(full_prompt, return_tensors="pt").to("cuda") outputs = model.generate( **inputs, max_new_tokens=1536, temperature=0.1, do_sample=True ) response = tokenizer.decode(outputs[0], skip_special_tokens=True) # Clean up prompt echo from response if needed return response # Build Gradio UI demo = gr.ChatInterface( fn=generate_code, title="Mistral Coding Agent (ZeroGPU)", description="An autonomous coding assistant powered by Mistral and dynamic ZeroGPU allocation." ) if __name__ == "__main__": demo.launch()