chaungo commited on
Commit
cd259d6
·
verified ·
1 Parent(s): 6a83d8e

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +24 -17
app.py CHANGED
@@ -1,48 +1,55 @@
1
  import spaces
2
  import torch
3
- from transformers import AutoModelForCausalLM, AutoTokenizer
4
  import gradio as gr
5
 
6
- # Model ID (e.g., Mistral NeMo or Devstral/Codestral variant)
7
- MODEL_ID = "mistralai/Devstral-Small-2-24B-Instruct-2512"
8
 
9
  print("Loading tokenizer...")
10
  tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
11
 
12
- print("Loading model structure (offloaded to CPU memory initially)...")
13
- # Note: Loading at module level works because ZeroGPU enables a CUDA emulation mode outside the decorator
 
 
 
 
 
 
14
  model = AutoModelForCausalLM.from_pretrained(
15
  MODEL_ID,
16
- torch_dtype=torch.bfloat16,
17
  device_map="auto"
18
  )
19
 
20
- # Decorate your inference function with @spaces.GPU
21
- # Use size="xlarge" if you are running a heavier model requiring >48GB VRAM
22
- @spaces.GPU(duration=120)
23
  def generate_code(prompt, history):
24
- system_prompt = "You are an expert agentic coding assistant. Write clean, efficient code and handle tool logic accurately."
25
 
26
- full_prompt = f"<s>[INST] {system_prompt}\n\nUser request: {prompt} [/INST]"
 
27
 
28
  inputs = tokenizer(full_prompt, return_tensors="pt").to("cuda")
29
 
30
  outputs = model.generate(
31
  **inputs,
32
- max_new_tokens=1536,
33
- temperature=0.1,
34
  do_sample=True
35
  )
36
 
37
  response = tokenizer.decode(outputs[0], skip_special_tokens=True)
38
- # Clean up prompt echo from response if needed
 
 
 
39
  return response
40
 
41
- # Build Gradio UI
42
  demo = gr.ChatInterface(
43
  fn=generate_code,
44
- title="Mistral Coding Agent (ZeroGPU)",
45
- description="An autonomous coding assistant powered by Mistral and dynamic ZeroGPU allocation."
46
  )
47
 
48
  if __name__ == "__main__":
 
1
  import spaces
2
  import torch
3
+ from transformers import AutoModelForCausalLM, AutoTokenizer, BitsAndBytesConfig
4
  import gradio as gr
5
 
6
+ # Latest model ID for Mistral Small 4 (or your preferred current variant)
7
+ MODEL_ID = "mistralai/Mistral-Small-4-119B-2603"
8
 
9
  print("Loading tokenizer...")
10
  tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
11
 
12
+ print("Configuring 4-bit quantization to fit MoE weights into ZeroGPU VRAM...")
13
+ bnb_config = BitsAndBytesConfig(
14
+ load_in_4bit=True,
15
+ bnb_4bit_compute_dtype=torch.bfloat16,
16
+ bnb_4bit_quant_type="nf4"
17
+ )
18
+
19
+ print("Loading model...")
20
  model = AutoModelForCausalLM.from_pretrained(
21
  MODEL_ID,
22
+ quantization_config=bnb_config,
23
  device_map="auto"
24
  )
25
 
26
+ @spaces.GPU(size="xlarge", duration=120)
 
 
27
  def generate_code(prompt, history):
28
+ system_message = "You are an elite, modern agentic coding assistant powered by Mistral. Write clean code, handle logic efficiently, and parse tool-calling instructions accurately."
29
 
30
+ # Standard instruction formatting for current Mistral architectures
31
+ full_prompt = f"<s>[INST] {system_message}\n\nUser request: {prompt} [/INST]"
32
 
33
  inputs = tokenizer(full_prompt, return_tensors="pt").to("cuda")
34
 
35
  outputs = model.generate(
36
  **inputs,
37
+ max_new_tokens=2048,
38
+ temperature=0.2,
39
  do_sample=True
40
  )
41
 
42
  response = tokenizer.decode(outputs[0], skip_special_tokens=True)
43
+
44
+ if "[/INST]" in response:
45
+ response = response.split("[/INST]")[-1].strip()
46
+
47
  return response
48
 
 
49
  demo = gr.ChatInterface(
50
  fn=generate_code,
51
+ title="Mistral Small 4 Agentic Coding Space",
52
+ description="Running the latest Mistral Small architecture dynamically on ZeroGPU."
53
  )
54
 
55
  if __name__ == "__main__":