NiftyOracleAI commited on
Commit
7bcda6b
·
verified ·
1 Parent(s): 9a9475d

Create app.py

Browse files
Files changed (1) hide show
  1. app.py +34 -0
app.py ADDED
@@ -0,0 +1,34 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import gradio as gr
2
+ import torch
3
+ from transformers import AutoModelForCausalLM, AutoTokenizer
4
+ import spaces
5
+
6
+ # Load model from your repository
7
+ MODEL_ID = "NiftyOracleAI/NiftyOracle-Coder-30B"
8
+
9
+ tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
10
+ model = AutoModelForCausalLM.from_pretrained(
11
+ MODEL_ID,
12
+ torch_dtype=torch.bfloat16,
13
+ device_map="auto"
14
+ )
15
+
16
+ # The magic decorator: claims the GPU only when answering!
17
+ @spaces.GPU
18
+ def generate_response(message, history):
19
+ prompt = f"<|system|>\nYou are CodeSymb, a world-class coding assistant.\n<|user|>\n{message}\n<|assistant|>\n"
20
+ inputs = tokenizer(prompt, return_tensors="pt").to("cuda")
21
+
22
+ with torch.no_grad():
23
+ outputs = model.generate(
24
+ **inputs,
25
+ max_new_tokens=512,
26
+ temperature=0.3,
27
+ do_sample=True
28
+ )
29
+
30
+ response = tokenizer.decode(outputs[0][inputs.input_ids.shape[1]:], skip_special_tokens=True)
31
+ return response
32
+
33
+ # Launch the Gradio Chat UI
34
+ gr.ChatInterface(fn=generate_response, title="CodeSymb-1B Live Demo").launch()