ItamarMordo commited on
Commit
f53c7ed
·
verified ·
1 Parent(s): d28203f

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +19 -6
app.py CHANGED
@@ -1,30 +1,43 @@
 
1
  import gradio as gr
2
  from transformers import AutoTokenizer, AutoModelForCausalLM
3
  import torch
4
 
5
  # ---- CONFIG ----
6
- MODEL_NAME = "nomic-ai/phi-2-3b" # Hugging Face model
 
 
7
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
8
 
9
  # ---- LOAD MODEL ----
10
- tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
11
- model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, device_map="auto")
12
  model.eval()
13
 
14
  # ---- CHAT FUNCTION ----
15
  def chat_fn(user_input, history=[]):
16
- # Build prompt with conversation history
17
  prompt = "\n".join([f"You: {u}\nAI: {a}" for u, a in history]) + f"\nYou: {user_input}\nAI:"
18
  inputs = tokenizer(prompt, return_tensors="pt").to(DEVICE)
19
- output_ids = model.generate(**inputs, max_new_tokens=200, do_sample=True, temperature=0.7, top_p=0.9)
 
 
 
 
 
 
 
 
20
  response = tokenizer.decode(output_ids[0][inputs['input_ids'].shape[1]:], skip_special_tokens=True)
 
 
21
  history.append((user_input, response))
22
  return history, history
23
 
24
  # ---- GRADIO INTERFACE ----
25
  with gr.Blocks() as demo:
26
  chatbot = gr.Chatbot()
27
- msg = gr.Textbox()
28
  state = gr.State([])
29
  msg.submit(chat_fn, [msg, state], [chatbot, state])
30
 
 
1
+ import os
2
  import gradio as gr
3
  from transformers import AutoTokenizer, AutoModelForCausalLM
4
  import torch
5
 
6
  # ---- CONFIG ----
7
+ MODEL_NAME = "nomic-ai/phi-2-3b" # Your private Phi-2 model
8
+ HF_TOKEN = os.environ.get("HF_TOKEN") # Secret stored in Space
9
+
10
  DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
11
 
12
  # ---- LOAD MODEL ----
13
+ tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME, use_auth_token=HF_TOKEN)
14
+ model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, use_auth_token=HF_TOKEN, device_map="auto")
15
  model.eval()
16
 
17
  # ---- CHAT FUNCTION ----
18
  def chat_fn(user_input, history=[]):
19
+ # Build prompt including conversation history
20
  prompt = "\n".join([f"You: {u}\nAI: {a}" for u, a in history]) + f"\nYou: {user_input}\nAI:"
21
  inputs = tokenizer(prompt, return_tensors="pt").to(DEVICE)
22
+
23
+ # Generate response
24
+ output_ids = model.generate(
25
+ **inputs,
26
+ max_new_tokens=200,
27
+ do_sample=True,
28
+ temperature=0.7,
29
+ top_p=0.9
30
+ )
31
  response = tokenizer.decode(output_ids[0][inputs['input_ids'].shape[1]:], skip_special_tokens=True)
32
+
33
+ # Update history
34
  history.append((user_input, response))
35
  return history, history
36
 
37
  # ---- GRADIO INTERFACE ----
38
  with gr.Blocks() as demo:
39
  chatbot = gr.Chatbot()
40
+ msg = gr.Textbox(placeholder="Type your message here...")
41
  state = gr.State([])
42
  msg.submit(chat_fn, [msg, state], [chatbot, state])
43