Seymour-Visions commited on
Commit
b176f6f
·
verified ·
1 Parent(s): 67b957b

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +37 -1
app.py CHANGED
@@ -1,3 +1,39 @@
1
  import gradio as gr
 
 
2
 
3
- gr.load("models/mistralai/Mixtral-8x7B-Instruct-v0.1").launch()
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  import gradio as gr
2
+ from transformers import AutoModelForCausalLM, AutoTokenizer
3
+ import torch
4
 
5
+ # Load the model and tokenizer
6
+ model_id = "mistralai/Mixtral-8x7B-Instruct-v0.1"
7
+ tokenizer = AutoTokenizer.from_pretrained(model_id)
8
+ model = AutoModelForCausalLM.from_pretrained(model_id, device_map="auto", torch_dtype=torch.float16)
9
+
10
+ # Load system message
11
+ with open("system_message.txt", "r") as f:
12
+ system_message = f.read().strip()
13
+
14
+ def generate_response(message, history):
15
+ # Prepare the input
16
+ messages = [{"role": "system", "content": system_message}]
17
+ for h in history:
18
+ messages.append({"role": "user", "content": h[0]})
19
+ messages.append({"role": "assistant", "content": h[1]})
20
+ messages.append({"role": "user", "content": message})
21
+
22
+ # Tokenize the input
23
+ inputs = tokenizer.apply_chat_template(messages, return_tensors="pt").to(model.device)
24
+
25
+ # Generate the response
26
+ outputs = model.generate(inputs, max_new_tokens=512, do_sample=True, temperature=0.7)
27
+ response = tokenizer.decode(outputs[0][inputs.shape[1]:], skip_special_tokens=True)
28
+
29
+ return response
30
+
31
+ # Create the Gradio interface
32
+ iface = gr.ChatInterface(
33
+ generate_response,
34
+ title="Mixtral-8x7B-Instruct Chat",
35
+ description="Chat with the Mixtral-8x7B-Instruct model using custom instructions."
36
+ )
37
+
38
+ # Launch the app
39
+ iface.launch()