Seymour-Visions commited on
Commit
67b957b
·
verified ·
1 Parent(s): 9a0f531

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +1 -83
app.py CHANGED
@@ -1,85 +1,3 @@
1
  import gradio as gr
2
- from transformers import AutoModelForCausalLM, AutoTokenizer
3
- import torch
4
- import logging
5
- from huggingface_hub import login
6
- import os
7
 
8
- logging.basicConfig(level=logging.DEBUG)
9
- logger = logging.getLogger(__name__)
10
-
11
- # Read the system message from the file
12
- with open("system_message.txt", "r") as file:
13
- system_message = file.read().strip()
14
-
15
- # Get the Hugging Face token from the environment variable
16
- hf_token = os.environ.get("testmix")
17
- if not hf_token:
18
- raise ValueError("testmix secret is not set in the environment variables")
19
-
20
- # Authenticate with Hugging Face
21
- login(token=hf_token)
22
-
23
- model_id = "mistralai/Mixtral-8x7B-Instruct-v0.1"
24
- tokenizer = AutoTokenizer.from_pretrained(model_id, use_auth_token=hf_token)
25
-
26
- # Check if GPU is available
27
- if torch.cuda.is_available():
28
- device_map = "auto"
29
- torch_dtype = torch.float16
30
- else:
31
- device_map = None
32
- torch_dtype = torch.float32
33
-
34
- model = AutoModelForCausalLM.from_pretrained(
35
- model_id,
36
- device_map=device_map,
37
- torch_dtype=torch_dtype,
38
- use_auth_token=hf_token,
39
- low_cpu_mem_usage=True
40
- )
41
-
42
- def respond(message, history):
43
- messages = [{"role": "system", "content": system_message}]
44
- for human, assistant in history:
45
- messages.append({"role": "user", "content": human})
46
- messages.append({"role": "assistant", "content": assistant})
47
- messages.append({"role": "user", "content": message})
48
-
49
- logger.debug(f"Preparing input messages: {messages}")
50
-
51
- inputs = tokenizer.apply_chat_template(messages, return_tensors="pt")
52
- if torch.cuda.is_available():
53
- inputs = inputs.to("cuda")
54
-
55
- logger.debug(f"Tokenized input shape: {inputs.shape}")
56
-
57
- try:
58
- with torch.no_grad():
59
- outputs = model.generate(inputs, max_new_tokens=512, do_sample=True, temperature=0.7, top_p=0.95)
60
- logger.debug(f"Generated output shape: {outputs.shape}")
61
-
62
- response = tokenizer.decode(outputs[0][inputs.shape[1]:], skip_special_tokens=True)
63
- logger.debug(f"Decoded response: {response}")
64
-
65
- return response
66
- except Exception as e:
67
- logger.error(f"Error in model generation: {e}")
68
- return "I'm sorry, but I encountered an error while processing your request."
69
-
70
- demo = gr.ChatInterface(
71
- respond,
72
- chatbot=gr.Chatbot(height=300),
73
- textbox=gr.Textbox(placeholder="Ask me anything", container=False, scale=7),
74
- title="Mixtral-8x7B-Instruct-v0.1 Chat",
75
- description="Chat with the Mixtral-8x7B-Instruct-v0.1 model.",
76
- theme="soft",
77
- examples=["What is machine learning?", "Explain quantum computing in simple terms."],
78
- cache_examples=False,
79
- retry_btn=None,
80
- undo_btn="Delete Previous",
81
- clear_btn="Clear",
82
- )
83
-
84
- if __name__ == "__main__":
85
- demo.launch()
 
1
  import gradio as gr
 
 
 
 
 
2
 
3
+ gr.load("models/mistralai/Mixtral-8x7B-Instruct-v0.1").launch()