Update app.py
Browse files
app.py
CHANGED
|
@@ -2,44 +2,49 @@ import gradio as gr
|
|
| 2 |
from huggingface_hub import hf_hub_download
|
| 3 |
from llama_cpp import Llama
|
| 4 |
|
|
|
|
| 5 |
model_name = "SWAH-KTH/lora_model_2"
|
| 6 |
model_file = "unsloth.F16.gguf"
|
| 7 |
model_path = hf_hub_download(model_name, filename=model_file)
|
| 8 |
|
| 9 |
llm = Llama(
|
| 10 |
model_path=model_path,
|
| 11 |
-
|
| 12 |
-
|
| 13 |
-
# n_gpu_layers=0 # Number of model layers to offload to GPU
|
| 14 |
)
|
| 15 |
-
prompt = "What is gradio"
|
| 16 |
-
|
| 17 |
-
generation_kwargs = {
|
| 18 |
-
"max_tokens":20000,
|
| 19 |
-
"stop":["</s>"],
|
| 20 |
-
"echo":False, # Echo the prompt in the output
|
| 21 |
-
"top_k":1 # This is essentially greedy decoding, since the model will always return the highest-probability token. Set this value > 1 for sampling decoding
|
| 22 |
-
}
|
| 23 |
-
|
| 24 |
|
| 25 |
def respond(message, history, scenario, max_tokens, temperature, top_p):
|
|
|
|
|
|
|
|
|
|
|
|
|
| 26 |
system_message = f"You are acting as a {scenario}. Respond appropriately."
|
| 27 |
full_prompt = system_message + "\n"
|
|
|
|
|
|
|
| 28 |
for user_msg, assistant_msg in history:
|
| 29 |
full_prompt += f"User: {user_msg}\nAssistant: {assistant_msg}\n"
|
| 30 |
full_prompt += f"User: {message}\nAssistant: "
|
|
|
|
|
|
|
| 31 |
response = ""
|
| 32 |
try:
|
| 33 |
res = llm(full_prompt, max_tokens=max_tokens, temperature=temperature, top_p=top_p)
|
| 34 |
response = res["choices"][0]["text"] if isinstance(res, dict) else res
|
| 35 |
except Exception as e:
|
| 36 |
response = f"Error: {str(e)}"
|
|
|
|
|
|
|
| 37 |
history.append((message, response))
|
| 38 |
return history, history
|
| 39 |
|
| 40 |
def clear_chat():
|
|
|
|
|
|
|
|
|
|
| 41 |
return [], []
|
| 42 |
|
|
|
|
| 43 |
with gr.Blocks() as demo:
|
| 44 |
gr.Markdown("# 🎭 Role-Playing Chatbot")
|
| 45 |
gr.Markdown("Select a scenario and engage with the chatbot!")
|
|
@@ -52,7 +57,7 @@ with gr.Blocks() as demo:
|
|
| 52 |
value="Job Interviewer",
|
| 53 |
)
|
| 54 |
max_tokens = gr.Slider(
|
| 55 |
-
minimum=1, maximum=
|
| 56 |
)
|
| 57 |
temperature = gr.Slider(
|
| 58 |
minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"
|
|
|
|
| 2 |
from huggingface_hub import hf_hub_download
|
| 3 |
from llama_cpp import Llama
|
| 4 |
|
| 5 |
+
# Model Configuration
|
| 6 |
model_name = "SWAH-KTH/lora_model_2"
|
| 7 |
model_file = "unsloth.F16.gguf"
|
| 8 |
model_path = hf_hub_download(model_name, filename=model_file)
|
| 9 |
|
| 10 |
llm = Llama(
|
| 11 |
model_path=model_path,
|
| 12 |
+
n_threads=8, # Adjust threads for your CPU
|
| 13 |
+
# n_gpu_layers=0 # Uncomment if using GPU
|
|
|
|
| 14 |
)
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 15 |
|
| 16 |
def respond(message, history, scenario, max_tokens, temperature, top_p):
|
| 17 |
+
"""
|
| 18 |
+
Respond to the user's message.
|
| 19 |
+
"""
|
| 20 |
+
# Build the dynamic system message with the chosen scenario
|
| 21 |
system_message = f"You are acting as a {scenario}. Respond appropriately."
|
| 22 |
full_prompt = system_message + "\n"
|
| 23 |
+
|
| 24 |
+
# Include the chat history in the prompt
|
| 25 |
for user_msg, assistant_msg in history:
|
| 26 |
full_prompt += f"User: {user_msg}\nAssistant: {assistant_msg}\n"
|
| 27 |
full_prompt += f"User: {message}\nAssistant: "
|
| 28 |
+
|
| 29 |
+
# Generate the response
|
| 30 |
response = ""
|
| 31 |
try:
|
| 32 |
res = llm(full_prompt, max_tokens=max_tokens, temperature=temperature, top_p=top_p)
|
| 33 |
response = res["choices"][0]["text"] if isinstance(res, dict) else res
|
| 34 |
except Exception as e:
|
| 35 |
response = f"Error: {str(e)}"
|
| 36 |
+
|
| 37 |
+
# Update the history
|
| 38 |
history.append((message, response))
|
| 39 |
return history, history
|
| 40 |
|
| 41 |
def clear_chat():
|
| 42 |
+
"""
|
| 43 |
+
Clear the conversation history.
|
| 44 |
+
"""
|
| 45 |
return [], []
|
| 46 |
|
| 47 |
+
# Gradio Interface
|
| 48 |
with gr.Blocks() as demo:
|
| 49 |
gr.Markdown("# 🎭 Role-Playing Chatbot")
|
| 50 |
gr.Markdown("Select a scenario and engage with the chatbot!")
|
|
|
|
| 57 |
value="Job Interviewer",
|
| 58 |
)
|
| 59 |
max_tokens = gr.Slider(
|
| 60 |
+
minimum=1, maximum=512, value=128, step=1, label="Max Response Tokens"
|
| 61 |
)
|
| 62 |
temperature = gr.Slider(
|
| 63 |
minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"
|