SimBombeRino commited on
Commit
7de421d
·
verified ·
1 Parent(s): def3c1a

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +18 -13
app.py CHANGED
@@ -2,44 +2,49 @@ import gradio as gr
2
  from huggingface_hub import hf_hub_download
3
  from llama_cpp import Llama
4
 
 
5
  model_name = "SWAH-KTH/lora_model_2"
6
  model_file = "unsloth.F16.gguf"
7
  model_path = hf_hub_download(model_name, filename=model_file)
8
 
9
  llm = Llama(
10
  model_path=model_path,
11
- # n_ctx=16000, # Context length to use
12
- n_threads=32, # Number of CPU threads to use
13
- # n_gpu_layers=0 # Number of model layers to offload to GPU
14
  )
15
- prompt = "What is gradio"
16
-
17
- generation_kwargs = {
18
- "max_tokens":20000,
19
- "stop":["</s>"],
20
- "echo":False, # Echo the prompt in the output
21
- "top_k":1 # This is essentially greedy decoding, since the model will always return the highest-probability token. Set this value > 1 for sampling decoding
22
- }
23
-
24
 
25
  def respond(message, history, scenario, max_tokens, temperature, top_p):
 
 
 
 
26
  system_message = f"You are acting as a {scenario}. Respond appropriately."
27
  full_prompt = system_message + "\n"
 
 
28
  for user_msg, assistant_msg in history:
29
  full_prompt += f"User: {user_msg}\nAssistant: {assistant_msg}\n"
30
  full_prompt += f"User: {message}\nAssistant: "
 
 
31
  response = ""
32
  try:
33
  res = llm(full_prompt, max_tokens=max_tokens, temperature=temperature, top_p=top_p)
34
  response = res["choices"][0]["text"] if isinstance(res, dict) else res
35
  except Exception as e:
36
  response = f"Error: {str(e)}"
 
 
37
  history.append((message, response))
38
  return history, history
39
 
40
  def clear_chat():
 
 
 
41
  return [], []
42
 
 
43
  with gr.Blocks() as demo:
44
  gr.Markdown("# 🎭 Role-Playing Chatbot")
45
  gr.Markdown("Select a scenario and engage with the chatbot!")
@@ -52,7 +57,7 @@ with gr.Blocks() as demo:
52
  value="Job Interviewer",
53
  )
54
  max_tokens = gr.Slider(
55
- minimum=1, maximum=2048, value=512, step=1, label="Max Response Tokens"
56
  )
57
  temperature = gr.Slider(
58
  minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"
 
2
  from huggingface_hub import hf_hub_download
3
  from llama_cpp import Llama
4
 
5
+ # Model Configuration
6
  model_name = "SWAH-KTH/lora_model_2"
7
  model_file = "unsloth.F16.gguf"
8
  model_path = hf_hub_download(model_name, filename=model_file)
9
 
10
  llm = Llama(
11
  model_path=model_path,
12
+ n_threads=8, # Adjust threads for your CPU
13
+ # n_gpu_layers=0 # Uncomment if using GPU
 
14
  )
 
 
 
 
 
 
 
 
 
15
 
16
  def respond(message, history, scenario, max_tokens, temperature, top_p):
17
+ """
18
+ Respond to the user's message.
19
+ """
20
+ # Build the dynamic system message with the chosen scenario
21
  system_message = f"You are acting as a {scenario}. Respond appropriately."
22
  full_prompt = system_message + "\n"
23
+
24
+ # Include the chat history in the prompt
25
  for user_msg, assistant_msg in history:
26
  full_prompt += f"User: {user_msg}\nAssistant: {assistant_msg}\n"
27
  full_prompt += f"User: {message}\nAssistant: "
28
+
29
+ # Generate the response
30
  response = ""
31
  try:
32
  res = llm(full_prompt, max_tokens=max_tokens, temperature=temperature, top_p=top_p)
33
  response = res["choices"][0]["text"] if isinstance(res, dict) else res
34
  except Exception as e:
35
  response = f"Error: {str(e)}"
36
+
37
+ # Update the history
38
  history.append((message, response))
39
  return history, history
40
 
41
  def clear_chat():
42
+ """
43
+ Clear the conversation history.
44
+ """
45
  return [], []
46
 
47
+ # Gradio Interface
48
  with gr.Blocks() as demo:
49
  gr.Markdown("# 🎭 Role-Playing Chatbot")
50
  gr.Markdown("Select a scenario and engage with the chatbot!")
 
57
  value="Job Interviewer",
58
  )
59
  max_tokens = gr.Slider(
60
+ minimum=1, maximum=512, value=128, step=1, label="Max Response Tokens"
61
  )
62
  temperature = gr.Slider(
63
  minimum=0.1, maximum=4.0, value=0.7, step=0.1, label="Temperature"