Ashrafb commited on
Commit
668fc97
·
1 Parent(s): 1e65544

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +40 -0
app.py CHANGED
@@ -1,6 +1,7 @@
1
  import streamlit as st
2
  import requests
3
  import base64
 
4
  import os
5
  from text_generation import Client
6
 
@@ -16,6 +17,45 @@ client = Client(
16
  EOS_STRING = "</s>"
17
  EOT_STRING = "<EOT>"
18
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
19
  def generate_image_caption(image_data):
20
  image_base64 = base64.b64encode(image_data).decode('utf-8')
21
  payload = {"data": ["data:image/jpeg;base64," + image_base64]}
 
1
  import streamlit as st
2
  import requests
3
  import base64
4
+ from typing import Iterator
5
  import os
6
  from text_generation import Client
7
 
 
17
  EOS_STRING = "</s>"
18
  EOT_STRING = "<EOT>"
19
 
20
+ def get_prompt(message: str, chat_history: list[tuple[str, str]],
21
+ system_prompt: str) -> str:
22
+ texts = [f'<s>[INST] <<SYS>>\n{system_prompt}\n<</SYS>>\n\n']
23
+ do_strip = False
24
+ for user_input, response in chat_history:
25
+ user_input = user_input.strip() if do_strip else user_input
26
+ do_strip = True
27
+ texts.append(f'{user_input} [/INST] {response.strip()} </s><s>[INST] ')
28
+ message = message.strip() if do_strip else message
29
+ texts.append(f'{message} [/INST]')
30
+ return ''.join(texts)
31
+
32
+
33
+ def run(message: str,
34
+ chat_history: list[tuple[str, str]],
35
+ system_prompt: str,
36
+ max_new_tokens: int = 1024,
37
+ temperature: float = 0.1,
38
+ top_p: float = 0.9,
39
+ top_k: int = 50) -> Iterator[str]:
40
+ prompt = get_prompt(message, chat_history, system_prompt)
41
+
42
+ generate_kwargs = dict(
43
+ max_new_tokens=max_new_tokens,
44
+ do_sample=True,
45
+ top_p=top_p,
46
+ top_k=top_k,
47
+ temperature=temperature,
48
+ )
49
+ stream = client.generate_stream(prompt, **generate_kwargs)
50
+ output = ""
51
+ for response in stream:
52
+ if any([end_token in response.token.text for end_token in [EOS_STRING, EOT_STRING]]):
53
+ yield output
54
+ output = ""
55
+ else:
56
+ output += response.token.text
57
+
58
+
59
  def generate_image_caption(image_data):
60
  image_base64 = base64.b64encode(image_data).decode('utf-8')
61
  payload = {"data": ["data:image/jpeg;base64," + image_base64]}