Luisgust commited on
Commit
fa2b514
·
verified ·
1 Parent(s): e470bbf

Update main.py

Browse files
Files changed (1) hide show
  1. main.py +10 -9
main.py CHANGED
@@ -34,13 +34,6 @@ def format_prompt(message: str, history: List[Tuple[str, str]]) -> str:
34
  prompt += f"[INST] {message} [/INST]"
35
  return prompt
36
 
37
- async def generate_stream(formatted_prompt, generate_kwargs):
38
- output = ""
39
- stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
40
- for response in stream:
41
- output += response.token.text
42
- yield translator_to_ar.translate(output) # Translate the response back to Arabic
43
-
44
  @app.post("/generate/")
45
  async def generate(
46
  prompt: str = Form(...),
@@ -67,8 +60,16 @@ async def generate(
67
  seed=random.randint(0, 10**7),
68
  )
69
 
70
- # Use StreamingResponse to stream the output
71
- return StreamingResponse(generate_stream(formatted_prompt, generate_kwargs), media_type="text/plain")
 
 
 
 
 
 
 
 
72
 
73
  except Exception as e:
74
  raise HTTPException(status_code=500, detail=str(e))
 
34
  prompt += f"[INST] {message} [/INST]"
35
  return prompt
36
 
 
 
 
 
 
 
 
37
  @app.post("/generate/")
38
  async def generate(
39
  prompt: str = Form(...),
 
60
  seed=random.randint(0, 10**7),
61
  )
62
 
63
+ # Generator function to stream responses
64
+ async def response_generator():
65
+ stream = client.text_generation(formatted_prompt, **generate_kwargs, stream=True, details=True, return_full_text=False)
66
+ output = ""
67
+ for response in stream:
68
+ output += response.token.text
69
+ translated_output = translator_to_ar.translate(output)
70
+ yield translated_output
71
+
72
+ return StreamingResponse(response_generator(), media_type="text/event-stream")
73
 
74
  except Exception as e:
75
  raise HTTPException(status_code=500, detail=str(e))