cazyundee commited on
Commit
fc0acf8
·
verified ·
1 Parent(s): 1d49bf6

Accept JSON body for TinyLlama inference

Browse files
Files changed (1) hide show
  1. app.py +8 -2
app.py CHANGED
@@ -9,6 +9,7 @@ import spaces
9
  import torch
10
  import torchaudio
11
  import gradio as gr
 
12
  from fastapi.responses import JSONResponse
13
  from einops import rearrange
14
  from huggingface_hub import login
@@ -651,11 +652,16 @@ def _patched_create_app(blocks, **kwargs):
651
  except Exception as e: return JSONResponse({"error": str(e)}, status_code=502)
652
 
653
  @fa_app.post("/respite/text/generate")
654
- def _tinyllama(prompt: str = "", max_new_tokens: int = 64, temperature: float = 0.7):
 
655
  if not prompt.strip():
656
  return JSONResponse({"error": "Missing 'prompt'"}, status_code=400)
657
  try:
658
- return JSONResponse(_generate_tinyllama(prompt, max_new_tokens, temperature))
 
 
 
 
659
  except Exception as e:
660
  return JSONResponse({"error": str(e)}, status_code=500)
661
 
 
9
  import torch
10
  import torchaudio
11
  import gradio as gr
12
+ from fastapi import Body
13
  from fastapi.responses import JSONResponse
14
  from einops import rearrange
15
  from huggingface_hub import login
 
652
  except Exception as e: return JSONResponse({"error": str(e)}, status_code=502)
653
 
654
  @fa_app.post("/respite/text/generate")
655
+ def _tinyllama(payload: dict = Body(...)):
656
+ prompt = str(payload.get("prompt", ""))
657
  if not prompt.strip():
658
  return JSONResponse({"error": "Missing 'prompt'"}, status_code=400)
659
  try:
660
+ return JSONResponse(_generate_tinyllama(
661
+ prompt,
662
+ payload.get("max_new_tokens", 64),
663
+ payload.get("temperature", 0.7),
664
+ ))
665
  except Exception as e:
666
  return JSONResponse({"error": str(e)}, status_code=500)
667