Spaces:
Running on Zero
Running on Zero
Accept JSON body for TinyLlama inference
Browse files
app.py
CHANGED
|
@@ -9,6 +9,7 @@ import spaces
|
|
| 9 |
import torch
|
| 10 |
import torchaudio
|
| 11 |
import gradio as gr
|
|
|
|
| 12 |
from fastapi.responses import JSONResponse
|
| 13 |
from einops import rearrange
|
| 14 |
from huggingface_hub import login
|
|
@@ -651,11 +652,16 @@ def _patched_create_app(blocks, **kwargs):
|
|
| 651 |
except Exception as e: return JSONResponse({"error": str(e)}, status_code=502)
|
| 652 |
|
| 653 |
@fa_app.post("/respite/text/generate")
|
| 654 |
-
def _tinyllama(
|
|
|
|
| 655 |
if not prompt.strip():
|
| 656 |
return JSONResponse({"error": "Missing 'prompt'"}, status_code=400)
|
| 657 |
try:
|
| 658 |
-
return JSONResponse(_generate_tinyllama(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 659 |
except Exception as e:
|
| 660 |
return JSONResponse({"error": str(e)}, status_code=500)
|
| 661 |
|
|
|
|
| 9 |
import torch
|
| 10 |
import torchaudio
|
| 11 |
import gradio as gr
|
| 12 |
+
from fastapi import Body
|
| 13 |
from fastapi.responses import JSONResponse
|
| 14 |
from einops import rearrange
|
| 15 |
from huggingface_hub import login
|
|
|
|
| 652 |
except Exception as e: return JSONResponse({"error": str(e)}, status_code=502)
|
| 653 |
|
| 654 |
@fa_app.post("/respite/text/generate")
|
| 655 |
+
def _tinyllama(payload: dict = Body(...)):
|
| 656 |
+
prompt = str(payload.get("prompt", ""))
|
| 657 |
if not prompt.strip():
|
| 658 |
return JSONResponse({"error": "Missing 'prompt'"}, status_code=400)
|
| 659 |
try:
|
| 660 |
+
return JSONResponse(_generate_tinyllama(
|
| 661 |
+
prompt,
|
| 662 |
+
payload.get("max_new_tokens", 64),
|
| 663 |
+
payload.get("temperature", 0.7),
|
| 664 |
+
))
|
| 665 |
except Exception as e:
|
| 666 |
return JSONResponse({"error": str(e)}, status_code=500)
|
| 667 |
|