Nafiul Huda commited on
Update app.py
Browse files
app.py
CHANGED
|
@@ -2,7 +2,7 @@ import os
|
|
| 2 |
import json
|
| 3 |
import gradio as gr
|
| 4 |
import spaces
|
| 5 |
-
from fastapi import Request
|
| 6 |
from fastapi.responses import JSONResponse, StreamingResponse
|
| 7 |
from huggingface_hub import hf_hub_download
|
| 8 |
from llama_cpp import Llama
|
|
@@ -71,17 +71,26 @@ with gr.Blocks() as demo:
|
|
| 71 |
examples=["Who are you?", "Write a python script to reverse a string.", "Explain quantum computing."],
|
| 72 |
)
|
| 73 |
|
| 74 |
-
# 5. INJECT FASTAPI ROUTES DIRECTLY INTO GRADIO
|
| 75 |
-
# This safely bypasses the ZeroGPU port collision by attaching our API
|
| 76 |
-
# directly to the Gradio app before the server boots up.
|
| 77 |
import gradio.routes
|
| 78 |
-
original_create_app = gradio.routes
|
| 79 |
|
| 80 |
def custom_create_app(*args, **kwargs):
|
| 81 |
-
# Let Gradio build its native FastAPI app
|
| 82 |
app = original_create_app(*args, **kwargs)
|
| 83 |
|
| 84 |
-
#
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 85 |
@app.post("/v1/chat/completions")
|
| 86 |
async def chat_api(request: Request):
|
| 87 |
try:
|
|
@@ -91,6 +100,12 @@ def custom_create_app(*args, **kwargs):
|
|
| 91 |
temperature = body.get("temperature", 0.7)
|
| 92 |
max_tokens = body.get("max_tokens", 1024)
|
| 93 |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 94 |
if stream:
|
| 95 |
def stream_generator():
|
| 96 |
for chunk in llm.create_chat_completion(
|
|
@@ -101,7 +116,11 @@ def custom_create_app(*args, **kwargs):
|
|
| 101 |
):
|
| 102 |
yield f"data: {json.dumps(chunk)}\n\n"
|
| 103 |
yield "data: [DONE]\n\n"
|
| 104 |
-
return StreamingResponse(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 105 |
else:
|
| 106 |
response = llm.create_chat_completion(
|
| 107 |
messages=messages,
|
|
@@ -109,16 +128,23 @@ def custom_create_app(*args, **kwargs):
|
|
| 109 |
temperature=temperature,
|
| 110 |
max_tokens=max_tokens
|
| 111 |
)
|
| 112 |
-
return JSONResponse(content=response)
|
| 113 |
except Exception as e:
|
| 114 |
-
return JSONResponse(
|
|
|
|
|
|
|
|
|
|
|
|
|
| 115 |
|
| 116 |
return app
|
| 117 |
|
| 118 |
-
# Apply the patch
|
| 119 |
-
gradio.routes.
|
|
|
|
|
|
|
|
|
|
|
|
|
| 120 |
|
| 121 |
# 6. LAUNCH GRADIO
|
| 122 |
-
# ZeroGPU will safely intercept this and route our API & UI perfectly!
|
| 123 |
if __name__ == "__main__":
|
| 124 |
demo.launch(server_name="0.0.0.0", server_port=7860)
|
|
|
|
| 2 |
import json
|
| 3 |
import gradio as gr
|
| 4 |
import spaces
|
| 5 |
+
from fastapi import Request, Response
|
| 6 |
from fastapi.responses import JSONResponse, StreamingResponse
|
| 7 |
from huggingface_hub import hf_hub_download
|
| 8 |
from llama_cpp import Llama
|
|
|
|
| 71 |
examples=["Who are you?", "Write a python script to reverse a string.", "Explain quantum computing."],
|
| 72 |
)
|
| 73 |
|
| 74 |
+
# 5. INJECT FASTAPI ROUTES DIRECTLY INTO GRADIO WITH CORS SUPPORT
|
|
|
|
|
|
|
| 75 |
import gradio.routes
|
| 76 |
+
original_create_app = getattr(gradio.routes, "create_app", None)
|
| 77 |
|
| 78 |
def custom_create_app(*args, **kwargs):
|
|
|
|
| 79 |
app = original_create_app(*args, **kwargs)
|
| 80 |
|
| 81 |
+
# CORS Preflight (OPTIONS) handler
|
| 82 |
+
@app.options("/v1/chat/completions")
|
| 83 |
+
async def chat_api_options():
|
| 84 |
+
return Response(
|
| 85 |
+
status_code=200,
|
| 86 |
+
headers={
|
| 87 |
+
"Access-Control-Allow-Origin": "*",
|
| 88 |
+
"Access-Control-Allow-Methods": "POST, OPTIONS",
|
| 89 |
+
"Access-Control-Allow-Headers": "Content-Type, Authorization",
|
| 90 |
+
}
|
| 91 |
+
)
|
| 92 |
+
|
| 93 |
+
# Custom OpenAI-compatible route with explicit CORS headers
|
| 94 |
@app.post("/v1/chat/completions")
|
| 95 |
async def chat_api(request: Request):
|
| 96 |
try:
|
|
|
|
| 100 |
temperature = body.get("temperature", 0.7)
|
| 101 |
max_tokens = body.get("max_tokens", 1024)
|
| 102 |
|
| 103 |
+
headers = {
|
| 104 |
+
"Access-Control-Allow-Origin": "*",
|
| 105 |
+
"Access-Control-Allow-Methods": "POST, OPTIONS",
|
| 106 |
+
"Access-Control-Allow-Headers": "Content-Type, Authorization",
|
| 107 |
+
}
|
| 108 |
+
|
| 109 |
if stream:
|
| 110 |
def stream_generator():
|
| 111 |
for chunk in llm.create_chat_completion(
|
|
|
|
| 116 |
):
|
| 117 |
yield f"data: {json.dumps(chunk)}\n\n"
|
| 118 |
yield "data: [DONE]\n\n"
|
| 119 |
+
return StreamingResponse(
|
| 120 |
+
stream_generator(),
|
| 121 |
+
media_type="text/event-stream",
|
| 122 |
+
headers=headers
|
| 123 |
+
)
|
| 124 |
else:
|
| 125 |
response = llm.create_chat_completion(
|
| 126 |
messages=messages,
|
|
|
|
| 128 |
temperature=temperature,
|
| 129 |
max_tokens=max_tokens
|
| 130 |
)
|
| 131 |
+
return JSONResponse(content=response, headers=headers)
|
| 132 |
except Exception as e:
|
| 133 |
+
return JSONResponse(
|
| 134 |
+
status_code=500,
|
| 135 |
+
content={"error": str(e)},
|
| 136 |
+
headers={"Access-Control-Allow-Origin": "*"}
|
| 137 |
+
)
|
| 138 |
|
| 139 |
return app
|
| 140 |
|
| 141 |
+
# Apply the module-level patch natively
|
| 142 |
+
gradio.routes.create_app = custom_create_app
|
| 143 |
+
|
| 144 |
+
# Backup patch for older classmethods
|
| 145 |
+
if hasattr(gradio.routes, "App") and hasattr(gradio.routes.App, "create_app"):
|
| 146 |
+
gradio.routes.App.create_app = custom_create_app
|
| 147 |
|
| 148 |
# 6. LAUNCH GRADIO
|
|
|
|
| 149 |
if __name__ == "__main__":
|
| 150 |
demo.launch(server_name="0.0.0.0", server_port=7860)
|