Nafiul Huda commited on
Commit
6e246ba
·
verified ·
1 Parent(s): 490abe1

Update app.py

Browse files
Files changed (1) hide show
  1. app.py +39 -13
app.py CHANGED
@@ -2,7 +2,7 @@ import os
2
  import json
3
  import gradio as gr
4
  import spaces
5
- from fastapi import Request
6
  from fastapi.responses import JSONResponse, StreamingResponse
7
  from huggingface_hub import hf_hub_download
8
  from llama_cpp import Llama
@@ -71,17 +71,26 @@ with gr.Blocks() as demo:
71
  examples=["Who are you?", "Write a python script to reverse a string.", "Explain quantum computing."],
72
  )
73
 
74
- # 5. INJECT FASTAPI ROUTES DIRECTLY INTO GRADIO
75
- # This safely bypasses the ZeroGPU port collision by attaching our API
76
- # directly to the Gradio app before the server boots up.
77
  import gradio.routes
78
- original_create_app = gradio.routes.App.create_app
79
 
80
  def custom_create_app(*args, **kwargs):
81
- # Let Gradio build its native FastAPI app
82
  app = original_create_app(*args, **kwargs)
83
 
84
- # Inject our custom OpenAI-compatible route onto it
 
 
 
 
 
 
 
 
 
 
 
 
85
  @app.post("/v1/chat/completions")
86
  async def chat_api(request: Request):
87
  try:
@@ -91,6 +100,12 @@ def custom_create_app(*args, **kwargs):
91
  temperature = body.get("temperature", 0.7)
92
  max_tokens = body.get("max_tokens", 1024)
93
 
 
 
 
 
 
 
94
  if stream:
95
  def stream_generator():
96
  for chunk in llm.create_chat_completion(
@@ -101,7 +116,11 @@ def custom_create_app(*args, **kwargs):
101
  ):
102
  yield f"data: {json.dumps(chunk)}\n\n"
103
  yield "data: [DONE]\n\n"
104
- return StreamingResponse(stream_generator(), media_type="text/event-stream")
 
 
 
 
105
  else:
106
  response = llm.create_chat_completion(
107
  messages=messages,
@@ -109,16 +128,23 @@ def custom_create_app(*args, **kwargs):
109
  temperature=temperature,
110
  max_tokens=max_tokens
111
  )
112
- return JSONResponse(content=response)
113
  except Exception as e:
114
- return JSONResponse(status_code=500, content={"error": str(e)})
 
 
 
 
115
 
116
  return app
117
 
118
- # Apply the patch
119
- gradio.routes.App.create_app = custom_create_app
 
 
 
 
120
 
121
  # 6. LAUNCH GRADIO
122
- # ZeroGPU will safely intercept this and route our API & UI perfectly!
123
  if __name__ == "__main__":
124
  demo.launch(server_name="0.0.0.0", server_port=7860)
 
2
  import json
3
  import gradio as gr
4
  import spaces
5
+ from fastapi import Request, Response
6
  from fastapi.responses import JSONResponse, StreamingResponse
7
  from huggingface_hub import hf_hub_download
8
  from llama_cpp import Llama
 
71
  examples=["Who are you?", "Write a python script to reverse a string.", "Explain quantum computing."],
72
  )
73
 
74
+ # 5. INJECT FASTAPI ROUTES DIRECTLY INTO GRADIO WITH CORS SUPPORT
 
 
75
  import gradio.routes
76
+ original_create_app = getattr(gradio.routes, "create_app", None)
77
 
78
  def custom_create_app(*args, **kwargs):
 
79
  app = original_create_app(*args, **kwargs)
80
 
81
+ # CORS Preflight (OPTIONS) handler
82
+ @app.options("/v1/chat/completions")
83
+ async def chat_api_options():
84
+ return Response(
85
+ status_code=200,
86
+ headers={
87
+ "Access-Control-Allow-Origin": "*",
88
+ "Access-Control-Allow-Methods": "POST, OPTIONS",
89
+ "Access-Control-Allow-Headers": "Content-Type, Authorization",
90
+ }
91
+ )
92
+
93
+ # Custom OpenAI-compatible route with explicit CORS headers
94
  @app.post("/v1/chat/completions")
95
  async def chat_api(request: Request):
96
  try:
 
100
  temperature = body.get("temperature", 0.7)
101
  max_tokens = body.get("max_tokens", 1024)
102
 
103
+ headers = {
104
+ "Access-Control-Allow-Origin": "*",
105
+ "Access-Control-Allow-Methods": "POST, OPTIONS",
106
+ "Access-Control-Allow-Headers": "Content-Type, Authorization",
107
+ }
108
+
109
  if stream:
110
  def stream_generator():
111
  for chunk in llm.create_chat_completion(
 
116
  ):
117
  yield f"data: {json.dumps(chunk)}\n\n"
118
  yield "data: [DONE]\n\n"
119
+ return StreamingResponse(
120
+ stream_generator(),
121
+ media_type="text/event-stream",
122
+ headers=headers
123
+ )
124
  else:
125
  response = llm.create_chat_completion(
126
  messages=messages,
 
128
  temperature=temperature,
129
  max_tokens=max_tokens
130
  )
131
+ return JSONResponse(content=response, headers=headers)
132
  except Exception as e:
133
+ return JSONResponse(
134
+ status_code=500,
135
+ content={"error": str(e)},
136
+ headers={"Access-Control-Allow-Origin": "*"}
137
+ )
138
 
139
  return app
140
 
141
+ # Apply the module-level patch natively
142
+ gradio.routes.create_app = custom_create_app
143
+
144
+ # Backup patch for older classmethods
145
+ if hasattr(gradio.routes, "App") and hasattr(gradio.routes.App, "create_app"):
146
+ gradio.routes.App.create_app = custom_create_app
147
 
148
  # 6. LAUNCH GRADIO
 
149
  if __name__ == "__main__":
150
  demo.launch(server_name="0.0.0.0", server_port=7860)