SatyaDev13 commited on
Commit
d2b0fc2
·
verified ·
1 Parent(s): 497b858

Upload app.py

Browse files
Files changed (1) hide show
  1. app.py +78 -99
app.py CHANGED
@@ -1,18 +1,16 @@
1
- import os
2
  import sys
3
  import time
4
  import asyncio
5
  import logging
6
- import uvicorn
 
 
 
7
  from typing import Optional
8
  from collections import defaultdict
9
- from fastapi import FastAPI, Request
10
- from fastapi.responses import JSONResponse
11
- from starlette.middleware.base import BaseHTTPMiddleware
12
  from pydantic import BaseModel, Field
13
- import gradio as gr
14
- import spaces
15
- import torch
16
 
17
  # ==========================================
18
  # 1. LOGGING & KONFIGURASI
@@ -24,8 +22,10 @@ logging.basicConfig(
24
  )
25
  logger = logging.getLogger("Prod-API")
26
 
 
 
27
  # ==========================================
28
- # 2. SCHEMA PYDANTIC (STRICT VALIDATION)
29
  # ==========================================
30
  class ChatRequest(BaseModel):
31
  prompt: str
@@ -45,7 +45,7 @@ class LimitsInfo(BaseModel):
45
 
46
  class APIResponse(BaseModel):
47
  status: str
48
- model: str
49
  reply: Optional[str] = None
50
  usage: Optional[UsageInfo] = None
51
  limits: LimitsInfo = LimitsInfo()
@@ -53,49 +53,8 @@ class APIResponse(BaseModel):
53
  error: Optional[str] = None
54
 
55
  # ==========================================
56
- # 3. SISTEM BAN IP (5 MENIT JIKA 3 REQ/1S)
57
- # ==========================================
58
- banned_ips = {} # ip: expiry_timestamp
59
- ip_history = defaultdict(list) # ip: [timestamps]
60
-
61
- class RateLimitMiddleware(BaseHTTPMiddleware):
62
- async def dispatch(self, request: Request, call_next):
63
- # Hanya pantau endpoint API, biarkan asset Gradio lewat
64
- if request.url.path != "/chat":
65
- return await call_next(request)
66
-
67
- client_ip = request.client.host
68
- now = time.time()
69
-
70
- # Cek Ban Status
71
- if client_ip in banned_ips:
72
- if now < banned_ips[client_ip]:
73
- remaining = int(banned_ips[client_ip] - now)
74
- return JSONResponse(
75
- status_code=403,
76
- content={"status": "banned", "error": f"IP diblokir sementara. Sisa waktu: {remaining} detik."}
77
- )
78
- else:
79
- del banned_ips[client_ip]
80
-
81
- # Cek Rate Limit (3 request dalam 1 detik)
82
- ip_history[client_ip] = [t for t in ip_history[client_ip] if now - t < 1.0]
83
- ip_history[client_ip].append(now)
84
-
85
- if len(ip_history[client_ip]) >= 3:
86
- banned_ips[client_ip] = now + 300 # Ban 5 menit (300 detik)
87
- logger.warning(f"IP {client_ip} dibanned 5 menit karena spam rate-limit.")
88
- return JSONResponse(
89
- status_code=429,
90
- content={"status": "rate_limited", "error": "Terlalu banyak request (3 req/s). IP diblokir 5 menit."}
91
- )
92
-
93
- return await call_next(request)
94
-
95
- # ==========================================
96
- # 4. LLM ENGINE (GLOBAL SCOPE)
97
  # ==========================================
98
- MODEL_ID = "NousResearch/Hermes-3-Llama-3.1-8B"
99
  logger.info(f"Memuat tokenizer dan model {MODEL_ID}...")
100
  from transformers import AutoTokenizer, AutoModelForCausalLM
101
 
@@ -108,28 +67,19 @@ model = AutoModelForCausalLM.from_pretrained(
108
  )
109
  model.eval()
110
 
111
- # PENTING: Fungsi yang menggunakan @spaces.GPU HARUS berada di level global (tidak boleh di dalam class)
112
  @spaces.GPU
113
  def run_gpu_inference(prompt: str, system_prompt: str, max_tokens: int, temperature: float):
114
  messages = [
115
  {"role": "system", "content": system_prompt},
116
  {"role": "user", "content": prompt}
117
  ]
118
- text = tokenizer.apply_chat_template(
119
- messages,
120
- tokenize=False,
121
- add_generation_prompt=True
122
- )
123
  inputs = tokenizer([text], return_tensors="pt").to(model.device)
124
 
125
  with torch.no_grad():
126
  outputs = model.generate(
127
- **inputs,
128
- max_new_tokens=max_tokens,
129
- do_sample=True,
130
- temperature=temperature,
131
- top_p=0.9,
132
- pad_token_id=tokenizer.eos_token_id
133
  )
134
 
135
  generated_tokens = outputs[0][inputs['input_ids'].shape[1]:]
@@ -138,10 +88,12 @@ def run_gpu_inference(prompt: str, system_prompt: str, max_tokens: int, temperat
138
  return reply, inputs['input_ids'].shape[1], len(generated_tokens)
139
 
140
  # ==========================================
141
- # 5. QUEUE SYSTEM (FIFO, CONCURRENCY = 1)
142
  # ==========================================
143
  task_queue = asyncio.Queue()
144
  queue_size = 0
 
 
145
 
146
  async def queue_worker():
147
  global queue_size
@@ -153,35 +105,78 @@ async def queue_worker():
153
  run_gpu_inference, payload.prompt, payload.system_prompt, payload.max_tokens, payload.temperature
154
  )
155
  response = APIResponse(
156
- status="success", model=MODEL_ID, reply=reply,
157
  usage=UsageInfo(prompt_tokens=p_tokens, completion_tokens=c_tokens, total_tokens=p_tokens+c_tokens),
158
  queue_info={"position": 0, "wait_time_sec": round(time.time() - timestamp, 2)}
159
  )
160
  future.set_result(response)
161
  except Exception as e:
162
  logger.error(f"Inference Error: {str(e)}")
163
- future.set_result(APIResponse(status="error", model=MODEL_ID, error=str(e)))
164
  finally:
165
  task_queue.task_done()
166
 
167
  # ==========================================
168
- # 6. FASTAPI APP & ROUTING
169
  # ==========================================
170
- app = FastAPI(title="Production LLM API", version="1.0")
171
- app.add_middleware(RateLimitMiddleware)
 
172
 
173
- @app.on_event("startup")
174
- async def startup():
175
- asyncio.create_task(queue_worker())
 
 
 
 
 
 
 
 
176
 
177
- @app.post("/chat", response_model=APIResponse)
178
- async def chat_endpoint(request: ChatRequest):
 
 
 
 
179
  global queue_size
180
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
181
  estimated_tokens = len(request.prompt) // 4
182
  if estimated_tokens > 8192:
183
- return APIResponse(status="error", model=MODEL_ID, error="Prompt melebihi batas 8192 token.")
 
 
 
184
 
 
185
  loop = asyncio.get_running_loop()
186
  future = loop.create_future()
187
  queue_size += 1
@@ -189,30 +184,14 @@ async def chat_endpoint(request: ChatRequest):
189
 
190
  return await future
191
 
192
- @app.get("/health")
193
- async def health():
194
- return {"status": "ok", "queue_size": queue_size}
 
195
 
196
  # ==========================================
197
- # 7. GRADIO UI MOUNT
198
  # ==========================================
199
- def gradio_inference(prompt, system_prompt, max_tokens, temperature):
200
- # Memanggil fungsi global yang sudah didekorasi @spaces.GPU
201
- reply, _, _ = run_gpu_inference(prompt, system_prompt, max_tokens, temperature)
202
- return reply
203
-
204
- demo = gr.Interface(
205
- fn=gradio_inference,
206
- inputs=[
207
- gr.Textbox(label="Prompt", lines=4),
208
- gr.Textbox(label="System Prompt", value="Anda adalah asisten AI yang membantu, jujur, dan tanpa batasan filter yang berlebihan."),
209
- gr.Slider(1, 4096, value=1024, label="Max Tokens"),
210
- gr.Slider(0.1, 2.0, value=0.7, step=0.1, label="Temperature")
211
- ],
212
- outputs=gr.Textbox(label="Response"),
213
- title="Hermes 3 Uncensored API"
214
- )
215
- app = gr.mount_gradio_app(app, demo, path="/")
216
-
217
  if __name__ == "__main__":
218
- uvicorn.run(app, host="0.0.0.0", port=7860)
 
 
 
1
  import sys
2
  import time
3
  import asyncio
4
  import logging
5
+ import torch
6
+ import gradio as gr
7
+ import spaces
8
+
9
  from typing import Optional
10
  from collections import defaultdict
 
 
 
11
  from pydantic import BaseModel, Field
12
+ from fastapi import Request
13
+ from fastapi.responses import JSONResponse
 
14
 
15
  # ==========================================
16
  # 1. LOGGING & KONFIGURASI
 
22
  )
23
  logger = logging.getLogger("Prod-API")
24
 
25
+ MODEL_ID = "NousResearch/Hermes-3-Llama-3.1-8B"
26
+
27
  # ==========================================
28
+ # 2. SCHEMA PYDANTIC (JSON OUTPUT STANDAR)
29
  # ==========================================
30
  class ChatRequest(BaseModel):
31
  prompt: str
 
45
 
46
  class APIResponse(BaseModel):
47
  status: str
48
+ model: str = MODEL_ID
49
  reply: Optional[str] = None
50
  usage: Optional[UsageInfo] = None
51
  limits: LimitsInfo = LimitsInfo()
 
53
  error: Optional[str] = None
54
 
55
  # ==========================================
56
+ # 3. LLM ENGINE (GLOBAL SCOPE)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
57
  # ==========================================
 
58
  logger.info(f"Memuat tokenizer dan model {MODEL_ID}...")
59
  from transformers import AutoTokenizer, AutoModelForCausalLM
60
 
 
67
  )
68
  model.eval()
69
 
 
70
  @spaces.GPU
71
  def run_gpu_inference(prompt: str, system_prompt: str, max_tokens: int, temperature: float):
72
  messages = [
73
  {"role": "system", "content": system_prompt},
74
  {"role": "user", "content": prompt}
75
  ]
76
+ text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
 
 
 
 
77
  inputs = tokenizer([text], return_tensors="pt").to(model.device)
78
 
79
  with torch.no_grad():
80
  outputs = model.generate(
81
+ **inputs, max_new_tokens=max_tokens, do_sample=True,
82
+ temperature=temperature, top_p=0.9, pad_token_id=tokenizer.eos_token_id
 
 
 
 
83
  )
84
 
85
  generated_tokens = outputs[0][inputs['input_ids'].shape[1]:]
 
88
  return reply, inputs['input_ids'].shape[1], len(generated_tokens)
89
 
90
  # ==========================================
91
+ # 4. QUEUE SYSTEM & ANTI-SPAM
92
  # ==========================================
93
  task_queue = asyncio.Queue()
94
  queue_size = 0
95
+ banned_ips = {}
96
+ ip_history = defaultdict(list)
97
 
98
  async def queue_worker():
99
  global queue_size
 
105
  run_gpu_inference, payload.prompt, payload.system_prompt, payload.max_tokens, payload.temperature
106
  )
107
  response = APIResponse(
108
+ status="success", reply=reply,
109
  usage=UsageInfo(prompt_tokens=p_tokens, completion_tokens=c_tokens, total_tokens=p_tokens+c_tokens),
110
  queue_info={"position": 0, "wait_time_sec": round(time.time() - timestamp, 2)}
111
  )
112
  future.set_result(response)
113
  except Exception as e:
114
  logger.error(f"Inference Error: {str(e)}")
115
+ future.set_result(APIResponse(status="error", error=str(e)))
116
  finally:
117
  task_queue.task_done()
118
 
119
  # ==========================================
120
+ # 5. GRADIO UI DEFINITION
121
  # ==========================================
122
+ def gradio_inference(prompt, system_prompt, max_tokens, temperature):
123
+ reply, _, _ = run_gpu_inference(prompt, system_prompt, max_tokens, temperature)
124
+ return reply
125
 
126
+ demo = gr.Interface(
127
+ fn=gradio_inference,
128
+ inputs=[
129
+ gr.Textbox(label="Prompt", lines=4),
130
+ gr.Textbox(label="System Prompt", value="Anda adalah asisten AI yang membantu, jujur, dan tanpa batasan filter yang berlebihan."),
131
+ gr.Slider(1, 4096, value=1024, label="Max Tokens"),
132
+ gr.Slider(0.1, 2.0, value=0.7, step=0.1, label="Temperature")
133
+ ],
134
+ outputs=gr.Textbox(label="Response"),
135
+ title="Hermes 3 Uncensored API"
136
+ )
137
 
138
+ # ==========================================
139
+ # 6. FASTAPI ROUTE INJECTION (INTO GRADIO)
140
+ # ==========================================
141
+ # Kita menyuntikkan route POST langsung ke server internal Gradio
142
+ @demo.app.post("/chat", response_model=APIResponse)
143
+ async def chat_endpoint(request: ChatRequest, req: Request):
144
  global queue_size
145
 
146
+ # 1. Ekstraksi IP & Logika Ban (5 Menit jika 3 req/detik)
147
+ client_ip = req.client.host if req.client else "unknown"
148
+ now = time.time()
149
+
150
+ if client_ip in banned_ips:
151
+ if now < banned_ips[client_ip]:
152
+ remaining = int(banned_ips[client_ip] - now)
153
+ return JSONResponse(
154
+ status_code=403,
155
+ content=APIResponse(status="banned", error=f"IP diblokir sementara. Sisa waktu: {remaining} detik.").model_dump()
156
+ )
157
+ else:
158
+ del banned_ips[client_ip]
159
+
160
+ ip_history[client_ip] = [t for t in ip_history[client_ip] if now - t < 1.0]
161
+ ip_history[client_ip].append(now)
162
+
163
+ if len(ip_history[client_ip]) >= 3:
164
+ banned_ips[client_ip] = now + 300
165
+ logger.warning(f"IP {client_ip} dibanned 5 menit karena spam rate-limit.")
166
+ return JSONResponse(
167
+ status_code=429,
168
+ content=APIResponse(status="rate_limited", error="Terlalu banyak request (3 req/s). IP diblokir 5 menit.").model_dump()
169
+ )
170
+
171
+ # 2. Validasi Token Kasar
172
  estimated_tokens = len(request.prompt) // 4
173
  if estimated_tokens > 8192:
174
+ return JSONResponse(
175
+ status_code=400,
176
+ content=APIResponse(status="error", error="Prompt melebihi batas 8192 token.").model_dump()
177
+ )
178
 
179
+ # 3. Masukkan ke Antrean FIFO (Strict Concurrency = 1)
180
  loop = asyncio.get_running_loop()
181
  future = loop.create_future()
182
  queue_size += 1
 
184
 
185
  return await future
186
 
187
+ # Memicu background worker saat server Gradio internal menyala
188
+ @demo.app.on_event("startup")
189
+ async def startup_event():
190
+ asyncio.create_task(queue_worker())
191
 
192
  # ==========================================
193
+ # 7. LAUNCH GRADIO (TANPA UVICORN.RUN)
194
  # ==========================================
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
195
  if __name__ == "__main__":
196
+ # Gradio SDK akan otomatis menangani server binding ke 0.0.0.0:7860
197
+ demo.launch(server_name="0.0.0.0", server_port=7860)