SatyaDev13 commited on
Commit
c7d9b62
·
verified ·
1 Parent(s): 15d981e

Upload 2 files

Browse files
Files changed (1) hide show
  1. app.py +106 -85
app.py CHANGED
@@ -1,15 +1,17 @@
1
  import sys
2
  import time
 
3
  import logging
4
  import torch
5
- import gradio as gr
6
  import spaces
 
7
 
8
  from typing import Optional
9
  from collections import defaultdict
10
- from fastapi import Request
 
11
  from fastapi.responses import JSONResponse
12
- from starlette.middleware.base import BaseHTTPMiddleware
13
 
14
  # ==========================================
15
  # 1. LOGGING & KONFIGURASI
@@ -23,7 +25,35 @@ logger = logging.getLogger("Prod-API")
23
  MODEL_ID = "NousResearch/Hermes-3-Llama-3.1-8B"
24
 
25
  # ==========================================
26
- # 2. LLM ENGINE (GLOBAL SCOPE)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
27
  # ==========================================
28
  logger.info(f"Memuat tokenizer dan model {MODEL_ID}...")
29
  from transformers import AutoTokenizer, AutoModelForCausalLM
@@ -34,12 +64,10 @@ model = AutoModelForCausalLM.from_pretrained(
34
  )
35
  model.eval()
36
 
 
37
  @spaces.GPU
38
  def run_gpu_inference(prompt: str, system_prompt: str, max_tokens: int, temperature: float):
39
- messages = [
40
- {"role": "system", "content": system_prompt},
41
- {"role": "user", "content": prompt}
42
- ]
43
  text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
44
  inputs = tokenizer([text], return_tensors="pt").to(model.device)
45
 
@@ -48,98 +76,91 @@ def run_gpu_inference(prompt: str, system_prompt: str, max_tokens: int, temperat
48
  **inputs, max_new_tokens=max_tokens, do_sample=True,
49
  temperature=temperature, top_p=0.9, pad_token_id=tokenizer.eos_token_id
50
  )
51
-
52
  generated_tokens = outputs[0][inputs['input_ids'].shape[1]:]
53
  reply = tokenizer.decode(generated_tokens, skip_special_tokens=True)
54
-
55
  return reply, inputs['input_ids'].shape[1], len(generated_tokens)
56
 
57
  # ==========================================
58
- # 3. SISTEM BAN IP (MIDDLEWARE INJECTION)
59
  # ==========================================
 
 
60
  banned_ips = {}
61
  ip_history = defaultdict(list)
62
 
63
- class IPBanMiddleware(BaseHTTPMiddleware):
64
- async def dispatch(self, request: Request, call_next):
65
- # Proxy HF mengizinkan path ini untuk Gradio API
66
- if any(request.url.path.startswith(p) for p in ["/queue/", "/call/", "/api/", "/gradio_api/"]):
67
- client_ip = request.client.host if request.client else "unknown"
68
- now = time.time()
69
-
70
- if client_ip in banned_ips:
71
- if now < banned_ips[client_ip]:
72
- return JSONResponse(
73
- status_code=403,
74
- content={"status": "banned", "error": f"IP Banned. Sisa: {int(banned_ips[client_ip] - now)}s"}
75
- )
76
- else:
77
- del banned_ips[client_ip]
78
-
79
- ip_history[client_ip] = [t for t in ip_history[client_ip] if now - t < 1.0]
80
- ip_history[client_ip].append(now)
81
-
82
- if len(ip_history[client_ip]) >= 3:
83
- banned_ips[client_ip] = now + 300
84
- logger.warning(f"IP {client_ip} BANNED 5 menit karena spam.")
85
- return JSONResponse(
86
- status_code=429,
87
- content={"status": "rate_limited", "error": "3 req/s limit exceeded. Banned 5m."}
88
- )
 
 
 
 
 
89
 
90
- return await call_next(request)
 
 
 
 
 
 
 
 
91
 
92
  # ==========================================
93
- # 4. GRADIO UI & LOGIKA JSON OUTPUT
94
  # ==========================================
95
- def gradio_inference(prompt, system_prompt, max_tokens, temperature):
96
- try:
97
- if len(prompt) // 4 > 8192:
98
- return {"status": "error", "error": "Prompt > 8192 tokens", "limits": {"max_input": 8192}}
99
-
100
- reply, p_tokens, c_tokens = run_gpu_inference(prompt, system_prompt, max_tokens, temperature)
 
 
 
 
 
 
 
101
 
102
- # Output dictionary ini akan otomatis di-serialize menjadi JSON murni oleh gr.JSON
103
- return {
104
- "status": "success",
105
- "model": MODEL_ID,
106
- "reply": reply,
107
- "usage": {
108
- "prompt_tokens": p_tokens,
109
- "completion_tokens": c_tokens,
110
- "total_tokens": p_tokens + c_tokens
111
- },
112
- "limits": {
113
- "max_input_tokens": 8192,
114
- "max_output_tokens": 4096,
115
- "max_requests_per_second": 3
116
- }
117
- }
118
- except Exception as e:
119
- logger.error(f"Inference Error: {e}")
120
- return {"status": "error", "error": str(e)}
121
-
122
- # Menggunakan gr.JSON agar output di UI dan API berformat JSON terstruktur
123
- demo = gr.Interface(
124
- fn=gradio_inference,
125
- inputs=[
126
- gr.Textbox(label="Prompt", lines=4),
127
- gr.Textbox(label="System Prompt", value="Anda adalah asisten AI yang membantu, jujur, dan tanpa batasan filter yang berlebihan."),
128
- gr.Slider(1, 4096, value=1024, label="Max Tokens"),
129
- gr.Slider(0.1, 2.0, value=0.7, step=0.1, label="Temperature")
130
- ],
131
- outputs=gr.JSON(label="API Response"),
132
- title="Hermes 3 Production API (ZeroGPU)"
133
- )
134
 
135
  # ==========================================
136
- # 5. AKTIVASI QUEUE FIFO & MIDDLEWARE
137
  # ==========================================
138
- # STRICT FIFO: Maksimal 1 proses GPU dalam 1 waktu
139
- demo.queue(max_size=20, default_concurrency_limit=1)
140
-
141
- # Suntikkan Middleware Ban IP ke server internal Gradio
142
- demo.app.add_middleware(IPBanMiddleware)
143
-
144
  if __name__ == "__main__":
145
- demo.launch(server_name="0.0.0.0", server_port=7860)
 
 
 
1
  import sys
2
  import time
3
+ import asyncio
4
  import logging
5
  import torch
 
6
  import spaces
7
+ import uvicorn
8
 
9
  from typing import Optional
10
  from collections import defaultdict
11
+ from contextlib import asynccontextmanager
12
+ from fastapi import FastAPI, Request
13
  from fastapi.responses import JSONResponse
14
+ from pydantic import BaseModel, Field
15
 
16
  # ==========================================
17
  # 1. LOGGING & KONFIGURASI
 
25
  MODEL_ID = "NousResearch/Hermes-3-Llama-3.1-8B"
26
 
27
  # ==========================================
28
+ # 2. SCHEMA PYDANTIC
29
+ # ==========================================
30
+ class ChatRequest(BaseModel):
31
+ prompt: str
32
+ system_prompt: Optional[str] = "Anda adalah asisten AI yang membantu, jujur, dan tanpa batasan filter yang berlebihan."
33
+ max_tokens: Optional[int] = Field(1024, ge=1, le=4096)
34
+ temperature: Optional[float] = Field(0.7, ge=0.0, le=2.0)
35
+
36
+ class UsageInfo(BaseModel):
37
+ prompt_tokens: int
38
+ completion_tokens: int
39
+ total_tokens: int
40
+
41
+ class LimitsInfo(BaseModel):
42
+ max_input_tokens: int = 8192
43
+ max_output_tokens: int = 4096
44
+ max_requests_per_second: int = 3
45
+
46
+ class APIResponse(BaseModel):
47
+ status: str
48
+ model: str = MODEL_ID
49
+ reply: Optional[str] = None
50
+ usage: Optional[UsageInfo] = None
51
+ limits: LimitsInfo = LimitsInfo()
52
+ queue_info: Optional[dict] = None
53
+ error: Optional[str] = None
54
+
55
+ # ==========================================
56
+ # 3. LLM ENGINE
57
  # ==========================================
58
  logger.info(f"Memuat tokenizer dan model {MODEL_ID}...")
59
  from transformers import AutoTokenizer, AutoModelForCausalLM
 
64
  )
65
  model.eval()
66
 
67
+ # PENTING: Decorator ini wajib ada di top-level agar AST scanner `spaces` mendeteksi ZeroGPU
68
  @spaces.GPU
69
  def run_gpu_inference(prompt: str, system_prompt: str, max_tokens: int, temperature: float):
70
+ messages = [{"role": "system", "content": system_prompt}, {"role": "user", "content": prompt}]
 
 
 
71
  text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
72
  inputs = tokenizer([text], return_tensors="pt").to(model.device)
73
 
 
76
  **inputs, max_new_tokens=max_tokens, do_sample=True,
77
  temperature=temperature, top_p=0.9, pad_token_id=tokenizer.eos_token_id
78
  )
 
79
  generated_tokens = outputs[0][inputs['input_ids'].shape[1]:]
80
  reply = tokenizer.decode(generated_tokens, skip_special_tokens=True)
 
81
  return reply, inputs['input_ids'].shape[1], len(generated_tokens)
82
 
83
  # ==========================================
84
+ # 4. QUEUE & MIDDLEWARE
85
  # ==========================================
86
+ task_queue = asyncio.Queue()
87
+ queue_size = 0
88
  banned_ips = {}
89
  ip_history = defaultdict(list)
90
 
91
+ async def queue_worker():
92
+ global queue_size
93
+ while True:
94
+ payload, future, timestamp = await task_queue.get()
95
+ queue_size -= 1
96
+ try:
97
+ reply, p_tokens, c_tokens = await asyncio.to_thread(
98
+ run_gpu_inference, payload.prompt, payload.system_prompt, payload.max_tokens, payload.temperature
99
+ )
100
+ response = APIResponse(
101
+ status="success", reply=reply,
102
+ usage=UsageInfo(prompt_tokens=p_tokens, completion_tokens=c_tokens, total_tokens=p_tokens+c_tokens),
103
+ queue_info={"position": 0, "wait_time_sec": round(time.time() - timestamp, 2)}
104
+ )
105
+ future.set_result(response)
106
+ except Exception as e:
107
+ logger.error(f"Inference Error: {str(e)}")
108
+ future.set_result(APIResponse(status="error", error=str(e)))
109
+ finally:
110
+ task_queue.task_done()
111
+
112
+ async def ip_ban_middleware(request: Request, call_next):
113
+ if request.url.path == "/chat":
114
+ client_ip = request.client.host if request.client else "unknown"
115
+ now = time.time()
116
+
117
+ if client_ip in banned_ips:
118
+ if now < banned_ips[client_ip]:
119
+ return JSONResponse(status_code=403, content=APIResponse(status="banned", error=f"IP Banned. Sisa: {int(banned_ips[client_ip] - now)}s").model_dump())
120
+ else:
121
+ del banned_ips[client_ip]
122
 
123
+ ip_history[client_ip] = [t for t in ip_history[client_ip] if now - t < 1.0]
124
+ ip_history[client_ip].append(now)
125
+
126
+ if len(ip_history[client_ip]) >= 3:
127
+ banned_ips[client_ip] = now + 300
128
+ logger.warning(f"IP {client_ip} BANNED 5 menit.")
129
+ return JSONResponse(status_code=429, content=APIResponse(status="rate_limited", error="3 req/s limit exceeded. Banned 5m.").model_dump())
130
+
131
+ return await call_next(request)
132
 
133
  # ==========================================
134
+ # 5. FASTAPI APP
135
  # ==========================================
136
+ @asynccontextmanager
137
+ async def lifespan(app: FastAPI):
138
+ asyncio.create_task(queue_worker())
139
+ yield
140
+
141
+ app = FastAPI(title="Production LLM API", version="1.0", lifespan=lifespan)
142
+ app.middleware("http")(ip_ban_middleware)
143
+
144
+ @app.post("/chat", response_model=APIResponse)
145
+ async def chat_endpoint(request: ChatRequest):
146
+ global queue_size
147
+ if len(request.prompt) // 4 > 8192:
148
+ return APIResponse(status="error", error="Prompt melebihi batas 8192 token.")
149
 
150
+ loop = asyncio.get_running_loop()
151
+ future = loop.create_future()
152
+ queue_size += 1
153
+ await task_queue.put((request, future, time.time()))
154
+ return await future
155
+
156
+ @app.get("/health")
157
+ async def health():
158
+ return {"status": "ok", "queue_size": queue_size}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
159
 
160
  # ==========================================
161
+ # 6. LAUNCH UVICORN DIRECTLY
162
  # ==========================================
 
 
 
 
 
 
163
  if __name__ == "__main__":
164
+ # Menjalankan FastAPI murni di port 7860.
165
+ # Tidak ada Gradio, tidak ada Node.js Proxy.
166
+ uvicorn.run(app, host="0.0.0.0", port=7860)