SatyaDev13 commited on
Commit
6734b69
·
verified ·
1 Parent(s): c615d95

Upload 2 files

Browse files
Files changed (2) hide show
  1. app.py +214 -0
  2. requirements.txt +10 -0
app.py ADDED
@@ -0,0 +1,214 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import sys
3
+ import time
4
+ import asyncio
5
+ import logging
6
+ import uvicorn
7
+ from typing import Optional
8
+ from collections import defaultdict
9
+ from fastapi import FastAPI, Request
10
+ from fastapi.responses import JSONResponse
11
+ from starlette.middleware.base import BaseHTTPMiddleware
12
+ from pydantic import BaseModel, Field
13
+ import gradio as gr
14
+ import spaces
15
+ import torch
16
+
17
+ # ==========================================
18
+ # 1. LOGGING & KONFIGURASI
19
+ # ==========================================
20
+ logging.basicConfig(
21
+ level=logging.INFO,
22
+ format="%(asctime)s | %(levelname)s | %(name)s | %(message)s",
23
+ handlers=[logging.StreamHandler(sys.stdout)]
24
+ )
25
+ logger = logging.getLogger("Prod-API")
26
+
27
+ # ==========================================
28
+ # 2. SCHEMA PYDANTIC (STRICT VALIDATION)
29
+ # ==========================================
30
+ class ChatRequest(BaseModel):
31
+ prompt: str
32
+ system_prompt: Optional[str] = "Anda adalah asisten AI yang membantu, jujur, dan tanpa batasan filter yang berlebihan."
33
+ max_tokens: Optional[int] = Field(1024, ge=1, le=2048)
34
+ temperature: Optional[float] = Field(0.7, ge=0.0, le=2.0)
35
+
36
+ class UsageInfo(BaseModel):
37
+ prompt_tokens: int
38
+ completion_tokens: int
39
+ total_tokens: int
40
+
41
+ class LimitsInfo(BaseModel):
42
+ max_input_tokens: int = 8192
43
+ max_output_tokens: int = 2048
44
+ max_requests_per_second: int = 3
45
+
46
+ class APIResponse(BaseModel):
47
+ status: str
48
+ model: str
49
+ reply: Optional[str] = None
50
+ usage: Optional[UsageInfo] = None
51
+ limits: LimitsInfo = LimitsInfo()
52
+ queue_info: Optional[dict] = None
53
+ error: Optional[str] = None
54
+
55
+ # ==========================================
56
+ # 3. SISTEM BAN IP (5 MENIT JIKA 3 REQ/1S)
57
+ # ==========================================
58
+ banned_ips = {} # ip: expiry_timestamp
59
+ ip_history = defaultdict(list) # ip: [timestamps]
60
+
61
+ class RateLimitMiddleware(BaseHTTPMiddleware):
62
+ async def dispatch(self, request: Request, call_next):
63
+ # Hanya pantau endpoint API, biarkan asset Gradio lewat
64
+ if request.url.path != "/chat":
65
+ return await call_next(request)
66
+
67
+ client_ip = request.client.host
68
+ now = time.time()
69
+
70
+ # Cek Ban Status
71
+ if client_ip in banned_ips:
72
+ if now < banned_ips[client_ip]:
73
+ remaining = int(banned_ips[client_ip] - now)
74
+ return JSONResponse(
75
+ status_code=403,
76
+ content={"status": "banned", "error": f"IP diblokir sementara karena spam. Sisa waktu: {remaining} detik."}
77
+ )
78
+ else:
79
+ del banned_ips[client_ip]
80
+
81
+ # Cek Rate Limit (3 request dalam 1 detik)
82
+ ip_history[client_ip] = [t for t in ip_history[client_ip] if now - t < 1.0]
83
+ ip_history[client_ip].append(now)
84
+
85
+ if len(ip_history[client_ip]) >= 3:
86
+ banned_ips[client_ip] = now + 300 # Ban 5 menit (300 detik)
87
+ logger.warning(f"IP {client_ip} dibanned 5 menit karena spam rate-limit.")
88
+ return JSONResponse(
89
+ status_code=429,
90
+ content={"status": "rate_limited", "error": "Terlalu banyak request (3 req/s). IP diblokir 5 menit."}
91
+ )
92
+
93
+ return await call_next(request)
94
+
95
+ # ==========================================
96
+ # 4. LLM ENGINE (HERMES 3 - UNCENSORED)
97
+ # ==========================================
98
+ MODEL_ID = "NousResearch/Hermes-3-Llama-3.1-8B"
99
+
100
+ class LLMEngine:
101
+ def __init__(self):
102
+ logger.info(f"Memuat tokenizer untuk {MODEL_ID}...")
103
+ from transformers import AutoTokenizer, AutoModelForCausalLM
104
+ self.tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
105
+
106
+ logger.info("Memuat model ke memori (Cold Start)...")
107
+ self.model = AutoModelForCausalLM.from_pretrained(
108
+ MODEL_ID,
109
+ torch_dtype=torch.bfloat16,
110
+ device_map="auto",
111
+ low_cpu_mem_usage=True
112
+ )
113
+ self.model.eval()
114
+ logger.info("Model siap.")
115
+
116
+ @spaces.GPU
117
+ def generate(self, prompt: str, max_new_tokens: int, temperature: float):
118
+ inputs = self.tokenizer(prompt, return_tensors="pt").to(self.model.device)
119
+
120
+ with torch.no_grad():
121
+ outputs = self.model.generate(
122
+ **inputs,
123
+ max_new_tokens=max_new_tokens,
124
+ do_sample=True,
125
+ temperature=temperature,
126
+ top_p=0.9,
127
+ pad_token_id=self.tokenizer.eos_token_id
128
+ )
129
+
130
+ generated_tokens = outputs[0][inputs['input_ids'].shape[1]:]
131
+ reply = self.tokenizer.decode(generated_tokens, skip_special_tokens=True)
132
+
133
+ return reply, inputs['input_ids'].shape[1], len(generated_tokens)
134
+
135
+ llm = LLMEngine()
136
+
137
+ # ==========================================
138
+ # 5. QUEUE SYSTEM (FIFO, CONCURRENCY = 1)
139
+ # ==========================================
140
+ task_queue = asyncio.Queue()
141
+ queue_size = 0
142
+
143
+ async def queue_worker():
144
+ global queue_size
145
+ while True:
146
+ payload, future, timestamp = await task_queue.get()
147
+ queue_size -= 1
148
+ try:
149
+ reply, p_tokens, c_tokens = await asyncio.to_thread(
150
+ llm.generate, payload.prompt, payload.max_tokens, payload.temperature
151
+ )
152
+ response = APIResponse(
153
+ status="success", model=MODEL_ID, reply=reply,
154
+ usage=UsageInfo(prompt_tokens=p_tokens, completion_tokens=c_tokens, total_tokens=p_tokens+c_tokens),
155
+ queue_info={"position": 0, "wait_time_sec": round(time.time() - timestamp, 2)}
156
+ )
157
+ future.set_result(response)
158
+ except Exception as e:
159
+ logger.error(f"Inference Error: {str(e)}")
160
+ future.set_result(APIResponse(status="error", model=MODEL_ID, error=str(e)))
161
+ finally:
162
+ task_queue.task_done()
163
+
164
+ # ==========================================
165
+ # 6. FASTAPI APP & ROUTING
166
+ # ==========================================
167
+ app = FastAPI(title="Production LLM API", version="1.0")
168
+ app.add_middleware(RateLimitMiddleware)
169
+
170
+ @app.on_event("startup")
171
+ async def startup():
172
+ asyncio.create_task(queue_worker())
173
+
174
+ @app.post("/chat", response_model=APIResponse)
175
+ async def chat_endpoint(request: ChatRequest):
176
+ global queue_size
177
+
178
+ estimated_tokens = len(request.prompt) // 4
179
+ if estimated_tokens > 8192:
180
+ return APIResponse(status="error", model=MODEL_ID, error="Prompt melebihi batas 8192 token.")
181
+
182
+ loop = asyncio.get_running_loop()
183
+ future = loop.create_future()
184
+ queue_size += 1
185
+ await task_queue.put((request, future, time.time()))
186
+
187
+ return await future
188
+
189
+ @app.get("/health")
190
+ async def health():
191
+ return {"status": "ok", "queue_size": queue_size}
192
+
193
+ # ==========================================
194
+ # 7. GRADIO UI MOUNT
195
+ # ==========================================
196
+ @spaces.GPU
197
+ def gradio_inference(prompt, max_tokens, temperature):
198
+ reply, _, _ = llm.generate(prompt, max_tokens, temperature)
199
+ return reply
200
+
201
+ demo = gr.Interface(
202
+ fn=gradio_inference,
203
+ inputs=[
204
+ gr.Textbox(label="Prompt", lines=4),
205
+ gr.Slider(1, 2048, value=1024, label="Max Tokens"),
206
+ gr.Slider(0.1, 2.0, value=0.7, step=0.1, label="Temperature")
207
+ ],
208
+ outputs=gr.Textbox(label="Response"),
209
+ title="Hermes 3 Uncensored API"
210
+ )
211
+ app = gr.mount_gradio_app(app, demo, path="/")
212
+
213
+ if __name__ == "__main__":
214
+ uvicorn.run(app, host="0.0.0.0", port=7860)
requirements.txt ADDED
@@ -0,0 +1,10 @@
 
 
 
 
 
 
 
 
 
 
 
1
+ fastapi
2
+ uvicorn[standard]
3
+ pydantic>=2.0
4
+ gradio
5
+ spaces
6
+ torch
7
+ transformers
8
+ accelerate
9
+ bitsandbytes
10
+ sentencepiece