SatyaDev13 commited on
Commit
dd20e4e
·
verified ·
1 Parent(s): d2b0fc2

Upload app.py

Browse files
Files changed (1) hide show
  1. app.py +27 -149
app.py CHANGED
@@ -1,150 +1,14 @@
1
- import sys
2
- import time
3
- import asyncio
4
- import logging
5
- import torch
6
- import gradio as gr
7
- import spaces
8
-
9
- from typing import Optional
10
- from collections import defaultdict
11
- from pydantic import BaseModel, Field
12
- from fastapi import Request
13
- from fastapi.responses import JSONResponse
14
-
15
- # ==========================================
16
- # 1. LOGGING & KONFIGURASI
17
  # ==========================================
18
- logging.basicConfig(
19
- level=logging.INFO,
20
- format="%(asctime)s | %(levelname)s | %(message)s",
21
- handlers=[logging.StreamHandler(sys.stdout)]
22
- )
23
- logger = logging.getLogger("Prod-API")
24
-
25
- MODEL_ID = "NousResearch/Hermes-3-Llama-3.1-8B"
26
-
27
- # ==========================================
28
- # 2. SCHEMA PYDANTIC (JSON OUTPUT STANDAR)
29
  # ==========================================
30
- class ChatRequest(BaseModel):
31
- prompt: str
32
- system_prompt: Optional[str] = "Anda adalah asisten AI yang membantu, jujur, dan tanpa batasan filter yang berlebihan."
33
- max_tokens: Optional[int] = Field(1024, ge=1, le=4096)
34
- temperature: Optional[float] = Field(0.7, ge=0.0, le=2.0)
35
-
36
- class UsageInfo(BaseModel):
37
- prompt_tokens: int
38
- completion_tokens: int
39
- total_tokens: int
40
-
41
- class LimitsInfo(BaseModel):
42
- max_input_tokens: int = 8192
43
- max_output_tokens: int = 4096
44
- max_requests_per_second: int = 3
45
-
46
- class APIResponse(BaseModel):
47
- status: str
48
- model: str = MODEL_ID
49
- reply: Optional[str] = None
50
- usage: Optional[UsageInfo] = None
51
- limits: LimitsInfo = LimitsInfo()
52
- queue_info: Optional[dict] = None
53
- error: Optional[str] = None
54
-
55
- # ==========================================
56
- # 3. LLM ENGINE (GLOBAL SCOPE)
57
- # ==========================================
58
- logger.info(f"Memuat tokenizer dan model {MODEL_ID}...")
59
- from transformers import AutoTokenizer, AutoModelForCausalLM
60
-
61
- tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
62
- model = AutoModelForCausalLM.from_pretrained(
63
- MODEL_ID,
64
- torch_dtype=torch.bfloat16,
65
- device_map="auto",
66
- low_cpu_mem_usage=True
67
- )
68
- model.eval()
69
-
70
- @spaces.GPU
71
- def run_gpu_inference(prompt: str, system_prompt: str, max_tokens: int, temperature: float):
72
- messages = [
73
- {"role": "system", "content": system_prompt},
74
- {"role": "user", "content": prompt}
75
- ]
76
- text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
77
- inputs = tokenizer([text], return_tensors="pt").to(model.device)
78
-
79
- with torch.no_grad():
80
- outputs = model.generate(
81
- **inputs, max_new_tokens=max_tokens, do_sample=True,
82
- temperature=temperature, top_p=0.9, pad_token_id=tokenizer.eos_token_id
83
- )
84
-
85
- generated_tokens = outputs[0][inputs['input_ids'].shape[1]:]
86
- reply = tokenizer.decode(generated_tokens, skip_special_tokens=True)
87
-
88
- return reply, inputs['input_ids'].shape[1], len(generated_tokens)
89
-
90
- # ==========================================
91
- # 4. QUEUE SYSTEM & ANTI-SPAM
92
- # ==========================================
93
- task_queue = asyncio.Queue()
94
- queue_size = 0
95
- banned_ips = {}
96
- ip_history = defaultdict(list)
97
-
98
- async def queue_worker():
99
- global queue_size
100
- while True:
101
- payload, future, timestamp = await task_queue.get()
102
- queue_size -= 1
103
- try:
104
- reply, p_tokens, c_tokens = await asyncio.to_thread(
105
- run_gpu_inference, payload.prompt, payload.system_prompt, payload.max_tokens, payload.temperature
106
- )
107
- response = APIResponse(
108
- status="success", reply=reply,
109
- usage=UsageInfo(prompt_tokens=p_tokens, completion_tokens=c_tokens, total_tokens=p_tokens+c_tokens),
110
- queue_info={"position": 0, "wait_time_sec": round(time.time() - timestamp, 2)}
111
- )
112
- future.set_result(response)
113
- except Exception as e:
114
- logger.error(f"Inference Error: {str(e)}")
115
- future.set_result(APIResponse(status="error", error=str(e)))
116
- finally:
117
- task_queue.task_done()
118
-
119
- # ==========================================
120
- # 5. GRADIO UI DEFINITION
121
- # ==========================================
122
- def gradio_inference(prompt, system_prompt, max_tokens, temperature):
123
- reply, _, _ = run_gpu_inference(prompt, system_prompt, max_tokens, temperature)
124
- return reply
125
-
126
- demo = gr.Interface(
127
- fn=gradio_inference,
128
- inputs=[
129
- gr.Textbox(label="Prompt", lines=4),
130
- gr.Textbox(label="System Prompt", value="Anda adalah asisten AI yang membantu, jujur, dan tanpa batasan filter yang berlebihan."),
131
- gr.Slider(1, 4096, value=1024, label="Max Tokens"),
132
- gr.Slider(0.1, 2.0, value=0.7, step=0.1, label="Temperature")
133
- ],
134
- outputs=gr.Textbox(label="Response"),
135
- title="Hermes 3 Uncensored API"
136
- )
137
 
138
- # ==========================================
139
- # 6. FASTAPI ROUTE INJECTION (INTO GRADIO)
140
- # ==========================================
141
- # Kita menyuntikkan route POST langsung ke server internal Gradio
142
- @demo.app.post("/chat", response_model=APIResponse)
143
- async def chat_endpoint(request: ChatRequest, req: Request):
144
  global queue_size
145
 
146
  # 1. Ekstraksi IP & Logika Ban (5 Menit jika 3 req/detik)
147
- client_ip = req.client.host if req.client else "unknown"
148
  now = time.time()
149
 
150
  if client_ip in banned_ips:
@@ -168,30 +32,44 @@ async def chat_endpoint(request: ChatRequest, req: Request):
168
  content=APIResponse(status="rate_limited", error="Terlalu banyak request (3 req/s). IP diblokir 5 menit.").model_dump()
169
  )
170
 
171
- # 2. Validasi Token Kasar
172
- estimated_tokens = len(request.prompt) // 4
 
 
 
 
 
 
 
 
 
 
 
 
173
  if estimated_tokens > 8192:
174
  return JSONResponse(
175
  status_code=400,
176
  content=APIResponse(status="error", error="Prompt melebihi batas 8192 token.").model_dump()
177
  )
178
 
179
- # 3. Masukkan ke Antrean FIFO (Strict Concurrency = 1)
180
  loop = asyncio.get_running_loop()
181
  future = loop.create_future()
182
  queue_size += 1
183
- await task_queue.put((request, future, time.time()))
184
 
185
- return await future
 
 
 
 
186
 
187
- # Memicu background worker saat server Gradio internal menyala
188
  @demo.app.on_event("startup")
189
  async def startup_event():
190
  asyncio.create_task(queue_worker())
191
 
192
  # ==========================================
193
- # 7. LAUNCH GRADIO (TANPA UVICORN.RUN)
194
  # ==========================================
195
  if __name__ == "__main__":
196
- # Gradio SDK akan otomatis menangani server binding ke 0.0.0.0:7860
197
  demo.launch(server_name="0.0.0.0", server_port=7860)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
  # ==========================================
2
+ # 6. FASTAPI ROUTE INJECTION (ANTI-BLOCK PROXY)
 
 
 
 
 
 
 
 
 
 
3
  # ==========================================
4
+ from fastapi import Request
5
+ from fastapi.responses import JSONResponse
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
6
 
7
+ async def chat_endpoint(request: Request):
 
 
 
 
 
8
  global queue_size
9
 
10
  # 1. Ekstraksi IP & Logika Ban (5 Menit jika 3 req/detik)
11
+ client_ip = request.client.host if request.client else "unknown"
12
  now = time.time()
13
 
14
  if client_ip in banned_ips:
 
32
  content=APIResponse(status="rate_limited", error="Terlalu banyak request (3 req/s). IP diblokir 5 menit.").model_dump()
33
  )
34
 
35
+ # 2. Parsing JSON Body secara manual (lebih aman dari proxy)
36
+ try:
37
+ body = await request.json()
38
+ prompt = body.get("prompt", "")
39
+ system_prompt = body.get("system_prompt", "Anda adalah asisten AI yang jujur.")
40
+ max_tokens = body.get("max_tokens", 1024)
41
+ temperature = body.get("temperature", 0.7)
42
+ except Exception:
43
+ return JSONResponse(status_code=400, content={"status": "error", "error": "Invalid JSON body"})
44
+
45
+ req_obj = ChatRequest(prompt=prompt, system_prompt=system_prompt, max_tokens=max_tokens, temperature=temperature)
46
+
47
+ # 3. Validasi Token Kasar
48
+ estimated_tokens = len(req_obj.prompt) // 4
49
  if estimated_tokens > 8192:
50
  return JSONResponse(
51
  status_code=400,
52
  content=APIResponse(status="error", error="Prompt melebihi batas 8192 token.").model_dump()
53
  )
54
 
55
+ # 4. Masukkan ke Antrean FIFO (Strict Concurrency = 1)
56
  loop = asyncio.get_running_loop()
57
  future = loop.create_future()
58
  queue_size += 1
59
+ await task_queue.put((req_obj, future, time.time()))
60
 
61
+ result = await future
62
+ return JSONResponse(content=result.model_dump(), status_code=200 if result.status == "success" else 500)
63
+
64
+ # INI KUNCINYA: Gunakan add_api_route dengan path /api/... untuk lolos dari proxy HF Space
65
+ demo.app.add_api_route("/api/chat", chat_endpoint, methods=["POST"])
66
 
 
67
  @demo.app.on_event("startup")
68
  async def startup_event():
69
  asyncio.create_task(queue_worker())
70
 
71
  # ==========================================
72
+ # 7. LAUNCH GRADIO
73
  # ==========================================
74
  if __name__ == "__main__":
 
75
  demo.launch(server_name="0.0.0.0", server_port=7860)