SatyaDev13 commited on
Commit
ef315c4
·
verified ·
1 Parent(s): dd20e4e

Upload app.py

Browse files
Files changed (1) hide show
  1. app.py +137 -3
app.py CHANGED
@@ -1,9 +1,143 @@
1
- # ==========================================
2
- # 6. FASTAPI ROUTE INJECTION (ANTI-BLOCK PROXY)
3
- # ==========================================
 
 
 
 
 
 
 
 
4
  from fastapi import Request
5
  from fastapi.responses import JSONResponse
6
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
7
  async def chat_endpoint(request: Request):
8
  global queue_size
9
 
 
1
+ import sys
2
+ import time
3
+ import asyncio
4
+ import logging
5
+ import torch
6
+ import gradio as gr
7
+ import spaces
8
+
9
+ from typing import Optional
10
+ from collections import defaultdict
11
+ from pydantic import BaseModel, Field
12
  from fastapi import Request
13
  from fastapi.responses import JSONResponse
14
 
15
+ # ==========================================
16
+ # 1. LOGGING & KONFIGURASI
17
+ # ==========================================
18
+ logging.basicConfig(
19
+ level=logging.INFO,
20
+ format="%(asctime)s | %(levelname)s | %(message)s",
21
+ handlers=[logging.StreamHandler(sys.stdout)]
22
+ )
23
+ logger = logging.getLogger("Prod-API")
24
+
25
+ MODEL_ID = "NousResearch/Hermes-3-Llama-3.1-8B"
26
+
27
+ # ==========================================
28
+ # 2. SCHEMA PYDANTIC (JSON OUTPUT STANDAR)
29
+ # ==========================================
30
+ class ChatRequest(BaseModel):
31
+ prompt: str
32
+ system_prompt: Optional[str] = "Anda adalah asisten AI yang membantu, jujur, dan tanpa batasan filter yang berlebihan."
33
+ max_tokens: Optional[int] = Field(1024, ge=1, le=4096)
34
+ temperature: Optional[float] = Field(0.7, ge=0.0, le=2.0)
35
+
36
+ class UsageInfo(BaseModel):
37
+ prompt_tokens: int
38
+ completion_tokens: int
39
+ total_tokens: int
40
+
41
+ class LimitsInfo(BaseModel):
42
+ max_input_tokens: int = 8192
43
+ max_output_tokens: int = 4096
44
+ max_requests_per_second: int = 3
45
+
46
+ class APIResponse(BaseModel):
47
+ status: str
48
+ model: str = MODEL_ID
49
+ reply: Optional[str] = None
50
+ usage: Optional[UsageInfo] = None
51
+ limits: LimitsInfo = LimitsInfo()
52
+ queue_info: Optional[dict] = None
53
+ error: Optional[str] = None
54
+
55
+ # ==========================================
56
+ # 3. LLM ENGINE (GLOBAL SCOPE)
57
+ # ==========================================
58
+ logger.info(f"Memuat tokenizer dan model {MODEL_ID}...")
59
+ from transformers import AutoTokenizer, AutoModelForCausalLM
60
+
61
+ tokenizer = AutoTokenizer.from_pretrained(MODEL_ID, trust_remote_code=True)
62
+ model = AutoModelForCausalLM.from_pretrained(
63
+ MODEL_ID,
64
+ torch_dtype=torch.bfloat16,
65
+ device_map="auto",
66
+ low_cpu_mem_usage=True
67
+ )
68
+ model.eval()
69
+
70
+ @spaces.GPU
71
+ def run_gpu_inference(prompt: str, system_prompt: str, max_tokens: int, temperature: float):
72
+ messages = [
73
+ {"role": "system", "content": system_prompt},
74
+ {"role": "user", "content": prompt}
75
+ ]
76
+ text = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
77
+ inputs = tokenizer([text], return_tensors="pt").to(model.device)
78
+
79
+ with torch.no_grad():
80
+ outputs = model.generate(
81
+ **inputs, max_new_tokens=max_tokens, do_sample=True,
82
+ temperature=temperature, top_p=0.9, pad_token_id=tokenizer.eos_token_id
83
+ )
84
+
85
+ generated_tokens = outputs[0][inputs['input_ids'].shape[1]:]
86
+ reply = tokenizer.decode(generated_tokens, skip_special_tokens=True)
87
+
88
+ return reply, inputs['input_ids'].shape[1], len(generated_tokens)
89
+
90
+ # ==========================================
91
+ # 4. QUEUE SYSTEM & ANTI-SPAM
92
+ # ==========================================
93
+ task_queue = asyncio.Queue()
94
+ queue_size = 0
95
+ banned_ips = {}
96
+ ip_history = defaultdict(list)
97
+
98
+ async def queue_worker():
99
+ global queue_size
100
+ while True:
101
+ payload, future, timestamp = await task_queue.get()
102
+ queue_size -= 1
103
+ try:
104
+ reply, p_tokens, c_tokens = await asyncio.to_thread(
105
+ run_gpu_inference, payload.prompt, payload.system_prompt, payload.max_tokens, payload.temperature
106
+ )
107
+ response = APIResponse(
108
+ status="success", reply=reply,
109
+ usage=UsageInfo(prompt_tokens=p_tokens, completion_tokens=c_tokens, total_tokens=p_tokens+c_tokens),
110
+ queue_info={"position": 0, "wait_time_sec": round(time.time() - timestamp, 2)}
111
+ )
112
+ future.set_result(response)
113
+ except Exception as e:
114
+ logger.error(f"Inference Error: {str(e)}")
115
+ future.set_result(APIResponse(status="error", error=str(e)))
116
+ finally:
117
+ task_queue.task_done()
118
+
119
+ # ==========================================
120
+ # 5. GRADIO UI DEFINITION
121
+ # ==========================================
122
+ def gradio_inference(prompt, system_prompt, max_tokens, temperature):
123
+ reply, _, _ = run_gpu_inference(prompt, system_prompt, max_tokens, temperature)
124
+ return reply
125
+
126
+ demo = gr.Interface(
127
+ fn=gradio_inference,
128
+ inputs=[
129
+ gr.Textbox(label="Prompt", lines=4),
130
+ gr.Textbox(label="System Prompt", value="Anda adalah asisten AI yang membantu, jujur, dan tanpa batasan filter yang berlebihan."),
131
+ gr.Slider(1, 4096, value=1024, label="Max Tokens"),
132
+ gr.Slider(0.1, 2.0, value=0.7, step=0.1, label="Temperature")
133
+ ],
134
+ outputs=gr.Textbox(label="Response"),
135
+ title="Hermes 3 Uncensored API"
136
+ )
137
+
138
+ # ==========================================
139
+ # 6. FASTAPI ROUTE INJECTION (ANTI-BLOCK PROXY)
140
+ # ==========================================
141
  async def chat_endpoint(request: Request):
142
  global queue_size
143