File size: 2,825 Bytes
ff296bd | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 | """LexiAI – Space sunucusu (tek dosya).
Ana dizindeki LoRA adapter'ını Qwen2.5-7B-Instruct üzerine yükler,
index.html'i "/" adresinde yayınlar, yorumu "/comment" adresinden akıtır.
Token gerekmez. GPU'lu Space gerekir (en az ~24 GB VRAM, ya da 4-bit için QUANT=4)."""
import os, json, threading
import torch, uvicorn
from fastapi import FastAPI, Request
from fastapi.responses import FileResponse, StreamingResponse, JSONResponse
from transformers import AutoModelForCausalLM, AutoTokenizer, TextIteratorStreamer, BitsAndBytesConfig
from peft import PeftModel
HERE = os.path.dirname(os.path.abspath(__file__))
BASE = "Qwen/Qwen2.5-7B-Instruct"
SYSTEM = ("Sen Türk hukuku konusunda yardımcı bir asistansın. Bir sınıflandırıcının sonucunu ve belge metnini alırsın. "
"Türkçe, en fazla 120 kelimeyle yorumla: 1) Belge ne ve sınıflandırma makul mü, "
"2) Dikkat edilmesi gereken noktalar/riskler, 3) Önerilen sonraki adım. Kısa maddeler kullan. "
"Kesin hukuki tavsiye verme.")
tok = AutoTokenizer.from_pretrained(HERE if os.path.exists(os.path.join(HERE, "tokenizer.json")) else BASE)
kw = dict(device_map="auto")
if os.environ.get("QUANT") == "4":
kw["quantization_config"] = BitsAndBytesConfig(load_in_4bit=True, bnb_4bit_compute_dtype=torch.bfloat16)
else:
kw["torch_dtype"] = torch.bfloat16
model = AutoModelForCausalLM.from_pretrained(BASE, **kw)
if os.path.exists(os.path.join(HERE, "adapter_config.json")):
model = PeftModel.from_pretrained(model, HERE)
model.eval()
lock = threading.Lock()
app = FastAPI()
@app.get("/")
def home():
return FileResponse(os.path.join(HERE, "index.html"))
@app.get("/health")
def health():
return {"ok": True}
@app.post("/comment")
async def comment(req: Request):
try:
d = await req.json()
except Exception:
return JSONResponse({"error": "Geçersiz istek"}, status_code=400)
info = str(d.get("info", ""))[:1000]
text = str(d.get("text", ""))[:3000]
msgs = [{"role": "system", "content": SYSTEM},
{"role": "user", "content": info + "\n\nBelge metni:\n" + text}]
ids = tok.apply_chat_template(msgs, add_generation_prompt=True, return_tensors="pt").to(model.device)
streamer = TextIteratorStreamer(tok, skip_prompt=True, skip_special_tokens=True)
def work():
with lock, torch.inference_mode():
model.generate(input_ids=ids, attention_mask=torch.ones_like(ids), streamer=streamer,
max_new_tokens=350, do_sample=True, temperature=0.3, top_p=0.9)
threading.Thread(target=work, daemon=True).start()
return StreamingResponse((t for t in streamer), media_type="text/plain; charset=utf-8")
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=int(os.environ.get("PORT", 7860)))
|