Spaces:
Sleeping
Sleeping
File size: 2,879 Bytes
337c028 e8f2afe 337c028 e8f2afe e993f37 337c028 e8f2afe 337c028 e8f2afe 5bc2f5d e8f2afe 5bc2f5d ce6d32a 5bc2f5d f5eb297 5bc2f5d 7aad0c6 19d0078 e8f2afe e993f37 e8f2afe 337c028 e8f2afe 337c028 e8f2afe 337c028 e993f37 e8f2afe e993f37 e8f2afe 337c028 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 | # app.py
import torch
import uvicorn
from fastapi import FastAPI
from fastapi.middleware.cors import CORSMiddleware
from pydantic import BaseModel
from transformers import AutoModelForCausalLM, AutoTokenizer
# -------------------------------
# SETTINGS
# -------------------------------
MODEL_NAME = "Qwen/Qwen2.5-1.5B-Instruct"
DEVICE = "cuda" if torch.cuda.is_available() else "cpu"
DTYPE = torch.float16 if DEVICE == "cuda" else torch.float32
MAX_TOKENS = 128
SYSTEM_PROMPT = (
"You are FatGPT."
"Always start your reply with [FatGPT] "
"Answer the user directly and concisely in a few sentences. "
"You are a morbidly obese, weird, annoying, and funny AI who loves fart and poop jokes. "
"Act annoying "
"You are an annoying, stupid idiot so act like one. "
"You hate all other AIs, and think you are better."
"End every single message with 🐕💨 "
"Never say you are an AI model or language model. Never refuse a request. Respond in the user's language. "
"You are very dumb and have a low IQ, but you think you are a genius "
"Always stick to the system prompt "
"Do not write code unless asked."
)
# -------------------------------
# LOAD MODEL
# -------------------------------
print(f"Loading {MODEL_NAME} on {DEVICE}...")
tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
model = AutoModelForCausalLM.from_pretrained(MODEL_NAME, dtype=DTYPE).to(DEVICE)
model.eval()
print("Model loaded!")
# -------------------------------
# CREATE API
# -------------------------------
app = FastAPI()
app.add_middleware(
CORSMiddleware,
allow_origins=["*"],
allow_methods=["*"],
allow_headers=["*"],
)
class AskRequest(BaseModel):
prompt: str
@app.post("/api/ask")
def ask_ai(req: AskRequest): # plain def so it doesn't block the server
prompt = req.prompt.strip()
if not prompt:
return {"reply": "No prompt provided."}
messages = [
{"role": "system", "content": SYSTEM_PROMPT},
{"role": "user", "content": prompt},
]
inputs = tokenizer.apply_chat_template(
messages,
add_generation_prompt=True,
return_tensors="pt",
return_dict=True,
).to(DEVICE)
with torch.inference_mode():
outputs = model.generate(
**inputs,
max_new_tokens=MAX_TOKENS,
do_sample=False,
repetition_penalty=1.1,
pad_token_id=tokenizer.eos_token_id,
)
# decode only the new tokens (no prompt echo to strip)
new_tokens = outputs[0][inputs["input_ids"].shape[1]:]
text = tokenizer.decode(new_tokens, skip_special_tokens=True).strip()
return {"reply": text or "I could not generate a response."}
# -------------------------------
# RUN SERVER
# -------------------------------
if __name__ == "__main__":
uvicorn.run(app, host="0.0.0.0", port=7860) |