mjpsm's picture
Remove raw_output from /predict response
1ae96b9
Raw History Blame Contribute Delete
3.25 kB
import os
import re
import threading
import torch
from fastapi import FastAPI
from pydantic import BaseModel, Field
from transformers import AutoTokenizer, AutoModelForCausalLM
MODEL_ID = os.environ.get("MODEL_ID", "mjpsm/project-creation-qwen2.5-1.5b-sft-v1")
SYSTEM_PROMPT = "You are the MyVillage educational project creator. Given a student project request, write one engaging, age-appropriate, feasible project that fits the requested subject and audience. Follow the examples' Markdown structure exactly: **Project Title: ...** followed by **Project Description:** and a detailed, actionable description. Do not add unrelated topics."
os.environ["TOKENIZERS_PARALLELISM"] = "false"
# os.cpu_count() reports the host's cores, not the container's vCPU limit
# (2 on the free Space tier); oversubscribing threads makes generation crawl.
NUM_THREADS = int(os.getenv("NUM_THREADS", "2"))
torch.set_num_threads(NUM_THREADS)
# Generation settings are fixed server-side; clients only send a prompt.
TEMPERATURE = 0.7
MAX_NEW_TOKENS = 650
app = FastAPI(title="MyVillage Project Creation API", version="1.0.0")
lock = threading.Lock() # keep CPU generations serialized on a small Space
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token
model = AutoModelForCausalLM.from_pretrained(
MODEL_ID,
torch_dtype=torch.float32, # conservative and compatible for CPU inference
low_cpu_mem_usage=True,
)
model.eval()
class ProjectRequest(BaseModel):
prompt: str = Field(min_length=8, max_length=1500)
class ProjectResponse(BaseModel):
title: str
description: str
model: str
@app.get("/")
def root():
return {"service": "MyVillage Project Creation API", "docs": "/docs"}
@app.get("/health")
def health():
return {"status": "ok", "model": MODEL_ID}
def parse_output(text: str):
title_match = re.search(r"\*\*Project Title:\s*(.*?)\*\*", text, flags=re.DOTALL | re.IGNORECASE)
desc_match = re.search(r"\*\*Project Description:\*\*\s*(.*)", text, flags=re.DOTALL | re.IGNORECASE)
title = title_match.group(1).strip() if title_match else ""
description = desc_match.group(1).strip() if desc_match else ""
return title, description
@app.post("/predict", response_model=ProjectResponse)
def predict(body: ProjectRequest):
messages = [
{"role": "system", "content": SYSTEM_PROMPT},
{"role": "user", "content": body.prompt},
]
rendered = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
inputs = tokenizer(rendered, return_tensors="pt")
with lock, torch.inference_mode():
ids = model.generate(
**inputs,
max_new_tokens=MAX_NEW_TOKENS,
do_sample=True,
temperature=TEMPERATURE,
top_p=0.9,
repetition_penalty=1.08,
eos_token_id=tokenizer.convert_tokens_to_ids("<|im_end|>"),
pad_token_id=tokenizer.pad_token_id,
)
raw = tokenizer.decode(ids[0][inputs["input_ids"].shape[-1]:], skip_special_tokens=True).strip()
title, description = parse_output(raw)
return ProjectResponse(title=title, description=description, model=MODEL_ID)