Spaces:
Running
Running
File size: 3,245 Bytes
d6083a7 7494838 d6083a7 7494838 d6083a7 7494838 d6083a7 1ae96b9 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 |
import os
import re
import threading
import torch
from fastapi import FastAPI
from pydantic import BaseModel, Field
from transformers import AutoTokenizer, AutoModelForCausalLM
MODEL_ID = os.environ.get("MODEL_ID", "mjpsm/project-creation-qwen2.5-1.5b-sft-v1")
SYSTEM_PROMPT = "You are the MyVillage educational project creator. Given a student project request, write one engaging, age-appropriate, feasible project that fits the requested subject and audience. Follow the examples' Markdown structure exactly: **Project Title: ...** followed by **Project Description:** and a detailed, actionable description. Do not add unrelated topics."
os.environ["TOKENIZERS_PARALLELISM"] = "false"
# os.cpu_count() reports the host's cores, not the container's vCPU limit
# (2 on the free Space tier); oversubscribing threads makes generation crawl.
NUM_THREADS = int(os.getenv("NUM_THREADS", "2"))
torch.set_num_threads(NUM_THREADS)
# Generation settings are fixed server-side; clients only send a prompt.
TEMPERATURE = 0.7
MAX_NEW_TOKENS = 650
app = FastAPI(title="MyVillage Project Creation API", version="1.0.0")
lock = threading.Lock() # keep CPU generations serialized on a small Space
tokenizer = AutoTokenizer.from_pretrained(MODEL_ID)
if tokenizer.pad_token is None:
tokenizer.pad_token = tokenizer.eos_token
model = AutoModelForCausalLM.from_pretrained(
MODEL_ID,
torch_dtype=torch.float32, # conservative and compatible for CPU inference
low_cpu_mem_usage=True,
)
model.eval()
class ProjectRequest(BaseModel):
prompt: str = Field(min_length=8, max_length=1500)
class ProjectResponse(BaseModel):
title: str
description: str
model: str
@app.get("/")
def root():
return {"service": "MyVillage Project Creation API", "docs": "/docs"}
@app.get("/health")
def health():
return {"status": "ok", "model": MODEL_ID}
def parse_output(text: str):
title_match = re.search(r"\*\*Project Title:\s*(.*?)\*\*", text, flags=re.DOTALL | re.IGNORECASE)
desc_match = re.search(r"\*\*Project Description:\*\*\s*(.*)", text, flags=re.DOTALL | re.IGNORECASE)
title = title_match.group(1).strip() if title_match else ""
description = desc_match.group(1).strip() if desc_match else ""
return title, description
@app.post("/predict", response_model=ProjectResponse)
def predict(body: ProjectRequest):
messages = [
{"role": "system", "content": SYSTEM_PROMPT},
{"role": "user", "content": body.prompt},
]
rendered = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True)
inputs = tokenizer(rendered, return_tensors="pt")
with lock, torch.inference_mode():
ids = model.generate(
**inputs,
max_new_tokens=MAX_NEW_TOKENS,
do_sample=True,
temperature=TEMPERATURE,
top_p=0.9,
repetition_penalty=1.08,
eos_token_id=tokenizer.convert_tokens_to_ids("<|im_end|>"),
pad_token_id=tokenizer.pad_token_id,
)
raw = tokenizer.decode(ids[0][inputs["input_ids"].shape[-1]:], skip_special_tokens=True).strip()
title, description = parse_output(raw)
return ProjectResponse(title=title, description=description, model=MODEL_ID)
|