import os import re import threading import torch from fastapi import FastAPI from pydantic import BaseModel, Field from transformers import AutoTokenizer, AutoModelForCausalLM MODEL_ID = os.environ.get("MODEL_ID", "mjpsm/project-creation-qwen2.5-1.5b-sft-v1") SYSTEM_PROMPT = "You are the MyVillage educational project creator. Given a student project request, write one engaging, age-appropriate, feasible project that fits the requested subject and audience. Follow the examples' Markdown structure exactly: **Project Title: ...** followed by **Project Description:** and a detailed, actionable description. Do not add unrelated topics." os.environ["TOKENIZERS_PARALLELISM"] = "false" # os.cpu_count() reports the host's cores, not the container's vCPU limit # (2 on the free Space tier); oversubscribing threads makes generation crawl. NUM_THREADS = int(os.getenv("NUM_THREADS", "2")) torch.set_num_threads(NUM_THREADS) # Generation settings are fixed server-side; clients only send a prompt. TEMPERATURE = 0.7 MAX_NEW_TOKENS = 650 app = FastAPI(title="MyVillage Project Creation API", version="1.0.0") lock = threading.Lock() # keep CPU generations serialized on a small Space tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) if tokenizer.pad_token is None: tokenizer.pad_token = tokenizer.eos_token model = AutoModelForCausalLM.from_pretrained( MODEL_ID, torch_dtype=torch.float32, # conservative and compatible for CPU inference low_cpu_mem_usage=True, ) model.eval() class ProjectRequest(BaseModel): prompt: str = Field(min_length=8, max_length=1500) class ProjectResponse(BaseModel): title: str description: str model: str @app.get("/") def root(): return {"service": "MyVillage Project Creation API", "docs": "/docs"} @app.get("/health") def health(): return {"status": "ok", "model": MODEL_ID} def parse_output(text: str): title_match = re.search(r"\*\*Project Title:\s*(.*?)\*\*", text, flags=re.DOTALL | re.IGNORECASE) desc_match = re.search(r"\*\*Project Description:\*\*\s*(.*)", text, flags=re.DOTALL | re.IGNORECASE) title = title_match.group(1).strip() if title_match else "" description = desc_match.group(1).strip() if desc_match else "" return title, description @app.post("/predict", response_model=ProjectResponse) def predict(body: ProjectRequest): messages = [ {"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": body.prompt}, ] rendered = tokenizer.apply_chat_template(messages, tokenize=False, add_generation_prompt=True) inputs = tokenizer(rendered, return_tensors="pt") with lock, torch.inference_mode(): ids = model.generate( **inputs, max_new_tokens=MAX_NEW_TOKENS, do_sample=True, temperature=TEMPERATURE, top_p=0.9, repetition_penalty=1.08, eos_token_id=tokenizer.convert_tokens_to_ids("<|im_end|>"), pad_token_id=tokenizer.pad_token_id, ) raw = tokenizer.decode(ids[0][inputs["input_ids"].shape[-1]:], skip_special_tokens=True).strip() title, description = parse_output(raw) return ProjectResponse(title=title, description=description, model=MODEL_ID)