import copy import os import re import threading import time from typing import List import torch from fastapi import FastAPI from pydantic import BaseModel, Field from transformers import AutoTokenizer, AutoModelForCausalLM, DynamicCache MODEL_ID = os.environ.get("MODEL_ID", "mjpsm/project-creation-qwen2.5-1.5b-sft-v2") SYSTEM_PROMPT = "You are the MyVillage educational project creator. Given a request that names a student audience (grade or age) and a theme, write exactly one project: one title and one general project description. Format: **Project Title:** , a blank line, **Project Description:**, a blank line, then the paragraphs separated by blank lines. For kindergarten through 5th grade write exactly four paragraphs: a fun introduction where the reader explores the theme with their teammates; the one technology-based creation the team will make and what they will learn; how parents and caregivers can support young creators; and a clear definition of done describing the finished creation. For 6th grade and above, including college, write exactly three paragraphs: a project overview where the reader collaborates with their teammates; the one technology-based creation and the learning involved; and a definition of done describing the finished, modest artifact. Match complexity and tone to the audience. Describe what the team makes, learns, and finishes, not how to build it: no step-by-step instructions, and do not name specific tools, programming languages, platforms, or products. Handle sensitive topics supportively without asking anyone to share private information." # Generation settings are fixed server-side (no longer request fields). MAX_NEW_TOKENS = 512 TEMPERATURE = 0.7 TOP_P = 0.9 REPETITION_PENALTY = 1.08 os.environ["TOKENIZERS_PARALLELISM"] = "false" # Size the thread pool from an env var (default = the Space's real vCPUs), not os.cpu_count(), # which reports the host's cores on Hugging Face Spaces. torch.set_num_threads(int(os.environ.get("TORCH_NUM_THREADS", "2"))) app = FastAPI(title="MyVillage Project Creation API (V2)", version="2.1.0") lock = threading.Lock() # keep CPU generations serialized on a small Space tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) if tokenizer.pad_token is None: tokenizer.pad_token = tokenizer.eos_token model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype=torch.float32, low_cpu_mem_usage=True) model.eval() EOS_ID = tokenizer.convert_tokens_to_ids("<|im_end|>") # --- System-prompt KV cache -------------------------------------------------- # Every request starts with the same system prompt, so its key/value tensors are # computed once here. Each request deep-copies this cache and only has to prefill # its own user prompt before decoding. _SENTINEL = "\x00USER_PROMPT\x00" _template = tokenizer.apply_chat_template( [{"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": _SENTINEL}], tokenize=False, add_generation_prompt=True, ) PREFIX_TEXT, SUFFIX_TEMPLATE = _template.split(_SENTINEL) PREFIX_IDS = tokenizer(PREFIX_TEXT, return_tensors="pt", add_special_tokens=False).input_ids _t0 = time.time() with torch.no_grad(): PREFIX_CACHE = DynamicCache() model(input_ids=PREFIX_IDS, past_key_values=PREFIX_CACHE, use_cache=True) print(f"[startup] cached system prompt: {PREFIX_IDS.shape[-1]} tokens in {time.time() - _t0:.2f}s", flush=True) class ProjectRequest(BaseModel): prompt: str = Field(min_length=8, max_length=1500) class ProjectResponse(BaseModel): title: str description: str paragraphs: List[str] model: str @app.get("/") def root(): return {"service": "MyVillage Project Creation API (V2)", "docs": "/docs"} @app.get("/health") def health(): return {"status": "ok", "model": MODEL_ID, "cached_prefix_tokens": int(PREFIX_IDS.shape[-1])} TITLE_RE = re.compile(r"^\*\*Project Title:\*\*[ \t]+(\S[^\n]*)", re.MULTILINE) DESC_RE = re.compile(r"\*\*Project Description:\*\*\s*(.*)", re.DOTALL) def parse_output(text: str): t = TITLE_RE.search(text) d = DESC_RE.search(text) title = t.group(1).strip() if t else "" description = d.group(1).strip() if d else "" paragraphs = [p.strip() for p in re.split(r"\n\s*\n", description) if p.strip()] return title, description, paragraphs @app.post("/predict", response_model=ProjectResponse) def predict(body: ProjectRequest): suffix_ids = tokenizer(body.prompt + SUFFIX_TEMPLATE, return_tensors="pt", add_special_tokens=False).input_ids input_ids = torch.cat([PREFIX_IDS, suffix_ids], dim=-1) attention_mask = torch.ones_like(input_ids) with lock, torch.no_grad(): t0 = time.time() cache = copy.deepcopy(PREFIX_CACHE) # per-request copy; generate() mutates the cache ids = model.generate( input_ids=input_ids, attention_mask=attention_mask, past_key_values=cache, max_new_tokens=MAX_NEW_TOKENS, do_sample=True, temperature=TEMPERATURE, top_p=TOP_P, repetition_penalty=REPETITION_PENALTY, eos_token_id=EOS_ID, pad_token_id=tokenizer.pad_token_id, ) new_tokens = ids.shape[-1] - input_ids.shape[-1] print(f"[predict] prompt={suffix_ids.shape[-1]} tok (+{PREFIX_IDS.shape[-1]} cached), " f"generated={new_tokens} tok in {time.time() - t0:.1f}s", flush=True) raw = tokenizer.decode(ids[0][input_ids.shape[-1]:], skip_special_tokens=True).strip() title, description, paragraphs = parse_output(raw) return ProjectResponse(title=title, description=description, paragraphs=paragraphs, model=MODEL_ID)