Spaces:
Running
Running
Download app.py from mjpsm/project-creation-api-v2: direct link, hf CLI and curl.
- Browser
- Download file 5.69 kB
-
https://huggingface.co/spaces/mjpsm/project-creation-api-v2/resolve/main/app.py
- Command line
-
hf download hf://spaces/mjpsm/project-creation-api-v2/app.py
-
curl -L -o app.py https://huggingface.co/spaces/mjpsm/project-creation-api-v2/resolve/main/app.py
5.69 kB
| import copy | |
| import os | |
| import re | |
| import threading | |
| import time | |
| from typing import List | |
| import torch | |
| from fastapi import FastAPI | |
| from pydantic import BaseModel, Field | |
| from transformers import AutoTokenizer, AutoModelForCausalLM, DynamicCache | |
| MODEL_ID = os.environ.get("MODEL_ID", "mjpsm/project-creation-qwen2.5-1.5b-sft-v2") | |
| SYSTEM_PROMPT = "You are the MyVillage educational project creator. Given a request that names a student audience (grade or age) and a theme, write exactly one project: one title and one general project description. Format: **Project Title:** <title>, a blank line, **Project Description:**, a blank line, then the paragraphs separated by blank lines. For kindergarten through 5th grade write exactly four paragraphs: a fun introduction where the reader explores the theme with their teammates; the one technology-based creation the team will make and what they will learn; how parents and caregivers can support young creators; and a clear definition of done describing the finished creation. For 6th grade and above, including college, write exactly three paragraphs: a project overview where the reader collaborates with their teammates; the one technology-based creation and the learning involved; and a definition of done describing the finished, modest artifact. Match complexity and tone to the audience. Describe what the team makes, learns, and finishes, not how to build it: no step-by-step instructions, and do not name specific tools, programming languages, platforms, or products. Handle sensitive topics supportively without asking anyone to share private information." | |
| # Generation settings are fixed server-side (no longer request fields). | |
| MAX_NEW_TOKENS = 512 | |
| TEMPERATURE = 0.7 | |
| TOP_P = 0.9 | |
| REPETITION_PENALTY = 1.08 | |
| os.environ["TOKENIZERS_PARALLELISM"] = "false" | |
| # Size the thread pool from an env var (default = the Space's real vCPUs), not os.cpu_count(), | |
| # which reports the host's cores on Hugging Face Spaces. | |
| torch.set_num_threads(int(os.environ.get("TORCH_NUM_THREADS", "2"))) | |
| app = FastAPI(title="MyVillage Project Creation API (V2)", version="2.1.0") | |
| lock = threading.Lock() # keep CPU generations serialized on a small Space | |
| tokenizer = AutoTokenizer.from_pretrained(MODEL_ID) | |
| if tokenizer.pad_token is None: | |
| tokenizer.pad_token = tokenizer.eos_token | |
| model = AutoModelForCausalLM.from_pretrained(MODEL_ID, torch_dtype=torch.float32, low_cpu_mem_usage=True) | |
| model.eval() | |
| EOS_ID = tokenizer.convert_tokens_to_ids("<|im_end|>") | |
| # --- System-prompt KV cache -------------------------------------------------- | |
| # Every request starts with the same system prompt, so its key/value tensors are | |
| # computed once here. Each request deep-copies this cache and only has to prefill | |
| # its own user prompt before decoding. | |
| _SENTINEL = "\x00USER_PROMPT\x00" | |
| _template = tokenizer.apply_chat_template( | |
| [{"role": "system", "content": SYSTEM_PROMPT}, {"role": "user", "content": _SENTINEL}], | |
| tokenize=False, | |
| add_generation_prompt=True, | |
| ) | |
| PREFIX_TEXT, SUFFIX_TEMPLATE = _template.split(_SENTINEL) | |
| PREFIX_IDS = tokenizer(PREFIX_TEXT, return_tensors="pt", add_special_tokens=False).input_ids | |
| _t0 = time.time() | |
| with torch.no_grad(): | |
| PREFIX_CACHE = DynamicCache() | |
| model(input_ids=PREFIX_IDS, past_key_values=PREFIX_CACHE, use_cache=True) | |
| print(f"[startup] cached system prompt: {PREFIX_IDS.shape[-1]} tokens in {time.time() - _t0:.2f}s", flush=True) | |
| class ProjectRequest(BaseModel): | |
| prompt: str = Field(min_length=8, max_length=1500) | |
| class ProjectResponse(BaseModel): | |
| title: str | |
| description: str | |
| paragraphs: List[str] | |
| model: str | |
| def root(): | |
| return {"service": "MyVillage Project Creation API (V2)", "docs": "/docs"} | |
| def health(): | |
| return {"status": "ok", "model": MODEL_ID, "cached_prefix_tokens": int(PREFIX_IDS.shape[-1])} | |
| TITLE_RE = re.compile(r"^\*\*Project Title:\*\*[ \t]+(\S[^\n]*)", re.MULTILINE) | |
| DESC_RE = re.compile(r"\*\*Project Description:\*\*\s*(.*)", re.DOTALL) | |
| def parse_output(text: str): | |
| t = TITLE_RE.search(text) | |
| d = DESC_RE.search(text) | |
| title = t.group(1).strip() if t else "" | |
| description = d.group(1).strip() if d else "" | |
| paragraphs = [p.strip() for p in re.split(r"\n\s*\n", description) if p.strip()] | |
| return title, description, paragraphs | |
| def predict(body: ProjectRequest): | |
| suffix_ids = tokenizer(body.prompt + SUFFIX_TEMPLATE, return_tensors="pt", add_special_tokens=False).input_ids | |
| input_ids = torch.cat([PREFIX_IDS, suffix_ids], dim=-1) | |
| attention_mask = torch.ones_like(input_ids) | |
| with lock, torch.no_grad(): | |
| t0 = time.time() | |
| cache = copy.deepcopy(PREFIX_CACHE) # per-request copy; generate() mutates the cache | |
| ids = model.generate( | |
| input_ids=input_ids, | |
| attention_mask=attention_mask, | |
| past_key_values=cache, | |
| max_new_tokens=MAX_NEW_TOKENS, | |
| do_sample=True, | |
| temperature=TEMPERATURE, | |
| top_p=TOP_P, | |
| repetition_penalty=REPETITION_PENALTY, | |
| eos_token_id=EOS_ID, | |
| pad_token_id=tokenizer.pad_token_id, | |
| ) | |
| new_tokens = ids.shape[-1] - input_ids.shape[-1] | |
| print(f"[predict] prompt={suffix_ids.shape[-1]} tok (+{PREFIX_IDS.shape[-1]} cached), " | |
| f"generated={new_tokens} tok in {time.time() - t0:.1f}s", flush=True) | |
| raw = tokenizer.decode(ids[0][input_ids.shape[-1]:], skip_special_tokens=True).strip() | |
| title, description, paragraphs = parse_output(raw) | |
| return ProjectResponse(title=title, description=description, paragraphs=paragraphs, model=MODEL_ID) | |