missionctrl_env / handler.py
Jit-fnc's picture
Upload 3 files
8723187 verified
Raw History Blame Contribute Delete
3.06 kB
"""Hugging Face Inference Endpoints custom handler for task=`custom`.
Loads Qwen2.5-0.5B-Instruct (or BASE_MODEL env) + LoRA adapter from the model repo.
Supports native `inputs` + `parameters` and optional `messages` payloads.
"""
from __future__ import annotations
import os
from typing import Any, Dict, List
import torch
from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer
BASE_MODEL = os.environ.get("BASE_MODEL", "Qwen/Qwen2.5-0.5B-Instruct")
class EndpointHandler:
"""Called by huggingface_inference_toolkit with model_dir as `path`."""
def __init__(self, path: str = "") -> None:
model_dir = path or os.environ.get("MODEL_DIR", ".")
self.tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)
base = AutoModelForCausalLM.from_pretrained(
BASE_MODEL,
torch_dtype=torch.float16,
device_map="auto",
trust_remote_code=True,
)
self.model = PeftModel.from_pretrained(base, model_dir)
self.model.eval()
self._device = next(self.model.parameters()).device
def _prompt_from_messages(self, messages: List[Dict[str, Any]]) -> str:
return self.tokenizer.apply_chat_template(
messages,
tokenize=False,
add_generation_prompt=True,
)
def __call__(self, data: Dict[str, Any]) -> List[Dict[str, str]]:
if "messages" in data:
prompt = self._prompt_from_messages(data["messages"])
params: Dict[str, Any] = {
k: data[k] for k in ("max_tokens", "temperature", "top_p") if k in data
}
if "max_tokens" in params:
params["max_new_tokens"] = int(params.pop("max_tokens"))
elif "max_new_tokens" not in params:
params["max_new_tokens"] = 120
else:
prompt = str(data.get("inputs") or "")
params = dict(data.get("parameters") or {})
return_full = bool(params.pop("return_full_text", False))
max_new = int(params.get("max_new_tokens", 120))
temperature = float(params.get("temperature", 0.0))
gen_kwargs: Dict[str, Any] = {
"max_new_tokens": max_new,
"temperature": temperature,
"do_sample": temperature > 0.0,
"pad_token_id": self.tokenizer.eos_token_id,
}
if "top_p" in params:
gen_kwargs["top_p"] = float(params["top_p"])
enc = self.tokenizer(prompt, return_tensors="pt")
enc = {k: v.to(self._device) for k, v in enc.items()}
with torch.no_grad():
out = self.model.generate(**enc, **gen_kwargs)
full_text = self.tokenizer.decode(out[0], skip_special_tokens=True)
prompt_text = self.tokenizer.decode(enc["input_ids"][0], skip_special_tokens=True)
if return_full:
generated = full_text
else:
generated = full_text[len(prompt_text) :].strip()
return [{"generated_text": generated}]