"""Hugging Face Inference Endpoints custom handler for task=`custom`. Loads Qwen2.5-0.5B-Instruct (or BASE_MODEL env) + LoRA adapter from the model repo. Supports native `inputs` + `parameters` and optional `messages` payloads. """ from __future__ import annotations import os from typing import Any, Dict, List import torch from peft import PeftModel from transformers import AutoModelForCausalLM, AutoTokenizer BASE_MODEL = os.environ.get("BASE_MODEL", "Qwen/Qwen2.5-0.5B-Instruct") class EndpointHandler: """Called by huggingface_inference_toolkit with model_dir as `path`.""" def __init__(self, path: str = "") -> None: model_dir = path or os.environ.get("MODEL_DIR", ".") self.tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True) base = AutoModelForCausalLM.from_pretrained( BASE_MODEL, torch_dtype=torch.float16, device_map="auto", trust_remote_code=True, ) self.model = PeftModel.from_pretrained(base, model_dir) self.model.eval() self._device = next(self.model.parameters()).device def _prompt_from_messages(self, messages: List[Dict[str, Any]]) -> str: return self.tokenizer.apply_chat_template( messages, tokenize=False, add_generation_prompt=True, ) def __call__(self, data: Dict[str, Any]) -> List[Dict[str, str]]: if "messages" in data: prompt = self._prompt_from_messages(data["messages"]) params: Dict[str, Any] = { k: data[k] for k in ("max_tokens", "temperature", "top_p") if k in data } if "max_tokens" in params: params["max_new_tokens"] = int(params.pop("max_tokens")) elif "max_new_tokens" not in params: params["max_new_tokens"] = 120 else: prompt = str(data.get("inputs") or "") params = dict(data.get("parameters") or {}) return_full = bool(params.pop("return_full_text", False)) max_new = int(params.get("max_new_tokens", 120)) temperature = float(params.get("temperature", 0.0)) gen_kwargs: Dict[str, Any] = { "max_new_tokens": max_new, "temperature": temperature, "do_sample": temperature > 0.0, "pad_token_id": self.tokenizer.eos_token_id, } if "top_p" in params: gen_kwargs["top_p"] = float(params["top_p"]) enc = self.tokenizer(prompt, return_tensors="pt") enc = {k: v.to(self._device) for k, v in enc.items()} with torch.no_grad(): out = self.model.generate(**enc, **gen_kwargs) full_text = self.tokenizer.decode(out[0], skip_special_tokens=True) prompt_text = self.tokenizer.decode(enc["input_ids"][0], skip_special_tokens=True) if return_full: generated = full_text else: generated = full_text[len(prompt_text) :].strip() return [{"generated_text": generated}]