File size: 3,063 Bytes
8723187
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
"""Hugging Face Inference Endpoints custom handler for task=`custom`.

Loads Qwen2.5-0.5B-Instruct (or BASE_MODEL env) + LoRA adapter from the model repo.
Supports native `inputs` + `parameters` and optional `messages` payloads.
"""

from __future__ import annotations

import os
from typing import Any, Dict, List

import torch
from peft import PeftModel
from transformers import AutoModelForCausalLM, AutoTokenizer

BASE_MODEL = os.environ.get("BASE_MODEL", "Qwen/Qwen2.5-0.5B-Instruct")


class EndpointHandler:
    """Called by huggingface_inference_toolkit with model_dir as `path`."""

    def __init__(self, path: str = "") -> None:
        model_dir = path or os.environ.get("MODEL_DIR", ".")
        self.tokenizer = AutoTokenizer.from_pretrained(model_dir, trust_remote_code=True)
        base = AutoModelForCausalLM.from_pretrained(
            BASE_MODEL,
            torch_dtype=torch.float16,
            device_map="auto",
            trust_remote_code=True,
        )
        self.model = PeftModel.from_pretrained(base, model_dir)
        self.model.eval()
        self._device = next(self.model.parameters()).device

    def _prompt_from_messages(self, messages: List[Dict[str, Any]]) -> str:
        return self.tokenizer.apply_chat_template(
            messages,
            tokenize=False,
            add_generation_prompt=True,
        )

    def __call__(self, data: Dict[str, Any]) -> List[Dict[str, str]]:
        if "messages" in data:
            prompt = self._prompt_from_messages(data["messages"])
            params: Dict[str, Any] = {
                k: data[k] for k in ("max_tokens", "temperature", "top_p") if k in data
            }
            if "max_tokens" in params:
                params["max_new_tokens"] = int(params.pop("max_tokens"))
            elif "max_new_tokens" not in params:
                params["max_new_tokens"] = 120
        else:
            prompt = str(data.get("inputs") or "")
            params = dict(data.get("parameters") or {})

        return_full = bool(params.pop("return_full_text", False))

        max_new = int(params.get("max_new_tokens", 120))
        temperature = float(params.get("temperature", 0.0))
        gen_kwargs: Dict[str, Any] = {
            "max_new_tokens": max_new,
            "temperature": temperature,
            "do_sample": temperature > 0.0,
            "pad_token_id": self.tokenizer.eos_token_id,
        }
        if "top_p" in params:
            gen_kwargs["top_p"] = float(params["top_p"])

        enc = self.tokenizer(prompt, return_tensors="pt")
        enc = {k: v.to(self._device) for k, v in enc.items()}

        with torch.no_grad():
            out = self.model.generate(**enc, **gen_kwargs)

        full_text = self.tokenizer.decode(out[0], skip_special_tokens=True)
        prompt_text = self.tokenizer.decode(enc["input_ids"][0], skip_special_tokens=True)

        if return_full:
            generated = full_text
        else:
            generated = full_text[len(prompt_text) :].strip()

        return [{"generated_text": generated}]