devankit7873 commited on
Commit
d9b5dc9
·
verified ·
1 Parent(s): 3abd7fd

Upload handler.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. handler.py +181 -0
handler.py ADDED
@@ -0,0 +1,181 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ import re
3
+ from typing import Dict, List, Any, Union
4
+ import torch
5
+ import torch.nn as nn
6
+ import torch.nn.functional as F
7
+ from transformers import AutoTokenizer, AutoModelForCausalLM
8
+ from safetensors.torch import load_file
9
+
10
+
11
+ class DecisionHead(nn.Module):
12
+ def __init__(self, hidden_size: int = 1536, intermediate_size: int = 512, max_choices: int = 10, dropout: float = 0.1):
13
+ super().__init__()
14
+ self.max_choices = max_choices
15
+ self.norm = nn.LayerNorm(hidden_size)
16
+ self.dense = nn.Linear(hidden_size, intermediate_size)
17
+ self.act = nn.GELU()
18
+ self.dropout = nn.Dropout(dropout)
19
+ self.out_proj = nn.Linear(intermediate_size, max_choices)
20
+ self.score_proj = nn.Linear(intermediate_size, 1)
21
+
22
+ def forward(self, x: torch.Tensor, num_choices: torch.Tensor = None) -> torch.Tensor:
23
+ h = self.norm(x)
24
+ h = self.act(self.dense(h))
25
+ h = self.dropout(h)
26
+ logits = self.out_proj(h)
27
+ if num_choices is not None:
28
+ batch_size = logits.size(0)
29
+ seq_range = torch.arange(self.max_choices, device=logits.device).unsqueeze(0).expand(batch_size, -1)
30
+ mask = seq_range >= num_choices.unsqueeze(1)
31
+ logits = logits.masked_fill(mask, -1e4)
32
+ return logits
33
+
34
+
35
+ class EndpointHandler:
36
+ """
37
+ Hugging Face Inference Endpoints Custom Handler for QJev-1.5B.
38
+ Accepts:
39
+ {
40
+ "inputs": "Situation or context description",
41
+ "parameters": {
42
+ "choices": ["Choice 1", "Choice 2", ...]
43
+ }
44
+ }
45
+ OR:
46
+ {
47
+ "context": "...",
48
+ "question": "...",
49
+ "choices": [...]
50
+ }
51
+ Returns standard text-classification scores:
52
+ [
53
+ {"label": "Choice 1", "score": 0.8672},
54
+ {"label": "Choice 2", "score": 0.0703},
55
+ ...
56
+ ]
57
+ """
58
+ def __init__(self, path: str = ""):
59
+ self.device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
60
+ dtype = torch.bfloat16 if torch.cuda.is_available() and torch.cuda.is_bf16_supported() else torch.float32
61
+
62
+ # 1. Load Tokenizer
63
+ self.tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True)
64
+ if self.tokenizer.pad_token is None:
65
+ self.tokenizer.pad_token = self.tokenizer.eos_token
66
+
67
+ # 2. Load Base Transformer Backbone
68
+ self.backbone = AutoModelForCausalLM.from_pretrained(
69
+ path,
70
+ torch_dtype=dtype,
71
+ device_map=self.device
72
+ )
73
+ self.backbone.eval()
74
+
75
+ # 3. Load MLP Decision Head
76
+ self.head = DecisionHead().to(device=self.device, dtype=dtype)
77
+ head_path = os.path.join(path, "decision_head.safetensors")
78
+ if not os.path.exists(head_path):
79
+ head_path = os.path.join(path, "decision_head.pt")
80
+
81
+ if os.path.exists(head_path):
82
+ if head_path.endswith(".safetensors"):
83
+ state = load_file(head_path)
84
+ else:
85
+ ckpt = torch.load(head_path, map_location="cpu")
86
+ state = ckpt.get("head_state_dict", ckpt)
87
+ self.head.load_state_dict(state)
88
+ self.head.eval()
89
+
90
+ def _extract_choices_from_text(self, text: str) -> List[str]:
91
+ # Try extracting patterns like "[1] ...", "1. ...", "Option A: ..."
92
+ lines = [line.strip() for line in text.split("\n") if line.strip()]
93
+ choices = []
94
+ for line in lines:
95
+ m = re.match(r"^(?:\[?\d+\]?[\.\)]?|[A-E][\.\)]|Option\s+[A-E]:?)\s*(.+)$", line, re.IGNORECASE)
96
+ if m:
97
+ choices.append(m.group(1).strip())
98
+ if len(choices) >= 2:
99
+ return choices[:10]
100
+ return ["Approve / Yes", "Reject / No"]
101
+
102
+ def _format_prompt(self, context: str, question: str, choices: List[str]) -> str:
103
+ choices_str = "\n".join([f"[{i+1}] {c}" for i, c in enumerate(choices)])
104
+ q_str = f"\n\nQuestion:\n{question.strip()}" if question else "\n\nQuestion:\nSelect the most appropriate option:"
105
+ return (
106
+ "<|im_start|>system\n"
107
+ "You are an expert decision judge. Evaluate the situation and determine the likelihood for each option.<|im_end|>\n"
108
+ f"<|im_start|>user\nContext:\n{context.strip()}{q_str}\n\nOptions:\n{choices_str}<|im_end|>\n"
109
+ "<|im_start|>assistant\nDecision:"
110
+ )
111
+
112
+ def _process_single(self, item: Dict[str, Any]) -> List[Dict[str, float]]:
113
+ # Parse inputs
114
+ context = ""
115
+ question = ""
116
+ choices = []
117
+
118
+ if isinstance(item, str):
119
+ context = item
120
+ choices = self._extract_choices_from_text(item)
121
+ elif isinstance(item, dict):
122
+ # Check standard Hugging Face inputs/parameters format
123
+ if "inputs" in item:
124
+ context = str(item.get("inputs", ""))
125
+ params = item.get("parameters", {}) or {}
126
+ if isinstance(params, dict):
127
+ choices = params.get("choices", [])
128
+ question = params.get("question", "")
129
+
130
+ # Check alternative keys
131
+ if not context and "context" in item:
132
+ context = str(item.get("context", ""))
133
+ if not question and "question" in item:
134
+ question = str(item.get("question", ""))
135
+ if not choices and "choices" in item:
136
+ choices = item.get("choices", [])
137
+ if not choices and "options" in item:
138
+ choices = item.get("options", [])
139
+
140
+ # Fallback if no choices provided
141
+ if not choices:
142
+ choices = self._extract_choices_from_text(context)
143
+
144
+ # Enforce max 10 choices (architecture limit)
145
+ choices = [str(c) for c in choices][:10]
146
+ if len(choices) < 2:
147
+ choices = ["Approve / Yes", "Reject / No"]
148
+
149
+ num_c = len(choices)
150
+ prompt = self._format_prompt(context, question, choices)
151
+ inputs = self.tokenizer(prompt, return_tensors="pt").to(self.device)
152
+
153
+ with torch.no_grad():
154
+ transformer = getattr(self.backbone, "model", self.backbone)
155
+ out = transformer(inputs["input_ids"], attention_mask=inputs["attention_mask"])
156
+ last_hidden = out.last_hidden_state[:, -1, :]
157
+ logits = self.head(last_hidden, num_choices=torch.tensor([num_c], device=self.device))
158
+ probs = F.softmax(logits[0, :num_c], dim=-1).cpu().tolist()
159
+
160
+ return [{"label": choice, "score": round(prob, 4)} for choice, prob in zip(choices, probs)]
161
+
162
+ def __call__(self, data: Union[Dict[str, Any], List[Dict[str, Any]]]) -> Union[List[Dict[str, Any]], List[List[Dict[str, Any]]]]:
163
+ """
164
+ Receives request payload, returns predictions.
165
+ """
166
+ if isinstance(data, list):
167
+ return [self._process_single(item) for item in data]
168
+ elif isinstance(data, dict):
169
+ # If payload is {"inputs": ["str1", "str2"], ...}
170
+ if isinstance(data.get("inputs"), list):
171
+ results = []
172
+ for inp in data["inputs"]:
173
+ single_data = dict(data)
174
+ single_data["inputs"] = inp
175
+ results.append(self._process_single(single_data))
176
+ return results
177
+ return self._process_single(data)
178
+ elif isinstance(data, str):
179
+ return self._process_single(data)
180
+ else:
181
+ raise ValueError(f"Unsupported payload type: {type(data)}")