GitHub Actions Claude Sonnet 4.6 commited on
Commit ·
cf80ac9
1
Parent(s): 688a2dc
fix: strip token_type_ids from generate kwargs for MiniCPM5-1B
Browse filesMiniCPM5-1B's generate() rejects token_type_ids even though some
tokenizer configs emit it. Use a denylist so input_ids/attention_mask
always pass through. Also switch torch_dtype -> dtype (4.45+ API).
Co-Authored-By: Claude Sonnet 4.6 <noreply@anthropic.com>
hearthnet/services/llm/backends/hf_local.py
CHANGED
|
@@ -118,7 +118,7 @@ class HfLocalBackend:
|
|
| 118 |
)
|
| 119 |
self._model = AutoModelForCausalLM.from_pretrained(
|
| 120 |
self._model_name,
|
| 121 |
-
|
| 122 |
trust_remote_code=True,
|
| 123 |
)
|
| 124 |
if target_device != "cpu":
|
|
@@ -165,8 +165,15 @@ class HfLocalBackend:
|
|
| 165 |
)
|
| 166 |
|
| 167 |
device = next(self._model.parameters()).device
|
| 168 |
-
|
| 169 |
-
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 170 |
|
| 171 |
streamer = TextIteratorStreamer(
|
| 172 |
tokenizer,
|
|
|
|
| 118 |
)
|
| 119 |
self._model = AutoModelForCausalLM.from_pretrained(
|
| 120 |
self._model_name,
|
| 121 |
+
dtype=dtype,
|
| 122 |
trust_remote_code=True,
|
| 123 |
)
|
| 124 |
if target_device != "cpu":
|
|
|
|
| 165 |
)
|
| 166 |
|
| 167 |
device = next(self._model.parameters()).device
|
| 168 |
+
raw_inputs = tokenizer([prompt_text], return_tensors="pt")
|
| 169 |
+
# token_type_ids is emitted by some tokenizers but rejected by causal LMs.
|
| 170 |
+
# Use a denylist (not allowlist) so input_ids/attention_mask always pass through.
|
| 171 |
+
_STRIP = {"token_type_ids"}
|
| 172 |
+
model_inputs = {
|
| 173 |
+
k: v.to(device)
|
| 174 |
+
for k, v in raw_inputs.items()
|
| 175 |
+
if k not in _STRIP
|
| 176 |
+
}
|
| 177 |
|
| 178 |
streamer = TextIteratorStreamer(
|
| 179 |
tokenizer,
|