zakerytclarke commited on
Commit
a1f3d0f
·
verified ·
1 Parent(s): 8a4e9e9

Update handler.py

Browse files
Files changed (1) hide show
  1. handler.py +19 -11
handler.py CHANGED
@@ -7,8 +7,12 @@ MAX_INPUT_TOKENS = 512
7
 
8
  class EndpointHandler:
9
  def __init__(self, path: str = ""):
10
- # EXACTLY like your snippet: load by repo id from Hugging Face Hub
11
- self.tokenizer = AutoTokenizer.from_pretrained(MODEL_NAME)
 
 
 
 
12
  self.model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)
13
 
14
  self.model.eval()
@@ -26,12 +30,11 @@ class EndpointHandler:
26
  @torch.inference_mode()
27
  def __call__(self, data):
28
  inputs = data.get("inputs")
 
29
  if inputs is None:
30
- raise ValueError("Missing required field 'inputs'.")
31
 
32
- # Support:
33
- # 1) inputs is a full prompt string (closest to your snippet)
34
- # 2) inputs is {"context": ..., "question": ...}
35
  if isinstance(inputs, str):
36
  prompt = inputs
37
  elif isinstance(inputs, dict):
@@ -39,14 +42,15 @@ class EndpointHandler:
39
  question = inputs.get("question", "")
40
  prompt = f"{context}\n{self.system_prompt}\n{question}\n"
41
  else:
42
- raise ValueError("inputs must be a string OR an object with 'context' and 'question'.")
43
 
44
- # Match your code path
45
  enc = self.tokenizer(prompt, return_tensors="pt")
 
46
  input_ids = enc["input_ids"]
47
  attention_mask = enc["attention_mask"]
48
 
49
- # Keep only the most recent tokens if too long
50
  if input_ids.shape[1] > MAX_INPUT_TOKENS:
51
  input_ids = input_ids[:, -MAX_INPUT_TOKENS:]
52
  attention_mask = attention_mask[:, -MAX_INPUT_TOKENS:]
@@ -54,11 +58,15 @@ class EndpointHandler:
54
  input_ids = input_ids.to(self.device)
55
  attention_mask = attention_mask.to(self.device)
56
 
 
57
  outputs = self.model.generate(
58
  input_ids=input_ids,
59
  attention_mask=attention_mask,
60
- do_sample=False,
61
  )
62
 
63
  answer = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
64
- return {"generated_text": answer}
 
 
 
 
7
 
8
  class EndpointHandler:
9
  def __init__(self, path: str = ""):
10
+ # EXACT same as your snippet BUT force slow tokenizer
11
+ # This prevents the fast tokenizer crash from extra_special_tokens list
12
+ self.tokenizer = AutoTokenizer.from_pretrained(
13
+ MODEL_NAME,
14
+ use_fast=False
15
+ )
16
  self.model = AutoModelForSeq2SeqLM.from_pretrained(MODEL_NAME)
17
 
18
  self.model.eval()
 
30
  @torch.inference_mode()
31
  def __call__(self, data):
32
  inputs = data.get("inputs")
33
+
34
  if inputs is None:
35
+ raise ValueError("Missing 'inputs' field")
36
 
37
+ # Match your ask() behavior
 
 
38
  if isinstance(inputs, str):
39
  prompt = inputs
40
  elif isinstance(inputs, dict):
 
42
  question = inputs.get("question", "")
43
  prompt = f"{context}\n{self.system_prompt}\n{question}\n"
44
  else:
45
+ raise ValueError("inputs must be a string or dict")
46
 
47
+ # EXACT tokenizer call like your code
48
  enc = self.tokenizer(prompt, return_tensors="pt")
49
+
50
  input_ids = enc["input_ids"]
51
  attention_mask = enc["attention_mask"]
52
 
53
+ # NEW requirement: keep most recent 512 tokens
54
  if input_ids.shape[1] > MAX_INPUT_TOKENS:
55
  input_ids = input_ids[:, -MAX_INPUT_TOKENS:]
56
  attention_mask = attention_mask[:, -MAX_INPUT_TOKENS:]
 
58
  input_ids = input_ids.to(self.device)
59
  attention_mask = attention_mask.to(self.device)
60
 
61
+ # EXACT generation settings from your snippet
62
  outputs = self.model.generate(
63
  input_ids=input_ids,
64
  attention_mask=attention_mask,
65
+ do_sample=False
66
  )
67
 
68
  answer = self.tokenizer.decode(outputs[0], skip_special_tokens=True)
69
+
70
+ return {
71
+ "generated_text": answer
72
+ }