Britania commited on
Commit
c703b48
·
verified ·
1 Parent(s): 2a15982

Update handler.py

Browse files
Files changed (1) hide show
  1. handler.py +26 -22
handler.py CHANGED
@@ -35,28 +35,10 @@ def parse_and_validate_json(output):
35
 
36
 
37
 
38
- def model_inference(model, tokenizer, text, **parameters):
39
- default_parameters = {
40
- "max_new_tokens": 300,
41
- "temperature": 0.5,
42
- "top_p": 0.99,
43
- "top_k": 20,
44
- "repetition_penalty": 1.2
45
- }
46
- default_parameters.update(parameters)
47
- inputs = tokenizer(text, return_tensors="pt").to("cuda:0")
48
- inputs_length = len(inputs["input_ids"][0])
49
- with torch.inference_mode():
50
- outputs = model.generate(**inputs, **default_parameters)
51
- return tokenizer.decode(outputs[0][inputs_length:], skip_special_tokens=True)
52
-
53
-
54
- def model_inference_with_validation(filled_template, **parameters):
55
- while True:
56
- result = model_inference(self.model, self.my_tokenizer, filled_template, **parameters)
57
- is_valid, response_or_error = parse_and_validate_json(result)
58
- if is_valid:
59
- return {'generated_text': [result]}
60
 
61
 
62
 
@@ -70,6 +52,28 @@ class EndpointHandler():
70
  )
71
  self.my_tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True)
72
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
73
  def __call__(self, data: Dict[str, Any]) -> Dict[str, str]:
74
  input_data = data.pop("inputs", data)
75
  parameters = data.pop("parameters", {})
 
35
 
36
 
37
 
38
+
39
+
40
+
41
+
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
42
 
43
 
44
 
 
52
  )
53
  self.my_tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True)
54
 
55
+ def model_inference(model, tokenizer, text, **parameters):
56
+ default_parameters = {
57
+ "max_new_tokens": 300,
58
+ "temperature": 0.5,
59
+ "top_p": 0.99,
60
+ "top_k": 20,
61
+ "repetition_penalty": 1.2
62
+ }
63
+ default_parameters.update(parameters)
64
+ inputs = tokenizer(text, return_tensors="pt").to("cuda:0")
65
+ inputs_length = len(inputs["input_ids"][0])
66
+ with torch.inference_mode():
67
+ outputs = model.generate(**inputs, **default_parameters)
68
+ return tokenizer.decode(outputs[0][inputs_length:], skip_special_tokens=True)
69
+
70
+ def model_inference_with_validation(filled_template, **parameters):
71
+ while True:
72
+ result = model_inference(self.model, self.my_tokenizer, filled_template, **parameters)
73
+ is_valid, response_or_error = parse_and_validate_json(result)
74
+ if is_valid:
75
+ return {'generated_text': [result]}
76
+
77
  def __call__(self, data: Dict[str, Any]) -> Dict[str, str]:
78
  input_data = data.pop("inputs", data)
79
  parameters = data.pop("parameters", {})