Britania commited on
Commit
9684946
·
verified ·
1 Parent(s): 9cc281f

Update handler.py

Browse files
Files changed (1) hide show
  1. handler.py +14 -9
handler.py CHANGED
@@ -4,8 +4,7 @@ import torch
4
  import json
5
 
6
  def model_inference(model, tokenizer, text, max_new_tokens, temperature, top_p=0.99, top_k=20, repetition_penalty=1.2):
7
- device = "cuda:0"
8
- inputs = tokenizer(text, return_tensors="pt").to(device)
9
  inputs_length = len(inputs["input_ids"][0])
10
  with torch.inference_mode():
11
  outputs = model.generate(**inputs,
@@ -20,17 +19,16 @@ def model_inference(model, tokenizer, text, max_new_tokens, temperature, top_p=0
20
 
21
  class EndpointHandler():
22
  def __init__(self, path=""):
23
- # Carga el modelo y el tokenizer
24
- self.my_model_name = "Britania/m2"
25
- self.fine_tuned_model = AutoModelForCausalLM.from_pretrained(
26
- self.my_model_name,
27
  trust_remote_code=True,
28
  torch_dtype=torch.bfloat16,
29
  device_map="auto"
30
  )
31
- self.my_tokenizer = AutoTokenizer.from_pretrained(self.my_model_name, trust_remote_code=True)
32
 
33
  def __call__(self, data):
 
34
  input_data = data
35
  template = """
36
  [INST] <<SYS>>
@@ -38,9 +36,16 @@ class EndpointHandler():
38
  <</SYS>>
39
  {data} [/INST]
40
  """.strip()
 
41
  data_text = json.dumps(input_data, indent=4)
42
  filled_template = template.replace("{data}", data_text)
 
 
43
  max_new_tokens = 1600
44
- temperature = 0.000001
45
- result = model_inference(self.fine_tuned_model, self.my_tokenizer, filled_template, max_new_tokens, temperature)
 
 
 
 
46
  return [{"generated_text": result}]
 
4
  import json
5
 
6
  def model_inference(model, tokenizer, text, max_new_tokens, temperature, top_p=0.99, top_k=20, repetition_penalty=1.2):
7
+ inputs = tokenizer(text, return_tensors="pt").to("cuda:0")
 
8
  inputs_length = len(inputs["input_ids"][0])
9
  with torch.inference_mode():
10
  outputs = model.generate(**inputs,
 
19
 
20
  class EndpointHandler():
21
  def __init__(self, path=""):
22
+ self.model = AutoModelForCausalLM.from_pretrained(
23
+ path,
 
 
24
  trust_remote_code=True,
25
  torch_dtype=torch.bfloat16,
26
  device_map="auto"
27
  )
28
+ self.my_tokenizer = AutoTokenizer.from_pretrained(path, trust_remote_code=True)
29
 
30
  def __call__(self, data):
31
+ print(data)
32
  input_data = data
33
  template = """
34
  [INST] <<SYS>>
 
36
  <</SYS>>
37
  {data} [/INST]
38
  """.strip()
39
+
40
  data_text = json.dumps(input_data, indent=4)
41
  filled_template = template.replace("{data}", data_text)
42
+ print("input:")
43
+ print(filled_template)
44
  max_new_tokens = 1600
45
+ temperature = 0.5
46
+
47
+ result = model_inference(self.model, self.my_tokenizer, filled_template, max_new_tokens, temperature)
48
+ print("output:")
49
+ print(result)
50
+
51
  return [{"generated_text": result}]