Mfusenig commited on
Commit
3eb7dcc
·
verified ·
1 Parent(s): 9012336

Upload handler.py with huggingface_hub

Browse files
Files changed (1) hide show
  1. handler.py +49 -0
handler.py ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ import os
2
+ from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
3
+
4
+ class EndpointHandler:
5
+ def __init__(self, path=""):
6
+ """
7
+ Initializes the handler by loading the T5Gemma model and tokenizer.
8
+
9
+ Args:
10
+ path (str): The path to the model directory. In the Hugging Face
11
+ Inference Endpoint environment, this is automatically
12
+ set to the directory containing your model files.
13
+ """
14
+ # Load the T5Gemma model and tokenizer from the specified path
15
+ self.model = AutoModelForSeq2SeqLM.from_pretrained(path)
16
+ self.tokenizer = AutoTokenizer.from_pretrained(path)
17
+ # Set the model to evaluation mode for inference
18
+ self.model.eval()
19
+
20
+ def __call__(self, data):
21
+ """
22
+ This method is called for each inference request.
23
+
24
+ Args:
25
+ data (dict): A dictionary containing the input data.
26
+ It expects a key "inputs" with the text (or a list of texts) to be processed.
27
+ Additional parameters for generation can also be passed.
28
+
29
+ Returns:
30
+ list: A list of strings with the generated text.
31
+ """
32
+ # Get inputs and ensure it's a list
33
+ inputs_text = data.pop("inputs", [])
34
+ if isinstance(inputs_text, str):
35
+ inputs_text = [inputs_text]
36
+
37
+ # Apply the specific T5Gemma prompt template used during fine-tuning
38
+ templated_inputs = [f"<start_of_turn>user\n{text}<end_of_turn>\n<start_of_turn>model\n" for text in inputs_text]
39
+
40
+ # Tokenize the templated input texts
41
+ # The tokenizer will handle padding for batched inputs.
42
+ inputs = self.tokenizer(templated_inputs, return_tensors="pt", padding=True)
43
+
44
+ # Generate text using the model. Any remaining items in `data` are
45
+ # passed as generation parameters (e.g., max_length, num_beams).
46
+ outputs = self.model.generate(**inputs, **data)
47
+
48
+ # Decode the generated tokens back to text
49
+ return self.tokenizer.batch_decode(outputs, skip_special_tokens=True)