capstone_t5 / handler.py
Hayleyy2024's picture lihan1999's picture
Create handler.py (#3)
5825634 verified
Raw History Blame Contribute Delete
2.42 kB
from transformers import AutoModelForSeq2SeqLM, AutoTokenizer
import torch
import yaml
from typing import Dict, List
class EndpointHandler:
def __init__(self, path=""):
# 加载配置
self.config = {
"max_seq_length": 512,
"model_path": "google-t5/t5-base"
}
# 设置设备
self.device = "cuda" if torch.cuda.is_available() else "cpu"
# 加载模型和分词器
self.tokenizer = AutoTokenizer.from_pretrained(path if path else self.config["model_path"])
self.model = AutoModelForSeq2SeqLM.from_pretrained(
path if path else self.config["model_path"]
).to(self.device)
# 将模型设置为评估模式
self.model.eval()
def __call__(self, data: Dict) -> Dict:
"""
处理输入请求并返回模型预测结果
Args:
data (Dict): 包含输入文本的字典,格式为 {"inputs": text}
Returns:
Dict: 包含模型输出的字典
"""
# 获取输入文本
inputs = data.pop("inputs", data)
# 确保输入是列表格式
if isinstance(inputs, str):
inputs = [inputs]
# 对输入进行分词处理
tokenized_inputs = self.tokenizer(
inputs,
max_length=self.config["max_seq_length"],
padding=True,
truncation=True,
return_tensors="pt"
).to(self.device)
# 生成预测结果
with torch.no_grad():
outputs = self.model.generate(
**tokenized_inputs,
max_new_tokens=self.config["max_seq_length"],
pad_token_id=self.tokenizer.pad_token_id
)
# 解码输出序列
decoded_outputs = self.tokenizer.batch_decode(outputs, skip_special_tokens=True)
# 构造返回结果
return {
"generated_text": decoded_outputs[0] if len(decoded_outputs) == 1 else decoded_outputs
}
@staticmethod
def format_response(response: Dict) -> Dict:
"""
格式化响应数据
Args:
response (Dict): 模型的原始输出
Returns:
Dict: 格式化后的响应
"""
return {
"status": 200,
"data": response
}