from transformers import AutoModelForSeq2SeqLM, AutoTokenizer import torch import yaml from typing import Dict, List class EndpointHandler: def __init__(self, path=""): # 加载配置 self.config = { "max_seq_length": 512, "model_path": "google-t5/t5-base" } # 设置设备 self.device = "cuda" if torch.cuda.is_available() else "cpu" # 加载模型和分词器 self.tokenizer = AutoTokenizer.from_pretrained(path if path else self.config["model_path"]) self.model = AutoModelForSeq2SeqLM.from_pretrained( path if path else self.config["model_path"] ).to(self.device) # 将模型设置为评估模式 self.model.eval() def __call__(self, data: Dict) -> Dict: """ 处理输入请求并返回模型预测结果 Args: data (Dict): 包含输入文本的字典,格式为 {"inputs": text} Returns: Dict: 包含模型输出的字典 """ # 获取输入文本 inputs = data.pop("inputs", data) # 确保输入是列表格式 if isinstance(inputs, str): inputs = [inputs] # 对输入进行分词处理 tokenized_inputs = self.tokenizer( inputs, max_length=self.config["max_seq_length"], padding=True, truncation=True, return_tensors="pt" ).to(self.device) # 生成预测结果 with torch.no_grad(): outputs = self.model.generate( **tokenized_inputs, max_new_tokens=self.config["max_seq_length"], pad_token_id=self.tokenizer.pad_token_id ) # 解码输出序列 decoded_outputs = self.tokenizer.batch_decode(outputs, skip_special_tokens=True) # 构造返回结果 return { "generated_text": decoded_outputs[0] if len(decoded_outputs) == 1 else decoded_outputs } @staticmethod def format_response(response: Dict) -> Dict: """ 格式化响应数据 Args: response (Dict): 模型的原始输出 Returns: Dict: 格式化后的响应 """ return { "status": 200, "data": response }