| import json | |
| from transformers.tokenization_utils_fast import PreTrainedTokenizerFast | |
| class MiniCPMVTokenizerFast(PreTrainedTokenizerFast): | |
| def __init__(self, **kwargs): | |
| super().__init__(**kwargs) | |
| self.eot_token = "<|eot_id|>" | |
| self.im_start = "<image>" | |
| self.im_end = "</image>" | |
| self.ref_start = "<ref>" | |
| self.ref_end = "</ref>" | |
| self.box_start = "<box>" | |
| self.box_end = "</box>" | |
| self.quad_start = "<quad>" | |
| self.quad_end = "</quad>" | |
| self.slice_start = "<slice>" | |
| self.slice_end = "</slice>" | |
| def eos_id(self): | |
| return self.eos_token_id | |
| def bos_id(self): | |
| return self.bos_token_id | |
| def unk_id(self): | |
| return self.unk_token_id | |
| def eot_id(self): | |
| return self.convert_tokens_to_ids(self.eot_token) | |
| def im_start_id(self): | |
| return self.convert_tokens_to_ids(self.im_start) | |
| def im_end_id(self): | |
| return self.convert_tokens_to_ids(self.im_end) | |
| def escape(text: str) -> str: | |
| return text | |
| def unescape(text: str) -> str: | |
| return text | |