| from typing import Dict, Optional, Union |
|
|
| import torch |
|
|
| from opencompass.utils.prompt import PromptList |
|
|
| from .huggingface import HuggingFace |
|
|
| PromptType = Union[PromptList, str] |
|
|
|
|
| class ModelScope(HuggingFace): |
| """Model wrapper around ModelScope models. |
| |
| Args: |
| path (str): The name or path to ModelScope's model. |
| ms_cache_dir: Set the cache dir to MS model cache dir. If None, it will |
| use the env variable MS_MODEL_HUB. Defaults to None. |
| max_seq_len (int): The maximum length of the input sequence. Defaults |
| to 2048. |
| tokenizer_path (str): The path to the tokenizer. Defaults to None. |
| tokenizer_kwargs (dict): Keyword arguments for the tokenizer. |
| Defaults to {}. |
| peft_path (str, optional): The name or path to the ModelScope's PEFT |
| model. If None, the original model will not be converted to PEFT. |
| Defaults to None. |
| tokenizer_only (bool): If True, only the tokenizer will be initialized. |
| Defaults to False. |
| model_kwargs (dict): Keyword arguments for the model, used in loader. |
| Defaults to dict(device_map='auto'). |
| meta_template (Dict, optional): The model's meta prompt |
| template if needed, in case the requirement of injecting or |
| wrapping of any meta instructions. |
| extract_pred_after_decode (bool): Whether to extract the prediction |
| string from the decoded output string, instead of extract the |
| prediction tokens before decoding. Defaults to False. |
| batch_padding (bool): If False, inference with be performed in for-loop |
| without batch padding. |
| pad_token_id (int): The id of the padding token. Defaults to None. Use |
| (#vocab + pad_token_id) if get negative value. |
| mode (str, optional): The method of input truncation when input length |
| exceeds max_seq_len. 'mid' represents the part of input to |
| truncate. Defaults to 'none'. |
| |
| Note: |
| About ``extract_pred_after_decode``: Commonly, we should extract the |
| the prediction tokens before decoding. But for some tokenizers using |
| ``sentencepiece``, like LLaMA, this behavior may change the number of |
| whitespaces, which is harmful for Python programming tasks. |
| """ |
|
|
| def __init__(self, |
| path: str, |
| ms_cache_dir: Optional[str] = None, |
| max_seq_len: int = 2048, |
| tokenizer_path: Optional[str] = None, |
| tokenizer_kwargs: dict = dict(), |
| peft_path: Optional[str] = None, |
| tokenizer_only: bool = False, |
| model_kwargs: dict = dict(device_map='auto'), |
| meta_template: Optional[Dict] = None, |
| extract_pred_after_decode: bool = False, |
| batch_padding: bool = False, |
| pad_token_id: Optional[int] = None, |
| mode: str = 'none'): |
| super().__init__( |
| path=path, |
| hf_cache_dir=ms_cache_dir, |
| max_seq_len=max_seq_len, |
| tokenizer_path=tokenizer_path, |
| tokenizer_kwargs=tokenizer_kwargs, |
| peft_path=peft_path, |
| tokenizer_only=tokenizer_only, |
| model_kwargs=model_kwargs, |
| meta_template=meta_template, |
| extract_pred_after_decode=extract_pred_after_decode, |
| batch_padding=batch_padding, |
| pad_token_id=pad_token_id, |
| mode=mode, |
| ) |
|
|
| def _load_tokenizer(self, path: str, tokenizer_path: Optional[str], |
| tokenizer_kwargs: dict): |
| from modelscope import AutoTokenizer |
| self.tokenizer = AutoTokenizer.from_pretrained( |
| tokenizer_path if tokenizer_path else path, **tokenizer_kwargs) |
|
|
| |
| if self.pad_token_id is not None: |
| if self.pad_token_id < 0: |
| self.pad_token_id += self.tokenizer.vocab_size |
| if self.tokenizer.pad_token_id is None: |
| self.logger.debug(f'Using {self.pad_token_id} as pad_token_id') |
| elif self.tokenizer.pad_token_id != self.pad_token_id: |
| self.logger.warning( |
| 'pad_token_id is not consistent with the tokenizer. Using ' |
| f'{self.pad_token_id} as pad_token_id') |
| self.tokenizer.pad_token_id = self.pad_token_id |
| elif self.tokenizer.pad_token_id is None: |
| self.logger.warning('pad_token_id is not set for the tokenizer.') |
| if self.tokenizer.eos_token is not None: |
| self.logger.warning( |
| f'Using eos_token_id {self.tokenizer.eos_token} ' |
| 'as pad_token_id.') |
| self.tokenizer.pad_token = self.tokenizer.eos_token |
| else: |
| from modelscope import GenerationConfig |
| gcfg = GenerationConfig.from_pretrained(path) |
|
|
| if gcfg.pad_token_id is not None: |
| self.logger.warning( |
| f'Using pad_token_id {gcfg.pad_token_id} ' |
| 'as pad_token_id.') |
| self.tokenizer.pad_token_id = gcfg.pad_token_id |
| else: |
| raise ValueError( |
| 'pad_token_id is not set for this tokenizer. Try to ' |
| 'set pad_token_id via passing ' |
| '`pad_token_id={PAD_TOKEN_ID}` in model_cfg.') |
|
|
| |
| if 'decapoda-research/llama' in path or \ |
| (tokenizer_path and |
| 'decapoda-research/llama' in tokenizer_path): |
| self.logger.warning('We set new pad_token_id for LLaMA model') |
| |
| |
| self.tokenizer.bos_token = '<s>' |
| self.tokenizer.eos_token = '</s>' |
| self.tokenizer.pad_token_id = 0 |
|
|
| def _set_model_kwargs_torch_dtype(self, model_kwargs): |
| if 'torch_dtype' not in model_kwargs: |
| torch_dtype = torch.float16 |
| else: |
| torch_dtype = { |
| 'torch.float16': torch.float16, |
| 'torch.bfloat16': torch.bfloat16, |
| 'torch.float': torch.float, |
| 'auto': 'auto', |
| 'None': None |
| }.get(model_kwargs['torch_dtype']) |
| self.logger.debug(f'MS using torch_dtype: {torch_dtype}') |
| if torch_dtype is not None: |
| model_kwargs['torch_dtype'] = torch_dtype |
|
|
| def _load_model(self, |
| path: str, |
| model_kwargs: dict, |
| peft_path: Optional[str] = None): |
| from modelscope import AutoModel, AutoModelForCausalLM |
|
|
| self._set_model_kwargs_torch_dtype(model_kwargs) |
| try: |
| self.model = AutoModelForCausalLM.from_pretrained( |
| path, **model_kwargs) |
| except ValueError: |
| self.model = AutoModel.from_pretrained(path, **model_kwargs) |
|
|
| if peft_path is not None: |
| from peft import PeftModel |
| self.model = PeftModel.from_pretrained(self.model, |
| peft_path, |
| is_trainable=False) |
| self.model.eval() |
| self.model.generation_config.do_sample = False |
|
|
| |
| if 'decapoda-research/llama' in path: |
| self.model.config.bos_token_id = 1 |
| self.model.config.eos_token_id = 2 |
| self.model.config.pad_token_id = self.tokenizer.pad_token_id |
|
|
|
|
| class ModelScopeCausalLM(ModelScope): |
| """Model wrapper around ModelScope CausalLM. |
| |
| Args: |
| path (str): The name or path to ModelScope's model. |
| ms_cache_dir: Set the cache dir to MS model cache dir. If None, it will |
| use the env variable MS_MODEL_HUB. Defaults to None. |
| max_seq_len (int): The maximum length of the input sequence. Defaults |
| to 2048. |
| tokenizer_path (str): The path to the tokenizer. Defaults to None. |
| tokenizer_kwargs (dict): Keyword arguments for the tokenizer. |
| Defaults to {}. |
| peft_path (str, optional): The name or path to the ModelScope's PEFT |
| model. If None, the original model will not be converted to PEFT. |
| Defaults to None. |
| tokenizer_only (bool): If True, only the tokenizer will be initialized. |
| Defaults to False. |
| model_kwargs (dict): Keyword arguments for the model, used in loader. |
| Defaults to dict(device_map='auto'). |
| meta_template (Dict, optional): The model's meta prompt |
| template if needed, in case the requirement of injecting or |
| wrapping of any meta instructions. |
| batch_padding (bool): If False, inference with be performed in for-loop |
| without batch padding. |
| """ |
|
|
| def _load_model(self, |
| path: str, |
| model_kwargs: dict, |
| peft_path: Optional[str] = None): |
| from modelscope import AutoModelForCausalLM |
|
|
| self._set_model_kwargs_torch_dtype(model_kwargs) |
| self.model = AutoModelForCausalLM.from_pretrained(path, **model_kwargs) |
| if peft_path is not None: |
| from peft import PeftModel |
| self.model = PeftModel.from_pretrained(self.model, |
| peft_path, |
| is_trainable=False) |
| self.model.eval() |
| self.model.generation_config.do_sample = False |
|
|
