Download inference.py from ReskiGaming/onner: direct link, hf CLI and curl.
- Browser
- Download file 2.69 kB
-
https://huggingface.co/ReskiGaming/onner/resolve/main/inference.py
- Command line
-
hf download hf://ReskiGaming/onner/inference.py
-
curl -L -o inference.py https://huggingface.co/ReskiGaming/onner/resolve/main/inference.py
2.69 kB
| import json | |
| import torch | |
| from tokenizers import Tokenizer | |
| from modeling_onner import ( | |
| OnnerReasearchConfig, | |
| OnnerReasearchModel, | |
| sample_next_token, | |
| ) | |
| def load_onner(model_dir, device=None): | |
| if device is None: | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| with open(f"{model_dir}/config.json", "r", encoding="utf-8") as f: | |
| cfg_dict = json.load(f) | |
| valid_keys = OnnerReasearchConfig.__dataclass_fields__.keys() | |
| cfg_kwargs = {k: v for k, v in cfg_dict.items() if k in valid_keys} | |
| if "mtp_loss_weights" in cfg_kwargs and isinstance(cfg_kwargs["mtp_loss_weights"], list): | |
| cfg_kwargs["mtp_loss_weights"] = tuple(cfg_kwargs["mtp_loss_weights"]) | |
| cfg = OnnerReasearchConfig(**cfg_kwargs) | |
| tokenizer = Tokenizer.from_file(f"{model_dir}/tokenizer.json") | |
| model = OnnerReasearchModel(cfg).to(device) | |
| state = torch.load(f"{model_dir}/pytorch_model.pt", map_location=device) | |
| model.load_state_dict(state, strict=True) | |
| model.eval() | |
| return model, tokenizer, cfg, device | |
| def encode_text(tokenizer, text): | |
| bos_id = tokenizer.token_to_id("<bos>") | |
| eos_id = tokenizer.token_to_id("<eos>") | |
| return [bos_id] + tokenizer.encode(text).ids + [eos_id] | |
| def decode_ids(tokenizer, ids): | |
| pad_id = tokenizer.token_to_id("<pad>") | |
| bos_id = tokenizer.token_to_id("<bos>") | |
| eos_id = tokenizer.token_to_id("<eos>") | |
| ids = [int(x) for x in ids] | |
| ids = [x for x in ids if x not in [pad_id, bos_id, eos_id]] | |
| return tokenizer.decode(ids) | |
| def generate( | |
| model, | |
| tokenizer, | |
| cfg, | |
| device, | |
| prompt, | |
| max_new_tokens=100, | |
| temperature=0.6, | |
| top_k=30, | |
| ): | |
| ids = encode_text(tokenizer, prompt) | |
| input_ids = torch.tensor( | |
| [ids], | |
| dtype=torch.long, | |
| device=device, | |
| ) | |
| for _ in range(max_new_tokens): | |
| context = input_ids[:, -cfg.seq_len:] | |
| with torch.autocast( | |
| device_type="cuda" if device == "cuda" else "cpu", | |
| dtype=torch.bfloat16 if device == "cuda" else torch.float32, | |
| ): | |
| logits_list = model(context) | |
| next_logits = logits_list[0][:, -1, :] | |
| next_id = sample_next_token( | |
| next_logits, | |
| temperature=temperature, | |
| top_k=top_k, | |
| ) | |
| input_ids = torch.cat([input_ids, next_id], dim=1) | |
| return decode_ids(tokenizer, input_ids[0].tolist()) | |
| if __name__ == "__main__": | |
| model, tokenizer, cfg, device = load_onner(".") | |
| output = generate( | |
| model, | |
| tokenizer, | |
| cfg, | |
| device, | |
| prompt="Indonesia adalah negara", | |
| max_new_tokens=100, | |
| ) | |
| print(output) | |