import json import torch from tokenizers import Tokenizer from modeling_onner import ( OnnerReasearchConfig, OnnerReasearchModel, sample_next_token, ) def load_onner(model_dir, device=None): if device is None: device = "cuda" if torch.cuda.is_available() else "cpu" with open(f"{model_dir}/config.json", "r", encoding="utf-8") as f: cfg_dict = json.load(f) valid_keys = OnnerReasearchConfig.__dataclass_fields__.keys() cfg_kwargs = {k: v for k, v in cfg_dict.items() if k in valid_keys} if "mtp_loss_weights" in cfg_kwargs and isinstance(cfg_kwargs["mtp_loss_weights"], list): cfg_kwargs["mtp_loss_weights"] = tuple(cfg_kwargs["mtp_loss_weights"]) cfg = OnnerReasearchConfig(**cfg_kwargs) tokenizer = Tokenizer.from_file(f"{model_dir}/tokenizer.json") model = OnnerReasearchModel(cfg).to(device) state = torch.load(f"{model_dir}/pytorch_model.pt", map_location=device) model.load_state_dict(state, strict=True) model.eval() return model, tokenizer, cfg, device def encode_text(tokenizer, text): bos_id = tokenizer.token_to_id("") eos_id = tokenizer.token_to_id("") return [bos_id] + tokenizer.encode(text).ids + [eos_id] def decode_ids(tokenizer, ids): pad_id = tokenizer.token_to_id("") bos_id = tokenizer.token_to_id("") eos_id = tokenizer.token_to_id("") ids = [int(x) for x in ids] ids = [x for x in ids if x not in [pad_id, bos_id, eos_id]] return tokenizer.decode(ids) @torch.no_grad() def generate( model, tokenizer, cfg, device, prompt, max_new_tokens=100, temperature=0.6, top_k=30, ): ids = encode_text(tokenizer, prompt) input_ids = torch.tensor( [ids], dtype=torch.long, device=device, ) for _ in range(max_new_tokens): context = input_ids[:, -cfg.seq_len:] with torch.autocast( device_type="cuda" if device == "cuda" else "cpu", dtype=torch.bfloat16 if device == "cuda" else torch.float32, ): logits_list = model(context) next_logits = logits_list[0][:, -1, :] next_id = sample_next_token( next_logits, temperature=temperature, top_k=top_k, ) input_ids = torch.cat([input_ids, next_id], dim=1) return decode_ids(tokenizer, input_ids[0].tolist()) if __name__ == "__main__": model, tokenizer, cfg, device = load_onner(".") output = generate( model, tokenizer, cfg, device, prompt="Indonesia adalah negara", max_new_tokens=100, ) print(output)