File size: 2,693 Bytes
d9c1356 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 |
import json
import torch
from tokenizers import Tokenizer
from modeling_onner import (
OnnerReasearchConfig,
OnnerReasearchModel,
sample_next_token,
)
def load_onner(model_dir, device=None):
if device is None:
device = "cuda" if torch.cuda.is_available() else "cpu"
with open(f"{model_dir}/config.json", "r", encoding="utf-8") as f:
cfg_dict = json.load(f)
valid_keys = OnnerReasearchConfig.__dataclass_fields__.keys()
cfg_kwargs = {k: v for k, v in cfg_dict.items() if k in valid_keys}
if "mtp_loss_weights" in cfg_kwargs and isinstance(cfg_kwargs["mtp_loss_weights"], list):
cfg_kwargs["mtp_loss_weights"] = tuple(cfg_kwargs["mtp_loss_weights"])
cfg = OnnerReasearchConfig(**cfg_kwargs)
tokenizer = Tokenizer.from_file(f"{model_dir}/tokenizer.json")
model = OnnerReasearchModel(cfg).to(device)
state = torch.load(f"{model_dir}/pytorch_model.pt", map_location=device)
model.load_state_dict(state, strict=True)
model.eval()
return model, tokenizer, cfg, device
def encode_text(tokenizer, text):
bos_id = tokenizer.token_to_id("<bos>")
eos_id = tokenizer.token_to_id("<eos>")
return [bos_id] + tokenizer.encode(text).ids + [eos_id]
def decode_ids(tokenizer, ids):
pad_id = tokenizer.token_to_id("<pad>")
bos_id = tokenizer.token_to_id("<bos>")
eos_id = tokenizer.token_to_id("<eos>")
ids = [int(x) for x in ids]
ids = [x for x in ids if x not in [pad_id, bos_id, eos_id]]
return tokenizer.decode(ids)
@torch.no_grad()
def generate(
model,
tokenizer,
cfg,
device,
prompt,
max_new_tokens=100,
temperature=0.6,
top_k=30,
):
ids = encode_text(tokenizer, prompt)
input_ids = torch.tensor(
[ids],
dtype=torch.long,
device=device,
)
for _ in range(max_new_tokens):
context = input_ids[:, -cfg.seq_len:]
with torch.autocast(
device_type="cuda" if device == "cuda" else "cpu",
dtype=torch.bfloat16 if device == "cuda" else torch.float32,
):
logits_list = model(context)
next_logits = logits_list[0][:, -1, :]
next_id = sample_next_token(
next_logits,
temperature=temperature,
top_k=top_k,
)
input_ids = torch.cat([input_ids, next_id], dim=1)
return decode_ids(tokenizer, input_ids[0].tolist())
if __name__ == "__main__":
model, tokenizer, cfg, device = load_onner(".")
output = generate(
model,
tokenizer,
cfg,
device,
prompt="Indonesia adalah negara",
max_new_tokens=100,
)
print(output)
|