File size: 2,693 Bytes
d9c1356
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105

import json
import torch
from tokenizers import Tokenizer
from modeling_onner import (
    OnnerReasearchConfig,
    OnnerReasearchModel,
    sample_next_token,
)


def load_onner(model_dir, device=None):
    if device is None:
        device = "cuda" if torch.cuda.is_available() else "cpu"

    with open(f"{model_dir}/config.json", "r", encoding="utf-8") as f:
        cfg_dict = json.load(f)

    valid_keys = OnnerReasearchConfig.__dataclass_fields__.keys()
    cfg_kwargs = {k: v for k, v in cfg_dict.items() if k in valid_keys}

    if "mtp_loss_weights" in cfg_kwargs and isinstance(cfg_kwargs["mtp_loss_weights"], list):
        cfg_kwargs["mtp_loss_weights"] = tuple(cfg_kwargs["mtp_loss_weights"])

    cfg = OnnerReasearchConfig(**cfg_kwargs)

    tokenizer = Tokenizer.from_file(f"{model_dir}/tokenizer.json")

    model = OnnerReasearchModel(cfg).to(device)
    state = torch.load(f"{model_dir}/pytorch_model.pt", map_location=device)
    model.load_state_dict(state, strict=True)
    model.eval()

    return model, tokenizer, cfg, device


def encode_text(tokenizer, text):
    bos_id = tokenizer.token_to_id("<bos>")
    eos_id = tokenizer.token_to_id("<eos>")
    return [bos_id] + tokenizer.encode(text).ids + [eos_id]


def decode_ids(tokenizer, ids):
    pad_id = tokenizer.token_to_id("<pad>")
    bos_id = tokenizer.token_to_id("<bos>")
    eos_id = tokenizer.token_to_id("<eos>")

    ids = [int(x) for x in ids]
    ids = [x for x in ids if x not in [pad_id, bos_id, eos_id]]

    return tokenizer.decode(ids)


@torch.no_grad()
def generate(
    model,
    tokenizer,
    cfg,
    device,
    prompt,
    max_new_tokens=100,
    temperature=0.6,
    top_k=30,
):
    ids = encode_text(tokenizer, prompt)

    input_ids = torch.tensor(
        [ids],
        dtype=torch.long,
        device=device,
    )

    for _ in range(max_new_tokens):
        context = input_ids[:, -cfg.seq_len:]

        with torch.autocast(
            device_type="cuda" if device == "cuda" else "cpu",
            dtype=torch.bfloat16 if device == "cuda" else torch.float32,
        ):
            logits_list = model(context)

        next_logits = logits_list[0][:, -1, :]
        next_id = sample_next_token(
            next_logits,
            temperature=temperature,
            top_k=top_k,
        )

        input_ids = torch.cat([input_ids, next_id], dim=1)

    return decode_ids(tokenizer, input_ids[0].tolist())


if __name__ == "__main__":
    model, tokenizer, cfg, device = load_onner(".")
    output = generate(
        model,
        tokenizer,
        cfg,
        device,
        prompt="Indonesia adalah negara",
        max_new_tokens=100,
    )
    print(output)