File size: 3,426 Bytes
d19188c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
"""Build calibration / evaluation token sequences. Usage: build_sequences.py <chats.jsonl> <out.json>"""
import glob, json, os, sys, sysconfig
import pyarrow.parquet as pq
from transformers import AutoTokenizer

sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from scenarios import TOOLS  # noqa: E402

ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
tok = AutoTokenizer.from_pretrained(os.path.join(ROOT, "upstream", "Inkling-2-Mini"))
MAX = 900
out = []

def render(messages, tools=None, effort="none"):
    r = tok.apply_chat_template(messages, tools=tools, reasoning_effort=effort, tokenize=True)
    return list(r["input_ids"]) if hasattr(r, "keys") else list(r)

def clip_text(text, budget):
    ids = tok.encode(text, add_special_tokens=False)[:budget]
    return tok.decode(ids)

# 1. model-generated chats
for line in open(sys.argv[1]):
    rec = json.loads(line)
    tools = [TOOLS[t] for t in rec.get("tools", [])] or None
    ids = render(rec["messages"], tools, rec.get("effort"))
    out.append(dict(name="chat:" + rec["id"], split=rec["split"], kind="chat:" + rec["category"], ids=ids))

# 2. WikiText-2 articles framed as an assistant answer
def articles(path):
    lines = pq.read_table(path).column("text").to_pylist()
    cur_title, cur = None, []
    for ln in lines:
        s = ln.strip()
        if s.startswith("= ") and s.endswith(" =") and not s.startswith("= ="):
            if cur_title and cur:
                yield cur_title, "".join(cur)
            cur_title, cur = s.strip("= ").strip(), []
        elif cur_title is not None:
            cur.append(ln)
    if cur_title and cur:
        yield cur_title, "".join(cur)

for split, fname, count in (("eval", "test", 30), ("calib", "train", 30)):
    for i, (title, body) in enumerate(articles(os.path.join(ROOT, "data/wikitext", fname + ".parquet"))):
        if i >= count:
            break
        msgs = [{"role": "user", "content": f"Write the encyclopedia article titled “{title}”."},
                {"role": "assistant", "content": clip_text(body.strip(), MAX)}]
        out.append(dict(name=f"wikitext:{fname}:{i}", split=split, kind="wikitext", ids=render(msgs)))

# 3. Python standard-library source framed as an assistant answer
stdlib = sysconfig.get_paths()["stdlib"]
for split, names in (("calib", ["heapq", "bisect", "textwrap", "colorsys", "fnmatch", "glob", "sched", "queue"]),
                     ("eval", ["shlex", "statistics", "difflib", "copy", "keyword", "reprlib"])):
    for name in names:
        src = open(os.path.join(stdlib, name + ".py"), encoding="utf-8").read()
        for part, offset in enumerate((0, 6000)):
            piece = src[offset:offset + 12000]
            if len(piece) < 800:
                continue
            msgs = [{"role": "user", "content": f"Show part {part + 1} of the source of Python's `{name}` module."},
                    {"role": "assistant", "content": "```python\n" + clip_text(piece, MAX) + "\n```"}]
            out.append(dict(name=f"code:{name}:{part}", split=split, kind="code", ids=render(msgs)))

json.dump(out, open(sys.argv[2], "w"))
for split in ("calib", "eval"):
    sel = [s for s in out if s["split"] == split]
    kinds = {}
    for s in sel:
        k = s["kind"].split(":")[0]
        kinds[k] = kinds.get(k, 0) + len(s["ids"])
    print(split, "sequences", len(sel), "tokens", sum(len(s["ids"]) for s in sel), kinds)