"""Build calibration / evaluation token sequences. Usage: build_sequences.py """ import glob, json, os, sys, sysconfig import pyarrow.parquet as pq from transformers import AutoTokenizer sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from scenarios import TOOLS # noqa: E402 ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__))) tok = AutoTokenizer.from_pretrained(os.path.join(ROOT, "upstream", "Inkling-2-Mini")) MAX = 900 out = [] def render(messages, tools=None, effort="none"): r = tok.apply_chat_template(messages, tools=tools, reasoning_effort=effort, tokenize=True) return list(r["input_ids"]) if hasattr(r, "keys") else list(r) def clip_text(text, budget): ids = tok.encode(text, add_special_tokens=False)[:budget] return tok.decode(ids) # 1. model-generated chats for line in open(sys.argv[1]): rec = json.loads(line) tools = [TOOLS[t] for t in rec.get("tools", [])] or None ids = render(rec["messages"], tools, rec.get("effort")) out.append(dict(name="chat:" + rec["id"], split=rec["split"], kind="chat:" + rec["category"], ids=ids)) # 2. WikiText-2 articles framed as an assistant answer def articles(path): lines = pq.read_table(path).column("text").to_pylist() cur_title, cur = None, [] for ln in lines: s = ln.strip() if s.startswith("= ") and s.endswith(" =") and not s.startswith("= ="): if cur_title and cur: yield cur_title, "".join(cur) cur_title, cur = s.strip("= ").strip(), [] elif cur_title is not None: cur.append(ln) if cur_title and cur: yield cur_title, "".join(cur) for split, fname, count in (("eval", "test", 30), ("calib", "train", 30)): for i, (title, body) in enumerate(articles(os.path.join(ROOT, "data/wikitext", fname + ".parquet"))): if i >= count: break msgs = [{"role": "user", "content": f"Write the encyclopedia article titled “{title}”."}, {"role": "assistant", "content": clip_text(body.strip(), MAX)}] out.append(dict(name=f"wikitext:{fname}:{i}", split=split, kind="wikitext", ids=render(msgs))) # 3. Python standard-library source framed as an assistant answer stdlib = sysconfig.get_paths()["stdlib"] for split, names in (("calib", ["heapq", "bisect", "textwrap", "colorsys", "fnmatch", "glob", "sched", "queue"]), ("eval", ["shlex", "statistics", "difflib", "copy", "keyword", "reprlib"])): for name in names: src = open(os.path.join(stdlib, name + ".py"), encoding="utf-8").read() for part, offset in enumerate((0, 6000)): piece = src[offset:offset + 12000] if len(piece) < 800: continue msgs = [{"role": "user", "content": f"Show part {part + 1} of the source of Python's `{name}` module."}, {"role": "assistant", "content": "```python\n" + clip_text(piece, MAX) + "\n```"}] out.append(dict(name=f"code:{name}:{part}", split=split, kind="code", ids=render(msgs))) json.dump(out, open(sys.argv[2], "w")) for split in ("calib", "eval"): sel = [s for s in out if s["split"] == split] kinds = {} for s in sel: k = s["kind"].split(":")[0] kinds[k] = kinds.get(k, 0) + len(s["ids"]) print(split, "sequences", len(sel), "tokens", sum(len(s["ids"]) for s in sel), kinds)