Inkling-2-Mini-MLX-2bit / extras /scripts /build_sequences.py
VisualInference's picture
Inkling 2 Mini for MLX, 2bit: weights, card, validation results
031489f verified
Raw History Blame Contribute Delete
3.43 kB
"""Build calibration / evaluation token sequences. Usage: build_sequences.py <chats.jsonl> <out.json>"""
import glob, json, os, sys, sysconfig
import pyarrow.parquet as pq
from transformers import AutoTokenizer
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from scenarios import TOOLS # noqa: E402
ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
tok = AutoTokenizer.from_pretrained(os.path.join(ROOT, "upstream", "Inkling-2-Mini"))
MAX = 900
out = []
def render(messages, tools=None, effort="none"):
r = tok.apply_chat_template(messages, tools=tools, reasoning_effort=effort, tokenize=True)
return list(r["input_ids"]) if hasattr(r, "keys") else list(r)
def clip_text(text, budget):
ids = tok.encode(text, add_special_tokens=False)[:budget]
return tok.decode(ids)
# 1. model-generated chats
for line in open(sys.argv[1]):
rec = json.loads(line)
tools = [TOOLS[t] for t in rec.get("tools", [])] or None
ids = render(rec["messages"], tools, rec.get("effort"))
out.append(dict(name="chat:" + rec["id"], split=rec["split"], kind="chat:" + rec["category"], ids=ids))
# 2. WikiText-2 articles framed as an assistant answer
def articles(path):
lines = pq.read_table(path).column("text").to_pylist()
cur_title, cur = None, []
for ln in lines:
s = ln.strip()
if s.startswith("= ") and s.endswith(" =") and not s.startswith("= ="):
if cur_title and cur:
yield cur_title, "".join(cur)
cur_title, cur = s.strip("= ").strip(), []
elif cur_title is not None:
cur.append(ln)
if cur_title and cur:
yield cur_title, "".join(cur)
for split, fname, count in (("eval", "test", 30), ("calib", "train", 30)):
for i, (title, body) in enumerate(articles(os.path.join(ROOT, "data/wikitext", fname + ".parquet"))):
if i >= count:
break
msgs = [{"role": "user", "content": f"Write the encyclopedia article titled “{title}”."},
{"role": "assistant", "content": clip_text(body.strip(), MAX)}]
out.append(dict(name=f"wikitext:{fname}:{i}", split=split, kind="wikitext", ids=render(msgs)))
# 3. Python standard-library source framed as an assistant answer
stdlib = sysconfig.get_paths()["stdlib"]
for split, names in (("calib", ["heapq", "bisect", "textwrap", "colorsys", "fnmatch", "glob", "sched", "queue"]),
("eval", ["shlex", "statistics", "difflib", "copy", "keyword", "reprlib"])):
for name in names:
src = open(os.path.join(stdlib, name + ".py"), encoding="utf-8").read()
for part, offset in enumerate((0, 6000)):
piece = src[offset:offset + 12000]
if len(piece) < 800:
continue
msgs = [{"role": "user", "content": f"Show part {part + 1} of the source of Python's `{name}` module."},
{"role": "assistant", "content": "```python\n" + clip_text(piece, MAX) + "\n```"}]
out.append(dict(name=f"code:{name}:{part}", split=split, kind="code", ids=render(msgs)))
json.dump(out, open(sys.argv[2], "w"))
for split in ("calib", "eval"):
sel = [s for s in out if s["split"] == split]
kinds = {}
for s in sel:
k = s["kind"].split(":")[0]
kinds[k] = kinds.get(k, 0) + len(s["ids"])
print(split, "sequences", len(sel), "tokens", sum(len(s["ids"]) for s in sel), kinds)