Buckets:
| """The gate that must pass before SFT starts: can every row be rendered? | |
| verify.py checks structure and evaluate.py checks quality, but neither one ever | |
| calls the chat template. That gap hid a real defect: 341,077 rows -- 30% of the | |
| set, and 86% of all tool-calling rows -- raised at render time, because | |
| assemble.py permitted `content: null` on tool-call turns (valid OpenAI schema) | |
| while the Qwen3 template dereferences content unconditionally. A sampled check | |
| would have found it; no check ran at all. | |
| It also reports the only token numbers that mean anything: the ones from the | |
| real tokenizer. assemble.py budgets on a characters-per-token estimate, and that | |
| estimate ran 1.73x low on English. | |
| python3 verify_render.py # ~50k row sample, a few minutes | |
| python3 verify_render.py --full # every row, ~1.5-2 hours | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| from collections import Counter, defaultdict | |
| from pathlib import Path | |
| os.environ.setdefault("TOKENIZERS_PARALLELISM", "true") | |
| from transformers import AutoTokenizer # noqa: E402 | |
| HERE = Path(__file__).resolve().parent | |
| MODEL = HERE.parent.parent / "model" | |
| FILES = [("train.jsonl", 32768), ("train_long64k.jsonl", 65536)] | |
| def main(): | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--full", action="store_true", help="render every row") | |
| ap.add_argument("--sample", type=int, default=50_000) | |
| ap.add_argument("--model", default=str(MODEL)) | |
| args = ap.parse_args() | |
| tok = AutoTokenizer.from_pretrained(args.model) | |
| # Assistant spans are delimited by <|im_start|>assistant ... <|im_end|>. | |
| # Under assistant-only masking these are the only tokens that carry loss, | |
| # so their share is the real trainable-token count -- not the file total. | |
| im_start = tok.convert_tokens_to_ids("<|im_start|>") | |
| im_end = tok.convert_tokens_to_ids("<|im_end|>") | |
| asst_ids = tok("assistant", add_special_tokens=False)["input_ids"] | |
| print(f"tokenizer {tok.__class__.__name__} " | |
| f"<|im_start|>={im_start} <|im_end|>={im_end} " | |
| f"assistant={asst_ids}\n", flush=True) | |
| grand_tok = grand_asst = grand_rows = grand_fail = 0 | |
| for fname, cap in FILES: | |
| path = HERE / fname | |
| if not path.exists(): | |
| print(f"{fname}: MISSING\n") | |
| continue | |
| total = sum(1 for _ in open(path, encoding="utf-8")) | |
| stride = 1 if args.full else max(1, total // args.sample) | |
| fails = Counter() | |
| fail_ex = [] | |
| lens = [] | |
| asst_frac = [] | |
| per_bucket = defaultdict(lambda: {"n": 0, "tok": 0, "asst": 0}) | |
| over = n = n_asst_tok = n_tok = 0 | |
| over_ex = [] | |
| print(f"{fname}: {total:,} rows, " | |
| f"{'EVERY row' if stride == 1 else f'every {stride}th'}", flush=True) | |
| for i, line in enumerate(open(path, encoding="utf-8")): | |
| if i % stride: | |
| continue | |
| try: | |
| r = json.loads(line) | |
| except json.JSONDecodeError as e: | |
| fails["unparseable"] += 1 | |
| if len(fail_ex) < 8: | |
| fail_ex.append(f"line {i+1}: unparseable: {e}") | |
| continue | |
| try: | |
| s = tok.apply_chat_template( | |
| r["messages"], tools=r.get("tools") or None, | |
| tokenize=False, add_generation_prompt=False) | |
| ids = tok(s, add_special_tokens=False)["input_ids"] | |
| except Exception as e: # noqa: BLE001 | |
| fails[type(e).__name__] += 1 | |
| if len(fail_ex) < 8: | |
| fail_ex.append(f"line {i+1} [{r.get('source','?')}]: " | |
| f"{type(e).__name__}: {str(e)[:70]}") | |
| continue | |
| # --- assistant-span accounting -------------------------------- | |
| a = 0 | |
| j = 0 | |
| while j < len(ids): | |
| if ids[j] == im_start and ids[j+1:j+1+len(asst_ids)] == asst_ids: | |
| k = j | |
| while k < len(ids) and ids[k] != im_end: | |
| k += 1 | |
| a += k - j | |
| j = k | |
| j += 1 | |
| t = len(ids) | |
| n += 1 | |
| n_tok += t | |
| n_asst_tok += a | |
| lens.append(t) | |
| asst_frac.append(a / max(t, 1)) | |
| b = per_bucket[r.get("bucket", "?")] | |
| b["n"] += 1 | |
| b["tok"] += t | |
| b["asst"] += a | |
| if t > cap: | |
| over += 1 | |
| if len(over_ex) < 5: | |
| over_ex.append(f"line {i+1} [{r.get('source','?')}]: {t:,} tok") | |
| lens.sort() | |
| def p(q): | |
| return lens[min(int(len(lens) * q), len(lens) - 1)] if lens else 0 | |
| scale = total / max(n, 1) | |
| print(f" rendered ok {n:>12,}") | |
| print(f" RENDER FAILURES {sum(fails.values()):>12,}" | |
| f" {'CLEAN' if not fails else 'BLOCKS TRAINING'}") | |
| for k, v in fails.most_common(): | |
| print(f" {k:26s} {v:>10,}") | |
| for x in fail_ex: | |
| print(f" {x}") | |
| print(f" real tokens p50 {p(.5):>8,} p90 {p(.9):>8,} " | |
| f"p99 {p(.99):>8,} max {lens[-1] if lens else 0:>9,}") | |
| print(f" over cap ({cap:,}) {over:,}" | |
| f" ({over/max(n,1)*100:.2f}%)" | |
| f" {'OK' if not over else 'WILL BE TRUNCATED'}") | |
| for x in over_ex: | |
| print(f" {x}") | |
| print(f" total real tokens {n_tok*scale/1e9:>12.2f} B" | |
| f"{'' if stride == 1 else ' (extrapolated)'}") | |
| print(f" ASSISTANT tokens {n_asst_tok*scale/1e9:>12.2f} B" | |
| f" = {n_asst_tok/max(n_tok,1)*100:.1f}% of the file") | |
| print(f" ^ the only tokens that carry loss under assistant-only masking") | |
| print(f" per bucket:") | |
| print(f" {'bucket':20s} {'rows':>9} {'real tok':>10} {'asst tok':>10} {'asst%':>7}") | |
| for k, v in sorted(per_bucket.items(), key=lambda x: -x[1]["tok"]): | |
| print(f" {k:20s} {v['n']:>9,} {v['tok']*scale/1e6:>9.0f}M " | |
| f"{v['asst']*scale/1e6:>9.0f}M " | |
| f"{v['asst']/max(v['tok'],1)*100:>6.1f}%") | |
| print(flush=True) | |
| grand_rows += total | |
| grand_fail += sum(fails.values()) * scale | |
| grand_tok += n_tok * scale | |
| grand_asst += n_asst_tok * scale | |
| print("=" * 74) | |
| print(f" rows {grand_rows:>14,}") | |
| print(f" render failures {grand_fail:>14,.0f}" | |
| f" {'CLEAN -- safe to pack' if grand_fail < 1 else 'FIX BEFORE PACKING'}") | |
| print(f" real tokens {grand_tok/1e9:>14.2f} B") | |
| print(f" trainable (asst) {grand_asst/1e9:>14.2f} B") | |
| print("=" * 74) | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 6.87 kB
- Xet hash:
- 2cc3eb4fe44ba48871cf13b5cdfa9e752b2cd9cf121cf393291f6d22c039713a
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.