DFveloper's picture
download
raw
23.9 kB
"""Assemble the final, filtered SFT training set.
Input is every validated pool; output is one directory holding only rows that
passed every gate, plus the manifest that says what was excluded and why.
Gates, in the order applied:
1. CONTAMINATION -- any row whose user/system text shares a 13-gram with a
benchmark TEST split is dropped. This is the only gate that cannot be
revisited after training, so it runs first and its exclusions are recorded
by id in validate/contaminated_*.jsonl.
2. STRUCTURAL -- empty/stub answers, consecutive assistant turns (they break
chat templating and assistant-only loss masking), empty tool results,
transcripts that never reach a final answer, template markers surviving into
the text, degenerate repetition, boilerplate refusals.
3. SEMANTIC -- openr1_math rows whose `correctness_math_verify` shows no
correct generation. 13.3% of that set reaches the WRONG answer behind fluent
reasoning; no text probe can see it.
4. LANGUAGE -- rows tagged `ko` containing no Hangul. They inflate the
Korean ratio the whole mix is balanced on.
5. LENGTH -- rows over the training seq_len are dropped rather than
truncated. Truncating a reasoning trace mid-thought teaches the model to
stop thinking halfway, which is worse than omitting the row.
Long context is a primary goal of this model alongside Korean, so the 32-64K
band is preserved deliberately and reported separately -- see LONG_CONTEXT in
the output manifest.
python3 assemble.py --seq-len 32768
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from collections import Counter, defaultdict
from pathlib import Path
HERE = Path(__file__).resolve().parent
DATA = HERE.parent
VAL = DATA / "validate"
HANGUL = re.compile("[가-힣]")
# Measured against the real Qwen3 tokenizer on this corpus, not assumed. The
# previous 5.95/1.45 estimates ran 1.73x low on English and 1.09x low on Korean,
# which is why 47% of train_long64k.jsonl exceeded its own 65,536 cap and some
# rows passed the model's 131,072 position limit.
CH_PER_TOK_EN = 3.44
CH_PER_TOK_KO = 1.33
BOILERPLATE = re.compile(
r"as an ai language model|i'm sorry,? but as an ai|i cannot fulfill|"
r"as a large language model", re.I)
TEMPLATE_LEAK = re.compile(
r"<\|im_start\|>|<\|im_end\|>|<\|User\|>|<\|Assistant\|>|<\|endoftext\|>|"
r"\[INST\]|<<SYS>>|### Instruction:|### Response:")
TRUNCATED = re.compile(r"(?:\.\.\.|…|\[truncated\]|\(truncated\))\s*$")
def degenerate(s: str) -> bool:
if len(s) < 400:
return False
return any(s.count(s[-n:]) >= 4 for n in (40, 80, 160) if s[-n:])
ROLE_PREFIX = {
"user": re.compile(r"^\s*(?:Human|User)\s*:\s*", re.I),
"assistant": re.compile(r"^\s*(?:GPT|Assistant|AI)\s*:\s*", re.I),
}
def strip_role_prefix(msgs) -> int:
"""Remove leaked 'Human:' / 'GPT:' turn markers from message content.
IkJun1/korean-qa-dataset ships every row this way -- 44,670 rows, 100% of
the source -- and the chat template then renders 'GPT: ...' *inside* the
assistant turn, teaching the model to prefix its own replies. TEMPLATE_LEAK
does not catch it because these are plain words, not special tokens.
Dropping the rows would be the easy call and the wrong one: Korean is the
supply-limited half of this mix, so 44,670 rows is worth more than the
two-line fix costs. The prefix is matched only against the message's OWN
role, so a user genuinely writing "GPT: ..." is left alone.
"""
fixed = 0
for m in msgs:
pat = ROLE_PREFIX.get(m.get("role"))
c = m.get("content")
if pat and isinstance(c, str):
new = pat.sub("", c, count=1)
if new != c:
m["content"] = new
fixed += 1
return fixed
def normalise_content(msgs) -> None:
"""Force every message's content to a string.
The chat template dereferences it unconditionally ("'</think>' in
message.content"), so a null -- or an absent key -- raises at render time,
not at training time. nexus ships content: null on tool-call turns, which is
valid OpenAI schema and fatal here; that alone is 30% of the set.
"""
for m in msgs:
if not isinstance(m.get("content"), str):
m["content"] = "" if m.get("content") is None else str(m["content"])
def structural(msgs) -> str | None:
if not msgs:
return "no_messages"
roles = [m.get("role") for m in msgs]
if "user" not in roles or "assistant" not in roles:
return "missing_turn"
if roles[-1] != "assistant":
return "no_final_answer"
for a, b in zip(roles, roles[1:]):
if a == b == "assistant":
return "consecutive_assistant"
for m in msgs:
if m.get("role") == "tool" and not str(m.get("content") or "").strip():
return "empty_tool_result"
asst = " ".join(str(m.get("content") or "") for m in msgs
if m.get("role") == "assistant")
if not any(m.get("tool_calls") for m in msgs):
if not asst.strip():
return "empty_answer"
if len(asst.strip()) < 10:
return "stub_answer"
whole = " ".join(str(m.get("content") or "") for m in msgs)
if TEMPLATE_LEAK.search(whole):
return "template_leak"
if BOILERPLATE.search(asst):
return "boilerplate_refusal"
if TRUNCATED.search(asst):
return "truncated"
# An opened <think> that never closes is a reasoning trace that ran out of
# generation budget upstream: 51-62K characters that stop mid-sentence, with
# no final answer to recover and sometimes degenerated into another language
# entirely. 66% of what openthoughts3 yields is this shape, and it teaches
# the model to enter thinking mode and never leave it. The LENGTH rule in the
# module docstring already covers the principle -- truncated reasoning is
# worse than an omitted row -- but TRUNCATED only matches a trailing
# ellipsis, so this form walked straight through.
if asst.count("<think>") != asst.count("</think>"):
return "unbalanced_think"
if degenerate(asst):
return "degenerate_repetition"
return None
def g_msgs(r):
return r.get("messages")
def g_conv(r):
role = {"human": "user", "user": "user", "gpt": "assistant",
"assistant": "assistant", "system": "system"}
out = []
for t in r.get("conversations") or []:
rr = role.get(t.get("from") or t.get("role"))
if rr is None:
return None
out.append({"role": rr, "content": t.get("value") or t.get("content") or ""})
return out
def g_ocr(r):
return [{"role": "user", "content": r.get("input") or ""},
{"role": "assistant", "content": r.get("output") or ""}]
def g_prompt_response(r):
"""LongCite ships {prompt, response}, not {messages}.
Missing this getter cost all 8,000 of its rows on the previous run -- and it
is the ONLY source in the pool that exceeds 32,768 tokens (max 62,652), so
losing it left positions 32,768-65,536 with no training signal at all on a
model whose 64K window is a primary goal.
"""
p, a = str(r.get("prompt") or ""), str(r.get("response") or "")
if not p.strip() or not a.strip():
return None
return [{"role": "user", "content": p}, {"role": "assistant", "content": a}]
def g_openr1(r):
"""openr1_math also carries the correctness signal used by gate 3."""
return r.get("messages")
# --- the tool-calling mix ------------------------------------------------
#
# Tool data used to arrive by accident: core_en read merged.jsonl in file order
# and kept whatever landed inside its token budget. Measured on that output,
# 89,003 English tool rows held 43 single-call examples -- 0.05% -- and 94.9%
# parallel calls. The model would have learned to fire 2-3 calls at everything
# and had almost no signal for "one call, read the result, answer", which is the
# ordinary case. No source was bad; nobody ever chose the mix.
#
# So tool calling gets its own bucket, and each source is capped by the SHAPE it
# teaches rather than by how much of it happens to exist:
#
# single call apigen (47% single, 40,880 distinct tools)
# nemotron (25% single, ZERO parallel, 3,839 tools)
# mkd_nova_tooluse (62% single, and 1,096 Korean rows)
# long-context tool mkd_nova_tooluse_32k (100% single at 117KB/row)
# multi-agent self-generated, category=multi_agent: its "tools" are
# sub-agents ("Delegate a subtask to the researcher
# sub-agent"), fanned out in parallel and then synthesised.
# Taken whole -- its 100% parallel rate IS the orchestration
# pattern, not a skew to correct.
# agentic loops nexus + nexus_recovery (10+ calls, 35,987 error-recovery
# rows) -- capped hard: between them just 25 tool names
# decline to call when2call_sft, mkd_toolcall_v1 (4,820 zero-call rows)
# template format hermes_glaive/singleturn carry <tool_call> markup as text,
# which is exactly what this chat template renders
#
# toolace is EXCLUDED, not capped. Its calls read
# '[Market Trends API(trend_type="MARKET_INDEXES")]' rather than
# <tool_call>{"name": ...}</tool_call>, and its tool names contain spaces.
# Training on it teaches the wrong call syntax; all 10,536 rows are unusable
# until someone writes a converter.
#
# Caps are ROW counts, and the bucket's token target is set far above what they
# can produce, so the caps bind instead of the budget. That ordering matters: a
# binding token budget is what silently starved nemotron and apigen last time.
TOOL_ROW_CAPS = {
# single + multi call backbone. 40,880 distinct tools and the cheapest rows
# in the pool, so it is taken whole.
"apigen": 92_912,
# 24,300 single-call rows and ZERO parallel ones -- the direct corrective to
# a set that was 94.9% parallel.
"nvidia_nemotron": 45_000,
# category=multi_agent: tools ARE sub-agents ("Delegate a subtask to the
# researcher sub-agent"), fanned out in parallel then synthesised. Its 100%
# parallel rate is the orchestration pattern, not a defect, so it is NOT
# capped -- this is the multi-agent/sub-agent/orchestration goal itself.
"self-generated": 49_979,
# agentic loops and error recovery. Capped hard: between them just 25 tool
# names, and nexus alone would otherwise be 30% of the dataset again.
"nexus": 15_000,
"nexus_recovery": 14_283,
# 62% single-call, and 1,096 of the ~5,100 Korean tool rows in existence.
"mkd_nova_tooluse": 12_804,
# the ONLY long-context tool data anywhere in the pool, at 117KB/row. The
# model's window is 64K native, so tool use over long inputs has to be
# trained somewhere -- this is the somewhere. Taken whole.
"mkd_nova_tooluse_32k": 3_000,
# "answer directly, do not call a tool" -- 1,638 no_tool_needed rows plus
# when2call's 10,090. Only 2.27% of the previous set taught restraint.
"when2call_sft": 10_090,
# smallest source, best designed: 12 edge-case categories (missing_params,
# tool_unavailable, ambiguous_tool, dependent_chain, self_correction,
# long_horizon...) and 4,003 Korean rows. Nothing else covers either.
"mkd_toolcall_v1": 6_253,
# ship <tool_call> markup as text, which is exactly what this template
# renders, and both are single-call heavy.
"hermes_glaive": 4_157,
"hermes_singleturn": 1_100,
"hermes_reasoning": 10_000,
}
# Held out of korean/core_en so a tool row is not also counted as general chat.
TOOL_SOURCES = frozenset(TOOL_ROW_CAPS)
# --- repetition for supply-limited buckets -------------------------------
#
# Korean is the constraint on this whole mix. The bucket reads all 2,306,564 rows
# of the pool at stride 1 and finds 261.6M tokens against a 400M target -- that
# is not a budget being spent, it is the pool being exhausted. Since Korean chat
# is the model's primary goal, the share has to come from repetition rather than
# from more data, which is what the note at the top of BUCKETS always said
# ("taken in full and repeated to reach its share") and what was never actually
# implemented.
#
# A bucket listed here re-reads its file until its token target is met or it runs
# out of passes. Korean's 445M target lands at ~1.7 passes, so roughly 70% of
# Korean rows appear twice. Every other bucket stays at a single pass.
REPEAT_MAX = {"korean": 3}
# --- the mix: token budget per bucket, not "filter everything" -----------
#
# The earlier version filtered every pool in full -- 26B tokens -- to build a
# 1.0B training set. For reasoning that meant processing 18.36B to use 220M
# (1.2%) and for code 5.5B to use 100M (1.8%): about two hours and 100GB of
# output that would never be sampled. Same error as over-collecting, one stage
# later.
#
# Now each pool is read with a STRIDE and stops as soon as its bucket is full.
# Stride sampling keeps the sample spread across the whole file (these files are
# grouped by sub-source, so taking the first N would take one sub-source), while
# skipping the JSON parse and the gate checks for rows we will not use -- which
# is where the time actually goes.
#
# Korean is the exception: it is supply-limited at 203M tokens, so it is taken in
# full and repeated to reach its share. See the project note on Korean and 64K
# being the two goals that define the mix.
BUCKETS = [
# (name, rel path, getter, token target, stride, lang filter, source filter)
#
# source filter is None, ("only", sources) or ("without", sources). The tool
# sources are held out of korean/core_en and handled by the tool bucket, so a
# tool row is never also counted as general chat.
#
# Korean and English are read from the SAME file but as separate buckets with
# separate budgets. Sharing one cap does not work: Korean rows average 578
# characters against English's 2,633, so English consumes the budget first and
# Korean takes what is left. Under a single 700M cap Korean landed at 111M --
# 9.3% of the mix on a Korean-focused model, against a 40% target.
# 445M = 1.7 passes over every Korean row that exists. See REPEAT_MAX.
("korean", "sft_dataset/merged.jsonl", g_msgs, 445_000_000, 1, "ko",
("without", TOOL_SOURCES)),
("core_en", "sft_dataset/merged.jsonl", g_msgs, 240_000_000, 1, "en",
("without", TOOL_SOURCES)),
# Tool calling, all shapes, both languages. The token target is deliberately
# far above what TOOL_ROW_CAPS can yield, so the row caps decide the mix and
# the budget never silently truncates the read the way it starved nemotron
# and apigen before. Korean tool rows (~5,100, all that exist anywhere) ride
# along here rather than in the korean bucket.
("tool", "sft_dataset/merged.jsonl", g_msgs, 900_000_000, 1, None,
("only", TOOL_SOURCES)),
# stride 37 -> 12 because the unbalanced_think gate now rejects ~66% of this
# pool. The budget is unchanged; a denser stride just parses enough rows from
# the same bytes to still fill 180M with VALID reasoning. Raising the stride
# instead of the target is what keeps thinking data from shrinking without
# growing the dataset.
# Korean reasoning. Generated rather than collected: a scan of every source
# returned zero Korean rows with a balanced <think> block, so thinking mode
# would be English-only on a Korean-first model. Deliberately NOT merged into
# merged.jsonl -- appending to a 12GB file to add 25k rows is a worse trade
# than one more bucket. Skipped automatically until the file exists.
("korean_cot", "korean_cot/ko_cot.jsonl", g_msgs, 60_000_000, 1, None, None),
("reasoning", "reasoning/openthoughts3.jsonl", g_conv, 180_000_000, 12, None, None),
# LongCite is trimmed from 400M to 150M. Its value is the 3,056 rows in the
# 32-64K band, not the 23,000 short rows that padded it to 34% of the whole
# mix and displaced the Korean this model exists for.
# LongCite is taken for its LONG rows only. Read sequentially it spends the
# budget on short rows -- 150M tokens bought just 1,140 band rows last build,
# because only 11.6% of the file is 32-64K. A floor of 16K tokens skips the
# short ones entirely and puts the whole budget where the 64K window needs it.
("long_context", "longctx/longcite_full.jsonl", g_prompt_response, 220_000_000, 1, ("any", 16384), None),
("code", "reasoning/opencodereasoning.jsonl", g_ocr, 80_000_000, 41, None, None),
("reasoning_math", "reasoning/openr1_math.jsonl", g_msgs, 40_000_000, 5, None, None),
("long_context", "longctx/longwriter.jsonl", g_msgs, 10_000_000, 1, None, None),
]
def load_contaminated() -> set:
ids = set()
for p in VAL.glob("contaminated_*.jsonl"):
for line in open(p, encoding="utf-8"):
try:
r = json.loads(line)
except json.JSONDecodeError:
continue
if r.get("id") is not None:
ids.add((p.stem.replace("contaminated_", ""), str(r["id"])))
return ids
def main():
ap = argparse.ArgumentParser()
ap.add_argument("--seq-len", type=int, default=32768)
ap.add_argument("--long-seq-len", type=int, default=65536)
args = ap.parse_args()
contaminated = load_contaminated()
print(f"contamination exclusions: {len(contaminated):,}")
budget = sum(b[3] for b in BUCKETS)
print(f"target: {budget/1e9:.2f}B tokens across {len(BUCKETS)} buckets\n")
out_main = open(HERE / "train.jsonl", "w", encoding="utf-8")
out_long = open(HERE / "train_long64k.jsonl", "w", encoding="utf-8")
report = {}
src_kept = Counter() # global, so a capped source cannot re-enter
# its quota in a later bucket
for name, rel, getter, target, stride, want_lang, src_filter in BUCKETS:
path = DATA / rel
if not path.exists():
print(f"{rel:38s} MISSING")
continue
st = {"read": 0, "kept": 0, "long": 0, "tok": 0.0, "ko": 0,
"prefix": 0, "drop": Counter()}
pool = rel.split("/")[-1].replace(".jsonl", "")
def _lines(_p=path, _n=name, _st=st, _t=target):
"""Lines of the pool, re-read while a repeat bucket is short."""
for _ in range(REPEAT_MAX.get(_n, 1)):
if _st["tok"] >= _t:
return
with open(_p, encoding="utf-8") as fh:
yield from fh
for i, line in enumerate(_lines()):
if st["tok"] >= target:
break
if i % stride:
continue # skipped without parsing
st["read"] += 1
try:
r = json.loads(line)
except json.JSONDecodeError:
st["drop"]["unparseable"] += 1
continue
if (pool, str(r.get("id"))) in contaminated:
st["drop"]["contaminated"] += 1
continue
src_name = r.get("source", pool)
if src_filter:
mode, srcs = src_filter
if (src_name in srcs) != (mode == "only"):
continue # belongs to a different bucket
if name == "tool":
cap = TOOL_ROW_CAPS.get(src_name)
if cap is not None and src_kept[src_name] >= cap:
st["drop"]["row_cap"] += 1
continue
msgs = getter(r)
if msgs:
normalise_content(msgs)
st["prefix"] += strip_role_prefix(msgs)
why = structural(msgs) if msgs else "no_messages"
if why:
st["drop"][why] += 1
continue
if pool == "openr1_math":
v = r.get("correctness_math_verify") or []
if v and not any(v):
st["drop"]["math_answer_wrong"] += 1
continue
text = " ".join(str(m.get("content") or "") for m in msgs)
ko = len(HANGUL.findall(text)) / max(len(text), 1) > 0.05
if r.get("lang") == "ko" and not ko:
st["drop"]["lang_tag_ko_no_hangul"] += 1
continue
wl, min_tok = (want_lang if isinstance(want_lang, tuple)
else (want_lang, 0))
if wl == "ko" and not ko:
continue # counted nowhere: this bucket is Korean only
if wl == "en" and ko:
continue
# The template renders every tool schema into the system prompt,
# so those characters are trained on and have to be budgeted.
# Counting message content alone under-read core_en by 26% (240M
# budgeted against 303M real) and let big-schema rows slip past
# the 64K gate. Schemas are JSON/English whatever the row's
# language, so they bill at the English rate.
ntok = len(text) / (CH_PER_TOK_KO if ko else CH_PER_TOK_EN)
if r.get("tools"):
ntok += (len(json.dumps(r["tools"], ensure_ascii=False))
/ CH_PER_TOK_EN)
if min_tok and ntok < min_tok:
continue # this bucket exists for long rows only
if ntok > args.long_seq_len:
st["drop"]["over_64k"] += 1
continue
rec = {"bucket": name, "pool": pool,
"source": src_name,
"lang": "ko" if ko else "en",
"category": r.get("category"), "messages": msgs}
if r.get("tools"):
rec["tools"] = r["tools"]
if ntok > args.seq_len:
out_long.write(json.dumps(rec, ensure_ascii=False) + "\n")
st["long"] += 1
else:
out_main.write(json.dumps(rec, ensure_ascii=False) + "\n")
st["kept"] += 1
st["ko"] += ko
st["tok"] += ntok
src_kept[src_name] += 1
print(f"{pool:24s} target {target/1e6:>5.0f}M kept {st['kept']:>8,} "
f"({st['tok']/1e6:>6.1f}M tok) long64k {st['long']:>5,} "
f"ko {st['ko']:>7,} stride {stride}", flush=True)
report.setdefault(name, []).append(
{"pool": pool, "read": st["read"], "kept": st["kept"],
"role_prefix_stripped": st["prefix"],
"tokens": int(st["tok"]), "korean": st["ko"], "long64k": st["long"],
"stride": stride, "dropped": dict(st["drop"])})
out_main.close()
out_long.close()
tk = sum(x["tokens"] for v in report.values() for x in v)
kp = sum(x["kept"] for v in report.values() for x in v)
ko = sum(x["korean"] for v in report.values() for x in v)
lg = sum(x["long64k"] for v in report.values() for x in v)
print(f"\n{'='*74}")
print(f" {kp:,} rows {tk/1e9:.2f}B tokens korean {ko:,} rows "
f"({ko/max(kp,1)*100:.1f}%)")
print(f" train.jsonl {kp-lg:,} | train_long64k.jsonl {lg:,}")
(HERE / "MANIFEST.json").write_text(json.dumps(
{"seq_len": args.seq_len, "long_seq_len": args.long_seq_len,
"total_rows": kp, "total_tokens": tk, "korean_rows": ko,
"long64k_rows": lg, "buckets": report}, indent=2, ensure_ascii=False),
encoding="utf-8")
print(f" -> {HERE}\n")
if __name__ == "__main__":
sys.exit(main())

Xet Storage Details

Size:
23.9 kB
·
Xet hash:
8a8f9d70c199d027ddb0ce1febe326dac4f486bd95e39fa6b50335384ba51a18

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.