simplixity-router-2b / training /make_dataset.py
stneric's picture
v1: Qwen3.5-2B router distilled from Haiku 5.5 (checkpoint 350), training code, synthetic data, v2 plan
ad9956e verified
Raw History Blame Contribute Delete
5.46 kB
"""Check the teacher's labels and write the training files.
A label is dropped, never repaired, when it breaks a rule the app or the
prompt states: an unknown shape, recency or complexity value, a facet name
used as a shape, facets beside a shape, more queries than the depth allows,
a URL in a query, a picture without a search, queries without a search. A
repaired label would teach the student an answer the teacher never gave.
Three fields the app ignores in a given situation are emptied (`neutral`).
The answer is written in schema order, which is the order strict JSON mode
fills it in and the order the student will generate it in.
python3 -I make_dataset.py
data/train.jsonl, data/valid.jsonl synthetic only (publishable)
data/private/train_real.jsonl, data/private/valid_real.jsonl
"""
from __future__ import annotations
import json
import os
from collections import Counter
HERE = os.path.dirname(os.path.abspath(__file__))
SCHEMA = json.load(open(os.path.join(HERE, "eval", "schema.json")))
ORDER = list(SCHEMA["properties"])
SHAPES = {"", "figure", "series", "table", "image", "match", "weather", "diagram", "profile", "steps",
"comparison", "explainer"}
FACETS = {"facts", "stats", "timeline", "proscons", "list", "grid", "callout"}
RECENCY = {"", "day", "week", "month", "year"}
COMPLEXITY = {"simple", "standard", "research"}
LENSES = {"web", "scholar", "news", "docs", "reference", "data", "library"}
def problems(label: dict, settings: dict) -> list[str]:
out = []
if set(label) != set(ORDER):
out.append("fields")
return out
if label["answer_shape"] not in SHAPES:
out.append("facet as shape" if label["answer_shape"] in FACETS else "shape")
if label["recency"] not in RECENCY:
out.append("recency")
if label["complexity"] not in COMPLEXITY:
out.append("complexity")
if not set(label["facets"]) <= FACETS:
out.append("facet")
if label["facets"] and label["answer_shape"]:
out.append("facets beside shape")
if not set(label["lenses"]) <= LENSES:
out.append("lens")
queries = label["search_queries"]
if len(queries) > settings["queries"]:
out.append("too many queries")
if any("://" in q for q in queries):
out.append("url in query")
if label["wants_image"] and not label["needs_search"]:
out.append("image without search")
if not label["needs_search"] and queries:
out.append("queries without search")
if label["needs_search"] and not queries:
out.append("search without queries")
if label["answer_shape"] == "weather" and not label["place"].strip():
out.append("weather without place")
return out
def neutral(label: dict, settings: dict) -> dict:
"""Empty the fields the app ignores in this situation, so the student is
not taught the teacher's noise in them. None of these changes a run:
`place` is read only by the weather lookup, `lenses` only on Auto sources
(`orchestrator._chosen_lenses`), `image_query` only when a picture is wanted."""
label = dict(label)
if label["answer_shape"] != "weather":
label["place"] = ""
if not settings["auto_lenses"]:
label["lenses"] = []
if not label["wants_image"]:
label["image_query"] = ""
return label
def load(prompts_path: str, labels_path: str):
labels = {}
if os.path.exists(labels_path):
for line in open(labels_path):
row = json.loads(line)
labels[row["id"]] = row["label"]
for line in open(prompts_path):
row = json.loads(line)
if row["id"] in labels:
yield row, labels[row["id"]]
def write(rows: list[dict], path: str, private: bool) -> None:
with open(path, "w") as f:
for row in rows:
f.write(json.dumps(row, ensure_ascii=False) + "\n")
if private:
os.chmod(path, 0o600)
def main() -> None:
sets = [
(os.path.join(HERE, "data", "prompts.jsonl"), os.path.join(HERE, "data", "labels.jsonl"),
os.path.join(HERE, "data"), "", False),
(os.path.join(HERE, "data", "private", "prompts_real.jsonl"),
os.path.join(HERE, "data", "private", "labels_real.jsonl"),
os.path.join(HERE, "data", "private"), "_real", True),
]
for prompts_path, labels_path, folder, suffix, private in sets:
kept = {"train": [], "dev": []}
dropped = Counter()
for row, label in load(prompts_path, labels_path):
bad = problems(label, row["settings"])
if bad:
dropped.update(bad)
continue
label = neutral(label, row["settings"])
answer = json.dumps({k: label[k] for k in ORDER}, ensure_ascii=False)
kept[row["split"]].append({"id": row["id"], "cat": row["cat"], "lang": row["lang"],
"settings": row["settings"],
"messages": row["messages"] + [{"role": "assistant", "content": answer}]})
write(kept["train"], os.path.join(folder, f"train{suffix}.jsonl"), private)
write(kept["dev"], os.path.join(folder, f"valid{suffix}.jsonl"), private)
print(f"{os.path.basename(labels_path)}: {len(kept['train'])} train, {len(kept['dev'])} valid, "
f"dropped {sum(dropped.values())} problems {dict(dropped)}")
if __name__ == "__main__":
main()