"""Check the teacher's labels and write the training files. A label is dropped, never repaired, when it breaks a rule the app or the prompt states: an unknown shape, recency or complexity value, a facet name used as a shape, facets beside a shape, more queries than the depth allows, a URL in a query, a picture without a search, queries without a search. A repaired label would teach the student an answer the teacher never gave. Three fields the app ignores in a given situation are emptied (`neutral`). The answer is written in schema order, which is the order strict JSON mode fills it in and the order the student will generate it in. python3 -I make_dataset.py data/train.jsonl, data/valid.jsonl synthetic only (publishable) data/private/train_real.jsonl, data/private/valid_real.jsonl """ from __future__ import annotations import json import os from collections import Counter HERE = os.path.dirname(os.path.abspath(__file__)) SCHEMA = json.load(open(os.path.join(HERE, "eval", "schema.json"))) ORDER = list(SCHEMA["properties"]) SHAPES = {"", "figure", "series", "table", "image", "match", "weather", "diagram", "profile", "steps", "comparison", "explainer"} FACETS = {"facts", "stats", "timeline", "proscons", "list", "grid", "callout"} RECENCY = {"", "day", "week", "month", "year"} COMPLEXITY = {"simple", "standard", "research"} LENSES = {"web", "scholar", "news", "docs", "reference", "data", "library"} def problems(label: dict, settings: dict) -> list[str]: out = [] if set(label) != set(ORDER): out.append("fields") return out if label["answer_shape"] not in SHAPES: out.append("facet as shape" if label["answer_shape"] in FACETS else "shape") if label["recency"] not in RECENCY: out.append("recency") if label["complexity"] not in COMPLEXITY: out.append("complexity") if not set(label["facets"]) <= FACETS: out.append("facet") if label["facets"] and label["answer_shape"]: out.append("facets beside shape") if not set(label["lenses"]) <= LENSES: out.append("lens") queries = label["search_queries"] if len(queries) > settings["queries"]: out.append("too many queries") if any("://" in q for q in queries): out.append("url in query") if label["wants_image"] and not label["needs_search"]: out.append("image without search") if not label["needs_search"] and queries: out.append("queries without search") if label["needs_search"] and not queries: out.append("search without queries") if label["answer_shape"] == "weather" and not label["place"].strip(): out.append("weather without place") return out def neutral(label: dict, settings: dict) -> dict: """Empty the fields the app ignores in this situation, so the student is not taught the teacher's noise in them. None of these changes a run: `place` is read only by the weather lookup, `lenses` only on Auto sources (`orchestrator._chosen_lenses`), `image_query` only when a picture is wanted.""" label = dict(label) if label["answer_shape"] != "weather": label["place"] = "" if not settings["auto_lenses"]: label["lenses"] = [] if not label["wants_image"]: label["image_query"] = "" return label def load(prompts_path: str, labels_path: str): labels = {} if os.path.exists(labels_path): for line in open(labels_path): row = json.loads(line) labels[row["id"]] = row["label"] for line in open(prompts_path): row = json.loads(line) if row["id"] in labels: yield row, labels[row["id"]] def write(rows: list[dict], path: str, private: bool) -> None: with open(path, "w") as f: for row in rows: f.write(json.dumps(row, ensure_ascii=False) + "\n") if private: os.chmod(path, 0o600) def main() -> None: sets = [ (os.path.join(HERE, "data", "prompts.jsonl"), os.path.join(HERE, "data", "labels.jsonl"), os.path.join(HERE, "data"), "", False), (os.path.join(HERE, "data", "private", "prompts_real.jsonl"), os.path.join(HERE, "data", "private", "labels_real.jsonl"), os.path.join(HERE, "data", "private"), "_real", True), ] for prompts_path, labels_path, folder, suffix, private in sets: kept = {"train": [], "dev": []} dropped = Counter() for row, label in load(prompts_path, labels_path): bad = problems(label, row["settings"]) if bad: dropped.update(bad) continue label = neutral(label, row["settings"]) answer = json.dumps({k: label[k] for k in ORDER}, ensure_ascii=False) kept[row["split"]].append({"id": row["id"], "cat": row["cat"], "lang": row["lang"], "settings": row["settings"], "messages": row["messages"] + [{"role": "assistant", "content": answer}]}) write(kept["train"], os.path.join(folder, f"train{suffix}.jsonl"), private) write(kept["dev"], os.path.join(folder, f"valid{suffix}.jsonl"), private) print(f"{os.path.basename(labels_path)}: {len(kept['train'])} train, {len(kept['dev'])} valid, " f"dropped {sum(dropped.values())} problems {dict(dropped)}") if __name__ == "__main__": main()