| """Public Jev-style typed-decision corpora for training (DATA["text3"], train split only), decontaminated: |
| * tasksource/procedural-typed-decisions (Apache-2.0): structured states, several typed questions, exact labels |
| * n4ze3m/typed-decisions-synth (MIT): 149 domains, multi-question, soft teacher labels |
| * tasksource/tasksource-jev-typed-decisions: real labels from 670 sources -- every source that overlaps a benchmark |
| we report (AG News, emotion, Banking77, MASSIVE, XNLI, JevBench, typed-decisions) is excluded |
| Every candidate whose state matches an eval state (normalised prefix hash) is dropped as well. |
| """ |
| import hashlib, json, random, re |
| from datasets import load_dataset, get_dataset_config_names |
| from mmjev import Seg, options_of |
|
|
| rng = random.Random(7) |
| BANNED_SRC = re.compile(r"ag_?news|emotion|banking|massive|xnli|jev|typed|clinc|amazon_massive", re.I) |
|
|
|
|
| def norm(t): |
| return hashlib.md5(re.sub(r"\s+", " ", str(t).lower())[:300].encode()).hexdigest() |
|
|
|
|
| EVAL_HASH = {norm(s.data) for v in DATA.values() for r in v if r["split"] == "eval" |
| for s in r["state"] if s.kind == "text"} |
| out, dropped = [], {"contaminated": 0, "banned_source": 0, "bad": 0} |
|
|
|
|
| def convert_q(q): |
| t, crit = q["type"], q.get("criteria") |
| if t == "noul": |
| c = crit or {} |
| return {"type": "noul", "instructions": q["instructions"], |
| "criteria": {"no": c.get("false", "no"), "yes": c.get("true", "yes")}} |
| if t == "score": |
| return {"type": "score", "instructions": q["instructions"], "criteria": [str(x) for x in crit]} |
| if isinstance(crit, list): |
| crit = {str(x): "" for x in crit} |
| return {"type": "choice", "instructions": q["instructions"], "criteria": {str(k): str(v or "") for k, v in crit.items()}} |
|
|
|
|
| def soft_target(qq, ans): |
| labels, _ = options_of(qq) |
| if qq["type"] == "noul": |
| p = float(ans.get("noul", ans.get("probabilities", {}).get("true", 0.5)) if isinstance(ans, dict) else ans) |
| return [1 - p, p] |
| pr = ans.get("probabilities", {}) if isinstance(ans, dict) else {} |
| v = [float(pr.get(l, 0.0)) for l in labels] |
| if sum(v) <= 0 and isinstance(ans, dict) and "choice" in ans and ans["choice"] in labels: |
| v = [1.0 if l == ans["choice"] else 0.0 for l in labels] |
| s = sum(v) |
| return [x / s for x in v] if s > 0 else None |
|
|
|
|
| def add_case(task, state, qs, answers): |
| if norm(state) in EVAL_HASH: |
| dropped["contaminated"] += 1; return |
| Q, Y, T = [], [], [] |
| for qid, q in qs.items(): |
| try: |
| qq = convert_q(q); tg = soft_target(qq, answers[qid]) |
| except Exception: |
| tg = None |
| if tg is None or len(tg) < 2 or len(tg) > 80: |
| continue |
| Q.append(qq); T.append(tg); Y.append(int(max(range(len(tg)), key=lambda i: tg[i]))) |
| if not Q: |
| dropped["bad"] += 1; return |
| r = rec(task, "text", "train", [Seg("text", state[:4000])], Q[:6], Y[:6], T[:6]) |
| r["raw"] = None |
| out.append(r) |
|
|
|
|
| def procedural(per_cfg=160): |
| for cfg in get_dataset_config_names("tasksource/procedural-typed-decisions"): |
| ds = load_dataset("tasksource/procedural-typed-decisions", cfg, split="train").shuffle(seed=7) |
| for ex in ds.select(range(min(per_cfg, len(ds)))): |
| add_case(f"procedural:{cfg}", ex["state"], json.loads(ex["questions"]), json.loads(ex["answers"])) |
|
|
|
|
| def synth(n=1800): |
| ds = load_dataset("json", data_files="hf://datasets/n4ze3m/typed-decisions-synth/data/train.jsonl", split="train") |
| for ex in ds.shuffle(seed=7).select(range(min(n, len(ds)))): |
| st = ex["state"] if isinstance(ex["state"], str) else json.dumps(ex["state"], ensure_ascii=False) |
| add_case("typed_synth", st, json.loads(ex["questions"]) if isinstance(ex["questions"], str) else ex["questions"], |
| json.loads(ex["teacher"]) if isinstance(ex["teacher"], str) else ex["teacher"]) |
|
|
|
|
| def tasksource(n=2500): |
| ds = load_dataset("tasksource/tasksource-jev-typed-decisions", split="train", streaming=True).shuffle(seed=7, buffer_size=20000) |
| got = 0 |
| for ex in ds: |
| if got >= n: |
| break |
| if BANNED_SRC.search(ex["source"] or ""): |
| dropped["banned_source"] += 1; continue |
| try: |
| opts = json.loads(ex["options"]) if ex["options"] else [] |
| tgt = json.loads(ex["target"]) |
| except Exception: |
| continue |
| kind, st = ex["kind"], ex["state"] or "" |
| if len(st) > 3000 or norm(st) in EVAL_HASH: |
| dropped["contaminated" if norm(st) in EVAL_HASH else "bad"] += 1; continue |
| if kind == "noul": |
| p = float(tgt[-1] if isinstance(tgt, list) else tgt) |
| q = {"type": "noul", "instructions": ex["question"]}; t = [1 - p, p] |
| elif kind in ("choice", "score") and 2 <= len(opts) <= 40 and len(tgt) == len(opts): |
| if kind == "score": |
| q = {"type": "score", "instructions": ex["question"], "criteria": [str(o)[:120] for o in opts]} |
| elif max(len(str(o)) for o in opts) > 60: |
| keys = [chr(65 + i) for i in range(len(opts))] |
| q = {"type": "choice", "instructions": ex["question"], "criteria": {k: str(o)[:300] for k, o in zip(keys, opts)}} |
| else: |
| q = {"type": "choice", "instructions": ex["question"], "criteria": {str(o): "" for o in opts}} |
| if len(q["criteria"]) != len(opts): |
| continue |
| t = [float(x) for x in tgt]; s = sum(t) |
| if s <= 0: |
| continue |
| t = [x / s for x in t] |
| else: |
| continue |
| r = rec(f"tasksource:{ex['source'].split('/')[0]}", "text", "train", [Seg("text", st)], q, |
| int(max(range(len(t)), key=lambda i: t[i])), [t]) |
| r["raw"] = None |
| out.append(r); got += 1 |
|
|
|
|
| for f in (procedural, synth): |
| try: |
| f() |
| log(f"[text3] {f.__name__}: total {len(out)} dropped {dropped}") |
| except Exception: |
| import traceback |
| log(f"[text3] {f.__name__} FAILED {traceback.format_exc()[-600:]}") |
| DATA["text3"] = out |
| log(f"[text3] DONE {len(out)} records, dropped {dropped}") |
|
|
