"""Extra TEXT training data, disjoint from every benchmark we report (no AG News, DAIR Emotion, Banking77, MASSIVE, XNLI, JevBench, typed-decisions). Appended to DATA["text2"] (train split only).""" import random, traceback from datasets import load_dataset from mmjev import Seg rng = random.Random(1) out = [] def add(task, state, q, y): r = rec(task, "text", "train", [Seg("text", state)], q, y) r["raw"] = None out.append(r) def safe(fn): try: fn() except Exception: log("text2 skip", fn.__name__, traceback.format_exc()[-300:]) def mnli(n=900): ds = load_dataset("nyu-mll/multi_nli", split="train").shuffle(seed=1).select(range(n)) crit = {"entailment": "the premise implies the hypothesis", "neutral": "the premise neither implies nor contradicts the hypothesis", "contradiction": "the premise contradicts the hypothesis"} for i, ex in enumerate(ds): if ex["label"] not in (0, 1, 2): continue st = f"Premise: {ex['premise']}\nHypothesis: {ex['hypothesis']}" if i % 2: add("mnli", st, {"type": "choice", "instructions": "What is the relation between the premise and the hypothesis?", "criteria": crit}, ex["label"]) else: add("mnli", st, {"type": "noul", "instructions": "Does the premise imply the hypothesis?"}, int(ex["label"] == 0)) def clinc(n=700): ds = load_dataset("clinc/clinc_oos", "plus", split="train").shuffle(seed=1) names = ds.features["intent"].names ds = ds.filter(lambda e: names[e["intent"]] != "oos").select(range(n)) for ex in ds: k = rng.choice([8, 16, 32, 64, 100]) gold = names[ex["intent"]] opts = [gold] + rng.sample([x for x in names if x not in (gold, "oos")], k - 1) rng.shuffle(opts) add("clinc150", ex["text"], {"type": "choice", "instructions": "Which single label best describes the input text?", "criteria": {o.replace("_", " "): "" for o in opts}}, opts.index(gold)) def goemo(n=700): ds = load_dataset("google-research-datasets/go_emotions", "simplified", split="train").shuffle(seed=1) names = ds.features["labels"].feature.names ds = ds.filter(lambda e: len(e["labels"]) == 1 and names[e["labels"][0]] != "neutral").select(range(n)) for ex in ds: gold = names[ex["labels"][0]] k = rng.choice([6, 10, 27]) opts = [gold] + rng.sample([x for x in names if x not in (gold, "neutral")], k - 1) rng.shuffle(opts) add("goemotions", ex["text"], {"type": "choice", "instructions": "Which emotion does the text express?", "criteria": {o: "" for o in opts}}, opts.index(gold)) def yahoo(n=500): ds = load_dataset("community-datasets/yahoo_answers_topics", split="train").shuffle(seed=1).select(range(n)) names = ds.features["topic"].names for ex in ds: st = f"Question: {ex['question_title']} {ex['question_content']}\nAnswer: {ex['best_answer']}"[:1500] add("yahoo_topics", st, {"type": "choice", "instructions": "Which topic is this post about?", "criteria": {x: "" for x in names}}, ex["topic"]) def yelp(n=500): ds = load_dataset("Yelp/yelp_review_full", split="train").shuffle(seed=1).select(range(n)) lv = ["1 star, terrible", "2 stars, poor", "3 stars, average", "4 stars, good", "5 stars, excellent"] for ex in ds: add("yelp", ex["text"][:1500], {"type": "score", "instructions": "What star rating does this review give?", "criteria": lv}, ex["label"]) def policies(n=600): """JevBench-style bounded policy checks, programmatic gold.""" rules = [("a receipt is provided", "the customer has a receipt", "the customer has no receipt"), ("the purchase was made within 30 days", "the purchase was 12 days ago", "the purchase was 45 days ago"), ("the item is unused", "the item is unopened", "the item has been used"), ("the account is verified", "the account is verified", "the account is not verified"), ("the amount is below $500", "the amount is $120", "the amount is $1,300"), ("a manager approved it", "a manager approved it", "no manager approval was given"), ("the request comes from the account owner", "the account owner made the request", "a third party made the request")] actions = ["issue a refund", "approve the transfer", "grant the exception", "release the order"] for _ in range(n): rs = rng.sample(rules, rng.randint(2, 3)) ok = [rng.random() < 0.7 for _ in rs] facts = [r[1] if o else r[2] for r, o in zip(rs, ok)] rng.shuffle(facts) act = rng.choice(actions) st = (f"Policy: we {act} only if " + " and ".join(r[0] for r in rs) + ". Case: " + "; ".join(facts) + ".") if rng.random() < 0.5: add("policy_syn", st, {"type": "noul", "instructions": f"Under the stated policy, may we {act}?"}, int(all(ok))) else: n_ok = sum(ok) add("policy_syn", st, {"type": "choice", "instructions": "What should happen under the policy?", "criteria": {"allow": "every condition holds", "deny": "at least one condition fails"}}, 0 if all(ok) else 1) def snake(n=400): """Snake-game states (demo task, disclosed in the card); gold = safe move that best approaches the food, ties broken by free space (flood fill).""" import sys; sys.path.insert(0, "/content") from demos import Snake, DIRS, OPP def space(g, d): hx, hy = g.body[0]; start = (hx + DIRS[d][0], hy + DIRS[d][1]) seen, st, blocked = {start}, [start], set(g.body[:-1]) while st and len(seen) < 60: x, y = st.pop() for dx, dy in DIRS.values(): c = (x + dx, y + dy) if 0 <= c[0] < g.n and 0 <= c[1] < g.n and c not in blocked and c not in seen: seen.add(c); st.append(c) return len(seen) def best(g): hx, hy = g.body[0]; fx, fy = g.food cand = [d for d in DIRS if d != OPP[g.dir] and not g.blocked(d)] if not cand: return None return max(cand, key=lambda d: (space(g, d) >= len(g.body), -(abs(fx - hx - DIRS[d][0]) + abs(fy - hy - DIRS[d][1])), space(g, d))) made = 0 for seed in range(10_000): g = Snake(seed=seed) while g.alive and made < n and g.steps < 120: b = best(g) if b is None: break if rng.random() < 0.35: q = g.question() add("snake_syn", g.state_text(), q, list(q["criteria"]).index(b)); made += 1 g.step(b if rng.random() > 0.1 else rng.choice([d for d in DIRS if d != OPP[g.dir]])) if made >= n: break for f in (mnli, clinc, goemo, yahoo, yelp, policies, snake): safe(f) DATA["text2"] = out