Buckets:
| """Re-split policy_v2: pool train+validation, keep 10 domains for train/val, 2 held-out domains for test. | |
| Validation is drawn per domain, as whole cases (so originals and their edits stay together), choosing | |
| the case subset in each domain whose size is closest to VAL_FRACTION of that domain and whose labels | |
| are closest to balanced. test_ood.jsonl (insurance_claims, travel_booking) is copied unchanged. | |
| """ | |
| import collections | |
| import hashlib | |
| import itertools | |
| import json | |
| import random | |
| import shutil | |
| import sys | |
| from pathlib import Path | |
| SRC, DST = Path(sys.argv[1]), Path(sys.argv[2]) | |
| VAL_FRACTION, SEED = 0.10, 19 | |
| LABELS = ("followed", "violated", "not_applicable") | |
| def read(path): | |
| return [json.loads(line) for line in path.open(encoding="utf-8")] | |
| def write(path, rows): | |
| with path.open("w", encoding="utf-8") as handle: | |
| for row in rows: | |
| handle.write(json.dumps(row, ensure_ascii=False) + "\n") | |
| pool = read(SRC / "train.jsonl") + read(SRC / "validation.jsonl") | |
| test = read(SRC / "test_ood.jsonl") | |
| assert not any(row["held_out"] for row in pool) | |
| train_domains = sorted({row["domain"] for row in pool}) | |
| test_domains = sorted({row["domain"] for row in test}) | |
| assert not set(train_domains) & set(test_domains) | |
| by_case = collections.defaultdict(list) | |
| for row in pool: | |
| by_case[row["case_id"]].append(row) | |
| cases_by_domain = collections.defaultdict(list) | |
| for case_id in sorted(by_case): | |
| cases_by_domain[by_case[case_id][0]["domain"]].append(case_id) | |
| rng = random.Random(SEED) | |
| val_cases = [] | |
| for domain in train_domains: | |
| cases = cases_by_domain[domain] | |
| total = sum(len(by_case[c]) for c in cases) | |
| target = VAL_FRACTION * total | |
| best = None | |
| for k in range(1, len(cases)): # always keep at least one case in train | |
| for combo in itertools.combinations(cases, k): | |
| counts = collections.Counter(r["label"] for c in combo for r in by_case[c]) | |
| size = sum(counts.values()) | |
| if size < 0.6 * target or size > 1.5 * target: | |
| continue | |
| imbalance = (max(counts[l] for l in LABELS) - min(counts[l] for l in LABELS)) / size | |
| score = abs(size - target) / target + imbalance + rng.random() * 1e-6 | |
| if best is None or score < best[0]: | |
| best = (score, combo) | |
| if best is None: # no subset in range: take the single case closest to target | |
| best = (0, (min(cases, key=lambda c: abs(len(by_case[c]) - target)),)) | |
| val_cases.extend(best[1]) | |
| val_set = set(val_cases) | |
| train = [r for r in pool if r["case_id"] not in val_set] | |
| validation = [r for r in pool if r["case_id"] in val_set] | |
| rng.shuffle(train) | |
| rng.shuffle(validation) | |
| DST.mkdir(parents=True, exist_ok=False) | |
| write(DST / "train.jsonl", train) | |
| write(DST / "validation.jsonl", validation) | |
| shutil.copyfile(SRC / "test_ood.jsonl", DST / "test_ood.jsonl") | |
| def summary(rows): | |
| return { | |
| "pairs": len(rows), | |
| "cases": len({r["case_id"] for r in rows}), | |
| "labels": dict(collections.Counter(r["label"] for r in rows)), | |
| "domains": { | |
| d: {"pairs": n, "labels": dict(collections.Counter(r["label"] for r in rows if r["domain"] == d))} | |
| for d, n in sorted(collections.Counter(r["domain"] for r in rows).items()) | |
| }, | |
| } | |
| manifest = { | |
| "selection_policy": "Pooled policy_v2 train+validation; 10 domains split by whole case into train and a " | |
| "per-domain ~10% validation; 2 held-out domains form test_ood (copied unchanged from policy_v2).", | |
| "source": str(SRC), | |
| "source_sha256": { | |
| name: hashlib.sha256((SRC / name).read_bytes()).hexdigest() | |
| for name in ("train.jsonl", "validation.jsonl", "test_ood.jsonl") | |
| }, | |
| "seed": SEED, | |
| "val_fraction_per_domain": VAL_FRACTION, | |
| "train_domains": train_domains, | |
| "test_domains": test_domains, | |
| "validation_cases": sorted(val_cases), | |
| "splits": {"train": summary(train), "validation": summary(validation), "test_ood": summary(test)}, | |
| } | |
| (DST / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8") | |
| print(json.dumps({k: {kk: v[kk] for kk in ("pairs", "cases", "labels")} for k, v in manifest["splits"].items()}, indent=2)) | |
| for d, v in manifest["splits"]["validation"]["domains"].items(): | |
| print(f" val {d:20s} {v['pairs']:5d} {v['labels']}") | |
Xet Storage Details
- Size:
- 4.32 kB
- Xet hash:
- b0a8b1fa92c9df0811080a815062d83c9ce01788e65eb9d0221ce21c763a8a8f
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.