Buckets:

Rishik001's picture
download
raw
4.32 kB
"""Re-split policy_v2: pool train+validation, keep 10 domains for train/val, 2 held-out domains for test.
Validation is drawn per domain, as whole cases (so originals and their edits stay together), choosing
the case subset in each domain whose size is closest to VAL_FRACTION of that domain and whose labels
are closest to balanced. test_ood.jsonl (insurance_claims, travel_booking) is copied unchanged.
"""
import collections
import hashlib
import itertools
import json
import random
import shutil
import sys
from pathlib import Path
SRC, DST = Path(sys.argv[1]), Path(sys.argv[2])
VAL_FRACTION, SEED = 0.10, 19
LABELS = ("followed", "violated", "not_applicable")
def read(path):
return [json.loads(line) for line in path.open(encoding="utf-8")]
def write(path, rows):
with path.open("w", encoding="utf-8") as handle:
for row in rows:
handle.write(json.dumps(row, ensure_ascii=False) + "\n")
pool = read(SRC / "train.jsonl") + read(SRC / "validation.jsonl")
test = read(SRC / "test_ood.jsonl")
assert not any(row["held_out"] for row in pool)
train_domains = sorted({row["domain"] for row in pool})
test_domains = sorted({row["domain"] for row in test})
assert not set(train_domains) & set(test_domains)
by_case = collections.defaultdict(list)
for row in pool:
by_case[row["case_id"]].append(row)
cases_by_domain = collections.defaultdict(list)
for case_id in sorted(by_case):
cases_by_domain[by_case[case_id][0]["domain"]].append(case_id)
rng = random.Random(SEED)
val_cases = []
for domain in train_domains:
cases = cases_by_domain[domain]
total = sum(len(by_case[c]) for c in cases)
target = VAL_FRACTION * total
best = None
for k in range(1, len(cases)): # always keep at least one case in train
for combo in itertools.combinations(cases, k):
counts = collections.Counter(r["label"] for c in combo for r in by_case[c])
size = sum(counts.values())
if size < 0.6 * target or size > 1.5 * target:
continue
imbalance = (max(counts[l] for l in LABELS) - min(counts[l] for l in LABELS)) / size
score = abs(size - target) / target + imbalance + rng.random() * 1e-6
if best is None or score < best[0]:
best = (score, combo)
if best is None: # no subset in range: take the single case closest to target
best = (0, (min(cases, key=lambda c: abs(len(by_case[c]) - target)),))
val_cases.extend(best[1])
val_set = set(val_cases)
train = [r for r in pool if r["case_id"] not in val_set]
validation = [r for r in pool if r["case_id"] in val_set]
rng.shuffle(train)
rng.shuffle(validation)
DST.mkdir(parents=True, exist_ok=False)
write(DST / "train.jsonl", train)
write(DST / "validation.jsonl", validation)
shutil.copyfile(SRC / "test_ood.jsonl", DST / "test_ood.jsonl")
def summary(rows):
return {
"pairs": len(rows),
"cases": len({r["case_id"] for r in rows}),
"labels": dict(collections.Counter(r["label"] for r in rows)),
"domains": {
d: {"pairs": n, "labels": dict(collections.Counter(r["label"] for r in rows if r["domain"] == d))}
for d, n in sorted(collections.Counter(r["domain"] for r in rows).items())
},
}
manifest = {
"selection_policy": "Pooled policy_v2 train+validation; 10 domains split by whole case into train and a "
"per-domain ~10% validation; 2 held-out domains form test_ood (copied unchanged from policy_v2).",
"source": str(SRC),
"source_sha256": {
name: hashlib.sha256((SRC / name).read_bytes()).hexdigest()
for name in ("train.jsonl", "validation.jsonl", "test_ood.jsonl")
},
"seed": SEED,
"val_fraction_per_domain": VAL_FRACTION,
"train_domains": train_domains,
"test_domains": test_domains,
"validation_cases": sorted(val_cases),
"splits": {"train": summary(train), "validation": summary(validation), "test_ood": summary(test)},
}
(DST / "manifest.json").write_text(json.dumps(manifest, indent=2), encoding="utf-8")
print(json.dumps({k: {kk: v[kk] for kk in ("pairs", "cases", "labels")} for k, v in manifest["splits"].items()}, indent=2))
for d, v in manifest["splits"]["validation"]["domains"].items():
print(f" val {d:20s} {v['pairs']:5d} {v['labels']}")

Xet Storage Details

Size:
4.32 kB
·
Xet hash:
b0a8b1fa92c9df0811080a815062d83c9ce01788e65eb9d0221ce21c763a8a8f

Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.