Buckets:
| """The honest transfer test: unseen MODEL *and* unseen TRAJECTORY at once. | |
| Why this exists. A plain leave-model-out split is not enough on this data. Every | |
| duel scores two models over ~2060 turns, and slices overlap heavily across | |
| duels, so the same turn appears under many models. Holding out one model still | |
| leaves its turns in training under everyone else's rows — and a classifier can | |
| then learn "turn X is a hard turn" rather than "this thought is a good thought". | |
| That inflates leave-model-out AUC and would evaporate in production, where the | |
| slice is freshly seeded per duel (`blake2b(reveal_block_hash || hotkey)`) and | |
| every turn is new. | |
| So each fold holds out BOTH: test = one model, restricted to trajectories that | |
| never appear in training under any model. | |
| Also reports a TURN-MEMORISATION control: train with the held-out model's turns | |
| present (trajectory leak allowed) vs absent. The gap between the two is the size | |
| of the leak. | |
| python3 harness/reason_transfer.py | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import collections | |
| import pathlib | |
| import sys | |
| import numpy as np | |
| sys.path.insert(0, str(pathlib.Path(__file__).resolve().parent)) | |
| import reason_signal as R | |
| def tfidf_auc(train, test): | |
| from sklearn.feature_extraction.text import TfidfVectorizer | |
| from sklearn.linear_model import LogisticRegression | |
| ytr = np.array([r["reason"] > 0 for r in train], dtype=int) | |
| yte = np.array([r["reason"] > 0 for r in test], dtype=int) | |
| if len(set(yte)) < 2 or len(set(ytr)) < 2: | |
| return None | |
| tv = TfidfVectorizer(min_df=5, ngram_range=(1, 2), max_features=60000, | |
| sublinear_tf=True) | |
| A = tv.fit_transform([r["z_a"] for r in train]) | |
| B = tv.transform([r["z_a"] for r in test]) | |
| lr = LogisticRegression(max_iter=3000, class_weight="balanced") | |
| lr.fit(A, ytr) | |
| p = lr.predict_proba(B)[:, 1] | |
| return R.metrics(yte, p), p, yte | |
| def main() -> None: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--data", default=str(R.ROOT / "data" / "reason.jsonl.gz")) | |
| ap.add_argument("--folds", type=int, default=5) | |
| args = ap.parse_args() | |
| rows = R.load(args.data, drop_silent=True) | |
| by_model = collections.defaultdict(list) | |
| for r in rows: | |
| by_model[r["model"]].append(r) | |
| big = sorted(by_model, key=lambda m: -len(by_model[m]))[: args.folds] | |
| print(f"{len(rows)} pairs, {len(by_model)} models; folds over the " | |
| f"{len(big)} largest\n") | |
| print(f"{'held-out model':<40} {'n_test':>7} {'AUCleak':>8} {'AUChonest':>10} " | |
| f"{'acc':>7} {'prec@top25%':>12}") | |
| leak, honest = [], [] | |
| for m in big: | |
| test_all = by_model[m] | |
| test_trajs = {r["traj_id"] for r in test_all} | |
| other = [r for r in rows if r["model"] != m] | |
| # (a) model-disjoint only — turns of the held-out model still in train | |
| a = tfidf_auc(other, test_all) | |
| # (b) model AND trajectory disjoint: drop every training row whose | |
| # trajectory appears in the test set, then test on what is left | |
| train_b = [r for r in other if r["traj_id"] not in test_trajs] | |
| if not train_b: | |
| print(f"{m[:38]:<40} no disjoint training trajectories — skipped") | |
| continue | |
| b = tfidf_auc(train_b, test_all) | |
| if a is None or b is None: | |
| continue | |
| ma, _, _ = a | |
| mb, pb, yb = b | |
| k = max(int(0.25 * len(pb)), 1) | |
| top = np.argsort(pb)[-k:] | |
| prec = float(yb[top].mean()) | |
| leak.append(ma["auc"]); honest.append(mb["auc"]) | |
| print(f"{m[:38]:<40} {mb['n']:>7} {ma['auc']:>8.4f} {mb['auc']:>10.4f} " | |
| f"{mb['acc']:>7.4f} {prec:>12.4f}") | |
| if honest: | |
| print(f"\n mean AUC with turn leak allowed : {np.mean(leak):.4f}") | |
| print(f" mean AUC model+trajectory disjoint: {np.mean(honest):.4f}") | |
| print(f" size of the turn-memorisation leak: {np.mean(leak)-np.mean(honest):+.4f}") | |
| print("\n A large gap means the classifier was scoring TURNS, not THOUGHTS,") | |
| print(" and would not transfer to a freshly seeded duel slice.") | |
| if __name__ == "__main__": | |
| main() | |
Xet Storage Details
- Size:
- 4.15 kB
- Xet hash:
- 76f023bdc4cb771dba301d09a51a199c1a66a16481ff6010c9741f4826b9a95c
·
Xet efficiently stores files, intelligently splitting them into unique chunks and accelerating uploads and downloads. More info.