Spaces:
Running on Zero
Running on Zero
File size: 5,576 Bytes
e8c5cc2 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 | """Prove that rows 4 and 5 are a valid controlled comparison.
The headline claim of this project is a delta between two training runs. That
claim is only worth anything if the runs were identical apart from the
intervention. This script checks that mechanically instead of on trust:
1. the two configs differ only in the negatives file, output dir and run name
2. the two datasets have the same size, the same positives, in the same order
3. the negatives really are drawn from different distributions
4. the two training runs used the same seed, epochs, LR and batch geometry
python scripts/verify_ablation.py
"""
from __future__ import annotations
import argparse
import json
import sys
from pathlib import Path
from typing import Any
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from src.config import load_config
from src.data.negatives import read_jsonl
ALLOWED_CONFIG_DIFFS = {"train.negatives_file", "train.output_dir", "run.name"}
def _flatten(node: Any, prefix: str = "") -> dict[str, Any]:
flat: dict[str, Any] = {}
for key, value in node.items():
path = f"{prefix}.{key}" if prefix else key
if isinstance(value, dict):
flat.update(_flatten(value, path))
else:
flat[path] = value
return flat
def check(label: str, passed: bool, detail: str = "") -> bool:
mark = "PASS" if passed else "FAIL"
print(f" [{mark}] {label}" + (f" - {detail}" if detail else ""))
return passed
def main() -> None:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--random-config", default="configs/train_random.yaml")
parser.add_argument("--hard-config", default="configs/train_hard.yaml")
args = parser.parse_args()
results: list[bool] = []
print("\n1. Config comparison")
rand_cfg = _flatten(load_config(args.random_config))
hard_cfg = _flatten(load_config(args.hard_config))
differing = {k for k in set(rand_cfg) | set(hard_cfg) if rand_cfg.get(k) != hard_cfg.get(k)}
unexpected = differing - ALLOWED_CONFIG_DIFFS
results.append(
check(
"configs differ only in negatives file / output dir / run name",
not unexpected,
f"unexpected: {sorted(unexpected)}" if unexpected else f"differ: {sorted(differing)}",
)
)
print("\n2. Dataset comparison")
rand_file = rand_cfg["train.negatives_file"]
hard_file = hard_cfg["train.negatives_file"]
if not (Path(rand_file).exists() and Path(hard_file).exists()):
print(f" [SKIP] datasets not built yet ({rand_file}, {hard_file})")
else:
rand_ds = read_jsonl(rand_file)
hard_ds = read_jsonl(hard_file)
results.append(check("same number of groups", len(rand_ds) == len(hard_ds), f"{len(hard_ds)}"))
results.append(
check(
"identical positives, in identical order",
all(r["positive_id"] == h["positive_id"] for r, h in zip(rand_ds, hard_ds)),
)
)
results.append(
check(
"identical negative counts per group",
all(len(r["negatives"]) == len(h["negatives"]) for r, h in zip(rand_ds, hard_ds)),
f"{len(hard_ds[0]['negatives'])} per group",
)
)
shared = sum(
len(set(r["negative_ids"]) & set(h["negative_ids"])) for r, h in zip(rand_ds, hard_ds)
)
total = sum(len(h["negative_ids"]) for h in hard_ds)
results.append(
check(
"negative sets are substantively different",
shared / total < 0.05,
f"{shared}/{total} shared ({shared / total * 100:.1f}%)",
)
)
print("\n3. Training run comparison")
states = {}
for name, cfg in (("random", rand_cfg), ("hard", hard_cfg)):
candidates = sorted(Path(cfg["train.output_dir"]).glob("checkpoint-*/trainer_state.json"))
if candidates:
states[name] = json.loads(candidates[-1].read_text(encoding="utf-8"))
if len(states) < 2:
print(f" [SKIP] need both runs trained (found: {sorted(states)})")
else:
rand_args = states["random"].get("train_batch_size"), states["random"].get("num_train_epochs")
hard_args = states["hard"].get("train_batch_size"), states["hard"].get("num_train_epochs")
results.append(check("same batch size and epoch count", rand_args == hard_args, f"{hard_args}"))
results.append(
check(
"same number of optimizer steps",
states["random"].get("global_step") == states["hard"].get("global_step"),
f"{states['hard'].get('global_step')} steps",
)
)
rand_lr = [e.get("learning_rate") for e in states["random"].get("log_history", []) if "learning_rate" in e]
hard_lr = [e.get("learning_rate") for e in states["hard"].get("log_history", []) if "learning_rate" in e]
results.append(
check(
"identical learning-rate schedule",
rand_lr == hard_lr and bool(rand_lr),
f"{len(hard_lr)} logged points, peak {max(hard_lr):.2e}" if hard_lr else "no lr logged",
)
)
print()
if all(results):
print(f"ABLATION VALID - {len(results)} checks passed. Rows 4 and 5 are comparable.\n")
else:
print(f"ABLATION INVALID - {results.count(False)} of {len(results)} checks failed.\n")
raise SystemExit(1)
if __name__ == "__main__":
main()
|