Download train.py from keethesh/flip-2: direct link, hf CLI and curl.
- Browser
- Download file 2.27 kB
-
https://huggingface.co/keethesh/flip-2/resolve/main/train.py
- Command line
-
hf download hf://keethesh/flip-2/train.py
-
curl -L -o train.py https://huggingface.co/keethesh/flip-2/resolve/main/train.py
2.27 kB
| # /// script | |
| # dependencies = ["torch", "safetensors", "matplotlib"] | |
| # /// | |
| """Train Flip-2: 2 parameters, 3 answers (yes / no / maybe). Run: uv run train.py""" | |
| import json | |
| import torch | |
| import matplotlib | |
| matplotlib.use("Agg") | |
| import matplotlib.pyplot as plt | |
| from safetensors.torch import save_file | |
| LABELS = ["yes", "no", "maybe"] | |
| torch.manual_seed(0) | |
| # Flip-Bench-3: 10k questions (content is irrelevant, ground truth is independent of it), | |
| # labelled yes/no/maybe uniformly at random. Split 8k train / 2k test. | |
| y = torch.randint(0, 3, (10_000,)) | |
| y_train, y_test = y[:8000], y[8000:] | |
| # logits = [w[0], w[1], 0]: "maybe" is the reference class, so 3 answers cost 2 parameters. | |
| w = torch.nn.Parameter(torch.randn(2) * 2) # start confidently wrong | |
| opt = torch.optim.Adam([w], lr=0.05) | |
| def logits(): | |
| return torch.cat([w, torch.zeros(1)]) | |
| losses = [] | |
| for step in range(500): | |
| loss = torch.nn.functional.cross_entropy(logits().expand(len(y_train), 3), y_train) | |
| opt.zero_grad() | |
| loss.backward() | |
| opt.step() | |
| losses.append(loss.item()) | |
| p = torch.softmax(logits().detach(), 0) | |
| pred = torch.multinomial(p, len(y_test), replacement=True) | |
| acc3 = (pred == y_test).float().mean().item() | |
| # yes/no bench: only yes/no questions, "maybe" is always wrong | |
| yn = y_test != 2 | |
| acc_yn = (pred[yn] == y_test[yn]).float().mean().item() | |
| # selective accuracy: accuracy on the questions where it actually commits to yes/no | |
| commit = (pred != 2) & yn | |
| acc_sel = (pred[commit] == y_test[commit]).float().mean().item() | |
| results = { | |
| "parameters": w.numel(), | |
| "final_train_loss": losses[-1], | |
| "ln3": float(torch.log(torch.tensor(3.0))), | |
| "probs": dict(zip(LABELS, p.tolist())), | |
| "flip_bench_3_acc": acc3, | |
| "yes_no_bench_acc": acc_yn, | |
| "selective_acc_when_committing": acc_sel, | |
| "abstain_rate": (pred == 2).float().mean().item(), | |
| } | |
| print(json.dumps(results, indent=2)) | |
| save_file({"w": w.detach().clone()}, "model.safetensors") | |
| json.dump(results, open("results.json", "w"), indent=2) | |
| plt.figure(figsize=(6, 3.5)) | |
| plt.plot(losses) | |
| plt.axhline(results["ln3"], ls="--", c="gray", label="ln 3 (the floor)") | |
| plt.xlabel("step"), plt.ylabel("train loss"), plt.title("Flip-2 training loss"), plt.legend() | |
| plt.tight_layout(), plt.savefig("loss.png", dpi=150) | |