flip-2 / train.py
keethesh's picture
Upload folder using huggingface_hub
313c5a1 verified
Raw History Blame Contribute Delete
2.27 kB
# /// script
# dependencies = ["torch", "safetensors", "matplotlib"]
# ///
"""Train Flip-2: 2 parameters, 3 answers (yes / no / maybe). Run: uv run train.py"""
import json
import torch
import matplotlib
matplotlib.use("Agg")
import matplotlib.pyplot as plt
from safetensors.torch import save_file
LABELS = ["yes", "no", "maybe"]
torch.manual_seed(0)
# Flip-Bench-3: 10k questions (content is irrelevant, ground truth is independent of it),
# labelled yes/no/maybe uniformly at random. Split 8k train / 2k test.
y = torch.randint(0, 3, (10_000,))
y_train, y_test = y[:8000], y[8000:]
# logits = [w[0], w[1], 0]: "maybe" is the reference class, so 3 answers cost 2 parameters.
w = torch.nn.Parameter(torch.randn(2) * 2) # start confidently wrong
opt = torch.optim.Adam([w], lr=0.05)
def logits():
return torch.cat([w, torch.zeros(1)])
losses = []
for step in range(500):
loss = torch.nn.functional.cross_entropy(logits().expand(len(y_train), 3), y_train)
opt.zero_grad()
loss.backward()
opt.step()
losses.append(loss.item())
p = torch.softmax(logits().detach(), 0)
pred = torch.multinomial(p, len(y_test), replacement=True)
acc3 = (pred == y_test).float().mean().item()
# yes/no bench: only yes/no questions, "maybe" is always wrong
yn = y_test != 2
acc_yn = (pred[yn] == y_test[yn]).float().mean().item()
# selective accuracy: accuracy on the questions where it actually commits to yes/no
commit = (pred != 2) & yn
acc_sel = (pred[commit] == y_test[commit]).float().mean().item()
results = {
"parameters": w.numel(),
"final_train_loss": losses[-1],
"ln3": float(torch.log(torch.tensor(3.0))),
"probs": dict(zip(LABELS, p.tolist())),
"flip_bench_3_acc": acc3,
"yes_no_bench_acc": acc_yn,
"selective_acc_when_committing": acc_sel,
"abstain_rate": (pred == 2).float().mean().item(),
}
print(json.dumps(results, indent=2))
save_file({"w": w.detach().clone()}, "model.safetensors")
json.dump(results, open("results.json", "w"), indent=2)
plt.figure(figsize=(6, 3.5))
plt.plot(losses)
plt.axhline(results["ln3"], ls="--", c="gray", label="ln 3 (the floor)")
plt.xlabel("step"), plt.ylabel("train loss"), plt.title("Flip-2 training loss"), plt.legend()
plt.tight_layout(), plt.savefig("loss.png", dpi=150)