Franok / source /training /third /preference.py
yunusemrejr's picture
Publish source, film and evaluations from 699be55df232
2bff5be verified
Raw History Blame Contribute Delete
5.35 kB
"""Learn a dialogue reward from ranked responses and same-conversation alternatives."""
from collections import Counter
import json
import numpy as np
import torch
from safetensors.torch import load_file, save_file
from torch.nn import functional as F
from franok.preference import ReplyPreference
def dialogue_source(name):
return name in {
"human",
"english-dialogue",
"short-dialogue",
"everyday-en",
"turkish-dialogue",
"turkish-tasks",
"native-chat",
"everyday-tr",
"short-replies",
}
def labelled_states(states, labels, early=None):
# labels begin at the assistant marker and end with the EOS target.
mask = labels != -100
marker = mask.int().argmax(1)
count = mask.sum(1) - 1
if early is not None:
count = count.clamp_max(early)
return ReplyPreference.features(states.detach(), marker, count)
class DialogueReward:
def __init__(self, width, device, root):
self.head = ReplyPreference(width).to(device)
self.opt = torch.optim.AdamW(
self.head.parameters(), lr=0.00065, weight_decay=0.035
)
self.steps = 0
self.total = Counter()
self.root = root
self.root.mkdir(exist_ok=True, parents=True)
if (root / "head.safetensors").exists():
self.head.load_state_dict(load_file(str(root / "head.safetensors")))
def update(self, features, names):
eligible = [i for i, name in enumerate(names) if dialogue_source(name)]
if not eligible:
return
good, bad = features.chunk(2)
self.head.train()
x = self.head(torch.cat((good[eligible], bad[eligible])))
preferred, rejected = x.chunk(2)
loss = F.softplus(rejected - preferred).mean() + 0.02 * x.square().mean()
self.opt.zero_grad(set_to_none=True)
loss.backward()
torch.nn.utils.clip_grad_norm_(
self.head.parameters(), 1.0, error_if_nonfinite=True
)
self.opt.step()
self.steps += 1
self.total.update(names[i] for i in eligible)
def save(self):
file = self.root / "head.next"
save_file(
{
k: v.detach().cpu().contiguous()
for k, v in self.head.state_dict().items()
},
str(file),
)
file.replace(self.root / "head.safetensors")
(self.root / "training.json").write_text(
json.dumps(dict(updates=self.steps, pairs=dict(self.total)), indent=2)
+ "\n"
)
@torch.no_grad()
def evaluate(self, model, conversations):
self.head.eval()
device = model.embed.weight.device
output = {}
for lang in ("en", "tr"):
rng = np.random.default_rng(99157)
results = Counter()
intervals = Counter()
length_checks = Counter()
for upper in (128, 256, 512):
for _ in range(10):
x, y, names = conversations.batch(
rng, upper, 4096, paired=True, lang=lang
)
idx = [i for i, m in enumerate(names) if dialogue_source(m)]
if not idx:
continue
order = idx + [i + len(names) for i in idx]
x = torch.as_tensor(x[order], device=device)
y = torch.as_tensor(y[order], device=device)
with torch.autocast(
device.type, enabled=device.type == "cuda", dtype=torch.bfloat16
):
states = model.hidden(x)
for name, cut in [("whole", None), ("opening", 20)]:
feat = labelled_states(states, y, cut)
good, bad = self.head(feat).chunk(2)
wins = good > bad
results[name] += int(wins.sum())
intervals[name] += len(good)
if name == "whole":
positive, negative = (y != -100).sum(1).chunk(2)
for label, selected in {
"shorter": positive < negative,
"longer": positive > negative,
"similar": (positive - negative).abs() <= 8,
}.items():
length_checks[label + "_wins"] += int(
(wins & selected).sum()
)
length_checks[label + "_n"] += int(selected.sum())
output[lang] = {
k: dict(wins=results[k], pairs=n, accuracy=results[k] / n)
for k, n in intervals.items()
}
output[lang]["by_length"] = {
k: dict(
wins=length_checks[k + "_wins"],
pairs=length_checks[k + "_n"],
accuracy=length_checks[k + "_wins"]
/ max(1, length_checks[k + "_n"]),
)
for k in ("shorter", "longer", "similar")
}
(self.root / "validation.json").write_text(json.dumps(output, indent=2) + "\n")
return output