Download source/training/third/preference.py from yunusemrejr/Franok: direct link, hf CLI and curl.
- Browser
- Download file 5.35 kB
-
https://huggingface.co/yunusemrejr/Franok/resolve/main/source/training/third/preference.py
- Command line
-
hf download hf://yunusemrejr/Franok/source/training/third/preference.py
-
curl -L -o preference.py https://huggingface.co/yunusemrejr/Franok/resolve/main/source/training/third/preference.py
5.35 kB
| """Learn a dialogue reward from ranked responses and same-conversation alternatives.""" | |
| from collections import Counter | |
| import json | |
| import numpy as np | |
| import torch | |
| from safetensors.torch import load_file, save_file | |
| from torch.nn import functional as F | |
| from franok.preference import ReplyPreference | |
| def dialogue_source(name): | |
| return name in { | |
| "human", | |
| "english-dialogue", | |
| "short-dialogue", | |
| "everyday-en", | |
| "turkish-dialogue", | |
| "turkish-tasks", | |
| "native-chat", | |
| "everyday-tr", | |
| "short-replies", | |
| } | |
| def labelled_states(states, labels, early=None): | |
| # labels begin at the assistant marker and end with the EOS target. | |
| mask = labels != -100 | |
| marker = mask.int().argmax(1) | |
| count = mask.sum(1) - 1 | |
| if early is not None: | |
| count = count.clamp_max(early) | |
| return ReplyPreference.features(states.detach(), marker, count) | |
| class DialogueReward: | |
| def __init__(self, width, device, root): | |
| self.head = ReplyPreference(width).to(device) | |
| self.opt = torch.optim.AdamW( | |
| self.head.parameters(), lr=0.00065, weight_decay=0.035 | |
| ) | |
| self.steps = 0 | |
| self.total = Counter() | |
| self.root = root | |
| self.root.mkdir(exist_ok=True, parents=True) | |
| if (root / "head.safetensors").exists(): | |
| self.head.load_state_dict(load_file(str(root / "head.safetensors"))) | |
| def update(self, features, names): | |
| eligible = [i for i, name in enumerate(names) if dialogue_source(name)] | |
| if not eligible: | |
| return | |
| good, bad = features.chunk(2) | |
| self.head.train() | |
| x = self.head(torch.cat((good[eligible], bad[eligible]))) | |
| preferred, rejected = x.chunk(2) | |
| loss = F.softplus(rejected - preferred).mean() + 0.02 * x.square().mean() | |
| self.opt.zero_grad(set_to_none=True) | |
| loss.backward() | |
| torch.nn.utils.clip_grad_norm_( | |
| self.head.parameters(), 1.0, error_if_nonfinite=True | |
| ) | |
| self.opt.step() | |
| self.steps += 1 | |
| self.total.update(names[i] for i in eligible) | |
| def save(self): | |
| file = self.root / "head.next" | |
| save_file( | |
| { | |
| k: v.detach().cpu().contiguous() | |
| for k, v in self.head.state_dict().items() | |
| }, | |
| str(file), | |
| ) | |
| file.replace(self.root / "head.safetensors") | |
| (self.root / "training.json").write_text( | |
| json.dumps(dict(updates=self.steps, pairs=dict(self.total)), indent=2) | |
| + "\n" | |
| ) | |
| def evaluate(self, model, conversations): | |
| self.head.eval() | |
| device = model.embed.weight.device | |
| output = {} | |
| for lang in ("en", "tr"): | |
| rng = np.random.default_rng(99157) | |
| results = Counter() | |
| intervals = Counter() | |
| length_checks = Counter() | |
| for upper in (128, 256, 512): | |
| for _ in range(10): | |
| x, y, names = conversations.batch( | |
| rng, upper, 4096, paired=True, lang=lang | |
| ) | |
| idx = [i for i, m in enumerate(names) if dialogue_source(m)] | |
| if not idx: | |
| continue | |
| order = idx + [i + len(names) for i in idx] | |
| x = torch.as_tensor(x[order], device=device) | |
| y = torch.as_tensor(y[order], device=device) | |
| with torch.autocast( | |
| device.type, enabled=device.type == "cuda", dtype=torch.bfloat16 | |
| ): | |
| states = model.hidden(x) | |
| for name, cut in [("whole", None), ("opening", 20)]: | |
| feat = labelled_states(states, y, cut) | |
| good, bad = self.head(feat).chunk(2) | |
| wins = good > bad | |
| results[name] += int(wins.sum()) | |
| intervals[name] += len(good) | |
| if name == "whole": | |
| positive, negative = (y != -100).sum(1).chunk(2) | |
| for label, selected in { | |
| "shorter": positive < negative, | |
| "longer": positive > negative, | |
| "similar": (positive - negative).abs() <= 8, | |
| }.items(): | |
| length_checks[label + "_wins"] += int( | |
| (wins & selected).sum() | |
| ) | |
| length_checks[label + "_n"] += int(selected.sum()) | |
| output[lang] = { | |
| k: dict(wins=results[k], pairs=n, accuracy=results[k] / n) | |
| for k, n in intervals.items() | |
| } | |
| output[lang]["by_length"] = { | |
| k: dict( | |
| wins=length_checks[k + "_wins"], | |
| pairs=length_checks[k + "_n"], | |
| accuracy=length_checks[k + "_wins"] | |
| / max(1, length_checks[k + "_n"]), | |
| ) | |
| for k in ("shorter", "longer", "similar") | |
| } | |
| (self.root / "validation.json").write_text(json.dumps(output, indent=2) + "\n") | |
| return output | |