s1d1-router / train.py
scombat's picture
Loader from the Hub (tokenizer config, vocab map, MLX/ONNX), four tiers, train.py, judge/, card fixes (docs/16)
58d8bb0 verified
Raw History Blame Contribute Delete
18.4 kB
"""Train the s1d1 router on your own labelled prompts: continue from the published weights, or start from mmBERT.
Python >= 3.10.
pip install "transformers>=5" torch pyarrow scikit-learn scipy
python train.py --train my_train.parquet --validation my_validation.parquet --out my-router --device mps
python train.py --train my_train.parquet --base jhu-clsp/mmBERT-base --epochs 3 --out my-router
Parquet columns: `prompt` (string), `complexity_score` (float in 0.01..1.00, null when the prompt cannot be judged),
`task` (one of the 19 names of s1d1_router.TASKS), `needs_tools` (bool). Optional `weight` (float, 1 by default): row
weight in the loss and in the validation metrics. To label your own traffic with the judge that labelled the published
model, see judge/. `--base` is a PyTorch repo or folder (this one, scombat/s1d1-router-6src-compact / -small / -tiny,
or a mmBERT); a pruned-vocabulary base brings its vocab_map.npy along.
Same recipe as the published model (pipeline/05_train.py in the project): full fine-tuning, AdamW (weight decay 0.01,
none on biases and norms), linear warm-up over 5 % of the steps then linear decay, gradient clipping at 1.0, batches of
--batch-size rows drawn from length-sorted buckets and cut into micro-batches of at most --micro-tokens padded tokens
(exact gradient accumulation). Loss: squared error of output 0 against (complexity_score - 0.5) / 0.25 (null rows
masked), cross-entropy of outputs 1..19 against task, binary cross-entropy of output 20 against needs_tools, each
averaged over the batch with the row weights, summed without coefficients. Prompts longer than 1,024 tokens are
truncated. The saved model is a plain AutoModelForSequenceClassification with the same 21 outputs, loadable with
`Router(out_dir)`; it is written after every epoch (a crash keeps the last finished epoch).
"""
import argparse
import collections
import json
import logging
import math
import os
import random
import sys
import time
# Hard cap on the Apple MPS pool (fractions of the recommended working set), set before torch loads: an allocation
# above it fails loudly instead of pushing the machine into swap. Ignored on other devices.
os.environ.setdefault("PYTORCH_MPS_HIGH_WATERMARK_RATIO", "0.4")
os.environ.setdefault("PYTORCH_MPS_LOW_WATERMARK_RATIO", "0.3")
import numpy as np
import pyarrow.parquet as pq
import torch
import torch.nn.functional as F
from transformers import AutoModelForSequenceClassification, AutoTokenizer
sys.path.insert(0, os.path.dirname(os.path.abspath(__file__)))
from s1d1_router import COMPLEXITY_CENTER, COMPLEXITY_SCALE, MAX_LENGTH, TASKS, TIERS, VOCAB_MAP, local_folder # noqa: E402
log = logging.getLogger("train")
N_OUTPUTS = 1 + len(TASKS) + 1
TIER_SCORES = np.array([score for _, score in TIERS])
DEFAULT_BASE = "scombat/s1d1-router"
OUTPUT_LAYOUT = ("0: complexity in tier units (score = 0.5 + 0.25 * logit, clipped to 0.01..1.00); "
"1-19: task logits in task_names order; 20: needs_tools logit")
LENGTH_BUCKET_BATCHES = 50 # batches per length-sorted bucket
WARMUP_FRACTION = 0.05
WEIGHT_DECAY = 0.01
MAX_GRAD_NORM = 1.0
WEIGHT_EPS = 1e-12
PAD_MULTIPLE = 64 # padded widths are multiples of this: fewer distinct shapes, so a smaller allocator cache
EMPTY_CACHE_EVERY = 25 # optimizer steps between two cache releases on MPS
# --- data --------------------------------------------------------------------------------------
def read_rows(path: str, tokenizer, vocab_map: np.ndarray | None) -> dict:
"""The rows of a labelled parquet file, tokenized; `vocab_map` renumbers the token ids of a pruned vocabulary."""
t = pq.read_table(path).to_pydict()
missing = [c for c in ("prompt", "complexity_score", "task", "needs_tools") if c not in t]
if missing:
sys.exit(f"{path}: missing columns {missing}")
unknown = sorted({x for x in t["task"] if x not in TASKS})
if unknown:
sys.exit(f"{path}: unknown task names {unknown[:10]} (expected one of {list(TASKS)})")
complexity = np.array([np.nan if v is None else float(v) for v in t["complexity_score"]])
if np.nanmin(complexity) < 0 or np.nanmax(complexity) > 1:
sys.exit(f"{path}: complexity_score must be in 0.01..1.00 or null")
ids = tokenizer(t["prompt"], truncation=True, max_length=MAX_LENGTH)["input_ids"]
if vocab_map is not None:
ids = [vocab_map[np.asarray(x)].tolist() for x in ids]
weight = np.array([1.0 if v is None else float(v) for v in t["weight"]]) if "weight" in t else np.ones(len(ids))
return {
"input_ids": ids, "complexity": complexity,
"task": np.array([TASKS.index(x) for x in t["task"]]),
"needs_tools": np.array([bool(v) for v in t["needs_tools"]], dtype=np.float32),
"weight": weight, "has_weights": "weight" in t,
}
def make_batches(lengths: list[int], batch_size: int, rng: random.Random) -> list[list[int]]:
"""Shuffle, sort buckets of LENGTH_BUCKET_BATCHES batches by length, cut into batches, shuffle the batches."""
order = list(range(len(lengths)))
rng.shuffle(order)
span = batch_size * LENGTH_BUCKET_BATCHES
batches = []
for start in range(0, len(order), span):
bucket = sorted(order[start:start + span], key=lambda i: lengths[i])
batches += [bucket[i:i + batch_size] for i in range(0, len(bucket), batch_size)]
rng.shuffle(batches)
return batches
def micro_batches(batch: list[int], lengths: list[int], micro_tokens: int) -> list[list[int]]:
"""Cut a batch into length-sorted pieces of at most micro_tokens padded tokens (one row at least)."""
batch = sorted(batch, key=lambda i: lengths[i])
out, current = [], []
for i in batch:
if current and (len(current) + 1) * -(-lengths[i] // PAD_MULTIPLE) * PAD_MULTIPLE > micro_tokens:
out.append(current)
current = []
current.append(i)
return out + [current]
def collate(data: dict, rows: list[int], pad_id: int, device: str) -> dict:
width = -(-max(len(data["input_ids"][i]) for i in rows) // PAD_MULTIPLE) * PAD_MULTIPLE
ids = torch.full((len(rows), width), pad_id, dtype=torch.long)
mask = torch.zeros((len(rows), width), dtype=torch.long)
for k, i in enumerate(rows):
n = len(data["input_ids"][i])
ids[k, :n] = torch.tensor(data["input_ids"][i])
mask[k, :n] = 1
return {"input_ids": ids.to(device), "attention_mask": mask.to(device)}
# --- loss and metrics --------------------------------------------------------------------------
def loss_terms(logits: torch.Tensor, data: dict, rows: list[int], device: str) -> dict:
"""Weighted sums of the three losses over the rows (to be divided by the batch denominators)."""
logits = logits.float()
w = torch.tensor(data["weight"][rows], dtype=torch.float32, device=device)
valid = torch.tensor(~np.isnan(data["complexity"][rows]), device=device)
target = torch.tensor(np.nan_to_num((data["complexity"][rows] - COMPLEXITY_CENTER) / COMPLEXITY_SCALE),
dtype=torch.float32, device=device)
task = torch.tensor(data["task"][rows], device=device)
tools = torch.tensor(data["needs_tools"][rows], device=device)
return {
"complexity": (w * valid * (logits[:, 0] - target) ** 2).sum(),
"task": (w * F.cross_entropy(logits[:, 1:1 + len(TASKS)], task, reduction="none")).sum(),
"needs_tools": (w * F.binary_cross_entropy_with_logits(logits[:, -1], tools, reduction="none")).sum(),
}
def denominators(data: dict, rows: list[int]) -> dict:
w, valid = data["weight"][rows], ~np.isnan(data["complexity"][rows])
return {"complexity": max(float((w * valid).sum()), WEIGHT_EPS), "task": float(w.sum()), "needs_tools": float(w.sum())}
def weighted_spearman(x: np.ndarray, y: np.ndarray, w: np.ndarray) -> float:
"""Weighted Pearson correlation of the weighted mid-ranks (equals Spearman when all weights are equal)."""
def midranks(v):
order = np.argsort(v, kind="stable")
cum = np.cumsum(w[order]) - w[order] / 2
ranks = np.empty(len(v))
for value in np.unique(v):
group = v[order] == value
ranks[order[group]] = np.average(cum[group], weights=w[order][group])
return ranks
rx, ry = midranks(x), midranks(y)
mx, my = np.average(rx, weights=w), np.average(ry, weights=w)
return float(np.sum(w * (rx - mx) * (ry - my)) / math.sqrt(np.sum(w * (rx - mx) ** 2) * np.sum(w * (ry - my) ** 2)))
def metrics(data: dict, out: np.ndarray, weighted: bool) -> dict:
"""Agreement of the outputs `out` with the labels: complexity (MAE, Spearman, quadratic kappa on the nearest tier,
confusion matrix), task (accuracy, macro-F1, F1 per class), needs_tools (F1 at 0.5, AUROC)."""
from sklearn.metrics import cohen_kappa_score, confusion_matrix, f1_score, roc_auc_score
w = data["weight"] if weighted else np.ones(len(out))
res = {"rows": len(out), "weighted": weighted}
ok = ~np.isnan(data["complexity"])
score = np.clip(COMPLEXITY_CENTER + COMPLEXITY_SCALE * out[ok, 0], 0.01, 1.0)
true, wc = data["complexity"][ok], w[ok]
pred_tier = np.abs(score[:, None] - TIER_SCORES[None]).argmin(1)
true_tier = np.abs(true[:, None] - TIER_SCORES[None]).argmin(1)
tiers = [name for name, _ in TIERS]
res["complexity"] = {
"rows": int(ok.sum()), "mae": float(np.average(np.abs(score - true), weights=wc)),
"spearman": weighted_spearman(score, true, wc),
"qwk_rounded": float(cohen_kappa_score(true_tier, pred_tier, weights="quadratic", sample_weight=wc)),
"confusion_true_by_pred": confusion_matrix(true_tier, pred_tier, labels=range(len(tiers)), sample_weight=wc).round(1).tolist(),
"tiers": tiers,
}
pred_task, true_task = out[:, 1:1 + len(TASKS)].argmax(1), data["task"]
f1 = f1_score(true_task, pred_task, labels=range(len(TASKS)), average=None, sample_weight=w, zero_division=0)
present = np.bincount(true_task, minlength=len(TASKS)) > 0
res["task"] = {
"accuracy": float(np.average(pred_task == true_task, weights=w)),
"macro_f1_present_classes": float(f1[present].mean()),
"per_class": {t: {"f1": float(f1[k]), "support": int((true_task == k).sum())} for k, t in enumerate(TASKS)},
}
prob, true_tools = 1 / (1 + np.exp(-out[:, -1])), data["needs_tools"]
res["needs_tools"] = {
"f1": float(f1_score(true_tools, prob >= 0.5, sample_weight=w, zero_division=0)),
"auroc": float(roc_auc_score(true_tools, prob, sample_weight=w)) if 0 < true_tools.sum() < len(true_tools) else None,
"positives": int(true_tools.sum()),
}
return res
@torch.no_grad()
def predict(model, data: dict, device: str, pad_id: int, precision: str, micro_tokens: int) -> np.ndarray:
model.eval()
lengths = [len(x) for x in data["input_ids"]]
order = sorted(range(len(lengths)), key=lambda i: lengths[i])
out = np.zeros((len(lengths), N_OUTPUTS), dtype=np.float32)
for rows in micro_batches(order, lengths, micro_tokens):
with torch.autocast(device, dtype=torch.bfloat16, enabled=precision == "bf16"):
logits = model(**collate(data, rows, pad_id, device)).logits
out[rows] = logits.float().cpu().numpy()
model.train()
return out
# --- training ----------------------------------------------------------------------------------
def parse_args(argv: list[str]) -> argparse.Namespace:
p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
p.add_argument("--train", required=True, help="parquet file of labelled prompts")
p.add_argument("--validation", help="parquet file evaluated after every epoch (same columns)")
p.add_argument("--out", required=True, help="folder of the trained model (written after every epoch)")
p.add_argument("--base", default=DEFAULT_BASE, help="Hub repo or folder to start from (PyTorch weights)")
p.add_argument("--epochs", type=int, default=3)
p.add_argument("--lr", type=float, default=3e-5)
p.add_argument("--batch-size", type=int, default=32, help="rows per optimizer step")
p.add_argument("--micro-tokens", type=int, default=4096, help="padded tokens per forward/backward pass")
p.add_argument("--eval-micro-tokens", type=int, default=16384, help="padded tokens per forward pass in evaluation")
p.add_argument("--precision", choices=("fp32", "bf16"), default="fp32", help="bf16 = autocast, weights stay in fp32")
p.add_argument("--device", default="auto", help="auto (cuda, then mps, then cpu), cuda, mps or cpu")
p.add_argument("--seed", type=int, default=42)
return p.parse_args(argv)
def pick_device(wanted: str) -> str:
if wanted != "auto":
return wanted
if torch.cuda.is_available():
return "cuda"
return "mps" if torch.backends.mps.is_available() else "cpu"
def synchronize(device: str) -> None:
if device.startswith("cuda"):
torch.cuda.synchronize()
elif device == "mps":
torch.mps.synchronize()
def save(model, tokenizer, base_folder: str, out: str, vocab_map: np.ndarray | None) -> None:
model.config.problem_type = "regression" # the text-classification pipeline then returns the raw outputs
model.config.task_names = list(TASKS)
model.config.output_layout = OUTPUT_LAYOUT
model.save_pretrained(out)
tokenizer.save_pretrained(out)
if vocab_map is not None:
np.save(os.path.join(out, VOCAB_MAP), vocab_map)
def main(argv: list[str]) -> None:
args = parse_args(argv)
logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", stream=sys.stderr)
device = pick_device(args.device)
random.seed(args.seed)
np.random.seed(args.seed)
torch.manual_seed(args.seed)
os.makedirs(args.out, exist_ok=True)
base = local_folder(args.base)
tokenizer = AutoTokenizer.from_pretrained(base)
map_path = os.path.join(base, VOCAB_MAP)
vocab_map = np.load(map_path) if os.path.isfile(map_path) else None
pad_id = int(vocab_map[tokenizer.pad_token_id]) if vocab_map is not None else tokenizer.pad_token_id
train = read_rows(args.train, tokenizer, vocab_map)
valid = read_rows(args.validation, tokenizer, vocab_map) if args.validation else None
log.info("device %s, train %d rows (%d with a complexity, %d needs_tools), base %s", device, len(train["input_ids"]),
int((~np.isnan(train["complexity"])).sum()), int(train["needs_tools"].sum()), args.base)
model = AutoModelForSequenceClassification.from_pretrained(base, num_labels=N_OUTPUTS, dtype=torch.float32).to(device)
model.train()
no_decay = ("bias", "norm")
groups = [{"params": [p for n, p in model.named_parameters() if not any(k in n for k in no_decay)], "weight_decay": WEIGHT_DECAY},
{"params": [p for n, p in model.named_parameters() if any(k in n for k in no_decay)], "weight_decay": 0.0}]
optimizer = torch.optim.AdamW(groups, lr=args.lr)
lengths = [len(x) for x in train["input_ids"]]
rng = random.Random(args.seed)
total_steps = math.ceil(len(lengths) / args.batch_size) * args.epochs
warmup = max(1, int(WARMUP_FRACTION * total_steps))
schedule = torch.optim.lr_scheduler.LambdaLR(
optimizer, lambda s: min((s + 1) / warmup, max(0.0, (total_steps - s) / (total_steps - warmup))))
history, validation, step = [], [], 0
started = time.monotonic()
for epoch in range(args.epochs):
for batch in make_batches(lengths, args.batch_size, rng):
denom = denominators(train, batch)
batch_loss = collections.Counter()
for rows in micro_batches(batch, lengths, args.micro_tokens):
with torch.autocast(device, dtype=torch.bfloat16, enabled=args.precision == "bf16"):
logits = model(**collate(train, rows, pad_id, device)).logits
terms = {k: v / denom[k] for k, v in loss_terms(logits, train, rows, device).items()}
sum(terms.values()).backward()
for k, v in terms.items():
batch_loss[k] += v.item()
grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), MAX_GRAD_NORM).item()
if not math.isfinite(grad_norm) or not all(math.isfinite(v) for v in batch_loss.values()):
raise RuntimeError(f"step {step}: non-finite loss or gradient ({dict(batch_loss)}, grad norm {grad_norm})")
optimizer.step()
schedule.step()
optimizer.zero_grad(set_to_none=True)
step += 1
if device == "mps" and step % EMPTY_CACHE_EVERY == 0:
torch.mps.empty_cache()
if step % 100 == 0 or step == total_steps:
synchronize(device)
history.append({"step": step, "loss": dict(batch_loss), "grad_norm": grad_norm, "lr": schedule.get_last_lr()[0],
"seconds": round(time.monotonic() - started)})
log.info("step %d/%d loss %s grad %.2f %.0f s", step, total_steps,
{k: round(v, 4) for k, v in batch_loss.items()}, grad_norm, time.monotonic() - started)
if valid is not None:
out = predict(model, valid, device, pad_id, args.precision, args.eval_micro_tokens)
res = {"epoch": epoch + 1, "unweighted": metrics(valid, out, False)}
if valid["has_weights"]:
res["weighted"] = metrics(valid, out, True)
validation.append(res)
u = res["unweighted"]
log.info("epoch %d: validation MAE %.3f, QWK %.3f, task accuracy %.3f, needs_tools F1 %s AUROC %s", epoch + 1,
u["complexity"]["mae"], u["complexity"]["qwk_rounded"], u["task"]["accuracy"],
round(u["needs_tools"]["f1"], 3), None if u["needs_tools"]["auroc"] is None else round(u["needs_tools"]["auroc"], 3))
save(model, tokenizer, base, args.out, vocab_map)
with open(os.path.join(args.out, "training.json"), "w", encoding="utf-8") as f:
json.dump({"args": vars(args), "device": device, "steps": step, "seconds": round(time.monotonic() - started),
"history": history, "validation": validation}, f, indent=1)
log.info("epoch %d saved to %s", epoch + 1, args.out)
if __name__ == "__main__":
main(sys.argv[1:])