"""Train the s1d1 router on your own labelled prompts: continue from the published weights, or start from mmBERT. Python >= 3.10. pip install "transformers>=5" torch pyarrow scikit-learn scipy python train.py --train my_train.parquet --validation my_validation.parquet --out my-router --device mps python train.py --train my_train.parquet --base jhu-clsp/mmBERT-base --epochs 3 --out my-router Parquet columns: `prompt` (string), `complexity_score` (float in 0.01..1.00, null when the prompt cannot be judged), `task` (one of the 19 names of s1d1_router.TASKS), `needs_tools` (bool). Optional `weight` (float, 1 by default): row weight in the loss and in the validation metrics. To label your own traffic with the judge that labelled the published model, see judge/. `--base` is a PyTorch repo or folder (this one, scombat/s1d1-router-6src-compact / -small / -tiny, or a mmBERT); a pruned-vocabulary base brings its vocab_map.npy along. Same recipe as the published model (pipeline/05_train.py in the project): full fine-tuning, AdamW (weight decay 0.01, none on biases and norms), linear warm-up over 5 % of the steps then linear decay, gradient clipping at 1.0, batches of --batch-size rows drawn from length-sorted buckets and cut into micro-batches of at most --micro-tokens padded tokens (exact gradient accumulation). Loss: squared error of output 0 against (complexity_score - 0.5) / 0.25 (null rows masked), cross-entropy of outputs 1..19 against task, binary cross-entropy of output 20 against needs_tools, each averaged over the batch with the row weights, summed without coefficients. Prompts longer than 1,024 tokens are truncated. The saved model is a plain AutoModelForSequenceClassification with the same 21 outputs, loadable with `Router(out_dir)`; it is written after every epoch (a crash keeps the last finished epoch). """ import argparse import collections import json import logging import math import os import random import sys import time # Hard cap on the Apple MPS pool (fractions of the recommended working set), set before torch loads: an allocation # above it fails loudly instead of pushing the machine into swap. Ignored on other devices. os.environ.setdefault("PYTORCH_MPS_HIGH_WATERMARK_RATIO", "0.4") os.environ.setdefault("PYTORCH_MPS_LOW_WATERMARK_RATIO", "0.3") import numpy as np import pyarrow.parquet as pq import torch import torch.nn.functional as F from transformers import AutoModelForSequenceClassification, AutoTokenizer sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) from s1d1_router import COMPLEXITY_CENTER, COMPLEXITY_SCALE, MAX_LENGTH, TASKS, TIERS, VOCAB_MAP, local_folder # noqa: E402 log = logging.getLogger("train") N_OUTPUTS = 1 + len(TASKS) + 1 TIER_SCORES = np.array([score for _, score in TIERS]) DEFAULT_BASE = "scombat/s1d1-router" OUTPUT_LAYOUT = ("0: complexity in tier units (score = 0.5 + 0.25 * logit, clipped to 0.01..1.00); " "1-19: task logits in task_names order; 20: needs_tools logit") LENGTH_BUCKET_BATCHES = 50 # batches per length-sorted bucket WARMUP_FRACTION = 0.05 WEIGHT_DECAY = 0.01 MAX_GRAD_NORM = 1.0 WEIGHT_EPS = 1e-12 PAD_MULTIPLE = 64 # padded widths are multiples of this: fewer distinct shapes, so a smaller allocator cache EMPTY_CACHE_EVERY = 25 # optimizer steps between two cache releases on MPS # --- data -------------------------------------------------------------------------------------- def read_rows(path: str, tokenizer, vocab_map: np.ndarray | None) -> dict: """The rows of a labelled parquet file, tokenized; `vocab_map` renumbers the token ids of a pruned vocabulary.""" t = pq.read_table(path).to_pydict() missing = [c for c in ("prompt", "complexity_score", "task", "needs_tools") if c not in t] if missing: sys.exit(f"{path}: missing columns {missing}") unknown = sorted({x for x in t["task"] if x not in TASKS}) if unknown: sys.exit(f"{path}: unknown task names {unknown[:10]} (expected one of {list(TASKS)})") complexity = np.array([np.nan if v is None else float(v) for v in t["complexity_score"]]) if np.nanmin(complexity) < 0 or np.nanmax(complexity) > 1: sys.exit(f"{path}: complexity_score must be in 0.01..1.00 or null") ids = tokenizer(t["prompt"], truncation=True, max_length=MAX_LENGTH)["input_ids"] if vocab_map is not None: ids = [vocab_map[np.asarray(x)].tolist() for x in ids] weight = np.array([1.0 if v is None else float(v) for v in t["weight"]]) if "weight" in t else np.ones(len(ids)) return { "input_ids": ids, "complexity": complexity, "task": np.array([TASKS.index(x) for x in t["task"]]), "needs_tools": np.array([bool(v) for v in t["needs_tools"]], dtype=np.float32), "weight": weight, "has_weights": "weight" in t, } def make_batches(lengths: list[int], batch_size: int, rng: random.Random) -> list[list[int]]: """Shuffle, sort buckets of LENGTH_BUCKET_BATCHES batches by length, cut into batches, shuffle the batches.""" order = list(range(len(lengths))) rng.shuffle(order) span = batch_size * LENGTH_BUCKET_BATCHES batches = [] for start in range(0, len(order), span): bucket = sorted(order[start:start + span], key=lambda i: lengths[i]) batches += [bucket[i:i + batch_size] for i in range(0, len(bucket), batch_size)] rng.shuffle(batches) return batches def micro_batches(batch: list[int], lengths: list[int], micro_tokens: int) -> list[list[int]]: """Cut a batch into length-sorted pieces of at most micro_tokens padded tokens (one row at least).""" batch = sorted(batch, key=lambda i: lengths[i]) out, current = [], [] for i in batch: if current and (len(current) + 1) * -(-lengths[i] // PAD_MULTIPLE) * PAD_MULTIPLE > micro_tokens: out.append(current) current = [] current.append(i) return out + [current] def collate(data: dict, rows: list[int], pad_id: int, device: str) -> dict: width = -(-max(len(data["input_ids"][i]) for i in rows) // PAD_MULTIPLE) * PAD_MULTIPLE ids = torch.full((len(rows), width), pad_id, dtype=torch.long) mask = torch.zeros((len(rows), width), dtype=torch.long) for k, i in enumerate(rows): n = len(data["input_ids"][i]) ids[k, :n] = torch.tensor(data["input_ids"][i]) mask[k, :n] = 1 return {"input_ids": ids.to(device), "attention_mask": mask.to(device)} # --- loss and metrics -------------------------------------------------------------------------- def loss_terms(logits: torch.Tensor, data: dict, rows: list[int], device: str) -> dict: """Weighted sums of the three losses over the rows (to be divided by the batch denominators).""" logits = logits.float() w = torch.tensor(data["weight"][rows], dtype=torch.float32, device=device) valid = torch.tensor(~np.isnan(data["complexity"][rows]), device=device) target = torch.tensor(np.nan_to_num((data["complexity"][rows] - COMPLEXITY_CENTER) / COMPLEXITY_SCALE), dtype=torch.float32, device=device) task = torch.tensor(data["task"][rows], device=device) tools = torch.tensor(data["needs_tools"][rows], device=device) return { "complexity": (w * valid * (logits[:, 0] - target) ** 2).sum(), "task": (w * F.cross_entropy(logits[:, 1:1 + len(TASKS)], task, reduction="none")).sum(), "needs_tools": (w * F.binary_cross_entropy_with_logits(logits[:, -1], tools, reduction="none")).sum(), } def denominators(data: dict, rows: list[int]) -> dict: w, valid = data["weight"][rows], ~np.isnan(data["complexity"][rows]) return {"complexity": max(float((w * valid).sum()), WEIGHT_EPS), "task": float(w.sum()), "needs_tools": float(w.sum())} def weighted_spearman(x: np.ndarray, y: np.ndarray, w: np.ndarray) -> float: """Weighted Pearson correlation of the weighted mid-ranks (equals Spearman when all weights are equal).""" def midranks(v): order = np.argsort(v, kind="stable") cum = np.cumsum(w[order]) - w[order] / 2 ranks = np.empty(len(v)) for value in np.unique(v): group = v[order] == value ranks[order[group]] = np.average(cum[group], weights=w[order][group]) return ranks rx, ry = midranks(x), midranks(y) mx, my = np.average(rx, weights=w), np.average(ry, weights=w) return float(np.sum(w * (rx - mx) * (ry - my)) / math.sqrt(np.sum(w * (rx - mx) ** 2) * np.sum(w * (ry - my) ** 2))) def metrics(data: dict, out: np.ndarray, weighted: bool) -> dict: """Agreement of the outputs `out` with the labels: complexity (MAE, Spearman, quadratic kappa on the nearest tier, confusion matrix), task (accuracy, macro-F1, F1 per class), needs_tools (F1 at 0.5, AUROC).""" from sklearn.metrics import cohen_kappa_score, confusion_matrix, f1_score, roc_auc_score w = data["weight"] if weighted else np.ones(len(out)) res = {"rows": len(out), "weighted": weighted} ok = ~np.isnan(data["complexity"]) score = np.clip(COMPLEXITY_CENTER + COMPLEXITY_SCALE * out[ok, 0], 0.01, 1.0) true, wc = data["complexity"][ok], w[ok] pred_tier = np.abs(score[:, None] - TIER_SCORES[None]).argmin(1) true_tier = np.abs(true[:, None] - TIER_SCORES[None]).argmin(1) tiers = [name for name, _ in TIERS] res["complexity"] = { "rows": int(ok.sum()), "mae": float(np.average(np.abs(score - true), weights=wc)), "spearman": weighted_spearman(score, true, wc), "qwk_rounded": float(cohen_kappa_score(true_tier, pred_tier, weights="quadratic", sample_weight=wc)), "confusion_true_by_pred": confusion_matrix(true_tier, pred_tier, labels=range(len(tiers)), sample_weight=wc).round(1).tolist(), "tiers": tiers, } pred_task, true_task = out[:, 1:1 + len(TASKS)].argmax(1), data["task"] f1 = f1_score(true_task, pred_task, labels=range(len(TASKS)), average=None, sample_weight=w, zero_division=0) present = np.bincount(true_task, minlength=len(TASKS)) > 0 res["task"] = { "accuracy": float(np.average(pred_task == true_task, weights=w)), "macro_f1_present_classes": float(f1[present].mean()), "per_class": {t: {"f1": float(f1[k]), "support": int((true_task == k).sum())} for k, t in enumerate(TASKS)}, } prob, true_tools = 1 / (1 + np.exp(-out[:, -1])), data["needs_tools"] res["needs_tools"] = { "f1": float(f1_score(true_tools, prob >= 0.5, sample_weight=w, zero_division=0)), "auroc": float(roc_auc_score(true_tools, prob, sample_weight=w)) if 0 < true_tools.sum() < len(true_tools) else None, "positives": int(true_tools.sum()), } return res @torch.no_grad() def predict(model, data: dict, device: str, pad_id: int, precision: str, micro_tokens: int) -> np.ndarray: model.eval() lengths = [len(x) for x in data["input_ids"]] order = sorted(range(len(lengths)), key=lambda i: lengths[i]) out = np.zeros((len(lengths), N_OUTPUTS), dtype=np.float32) for rows in micro_batches(order, lengths, micro_tokens): with torch.autocast(device, dtype=torch.bfloat16, enabled=precision == "bf16"): logits = model(**collate(data, rows, pad_id, device)).logits out[rows] = logits.float().cpu().numpy() model.train() return out # --- training ---------------------------------------------------------------------------------- def parse_args(argv: list[str]) -> argparse.Namespace: p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) p.add_argument("--train", required=True, help="parquet file of labelled prompts") p.add_argument("--validation", help="parquet file evaluated after every epoch (same columns)") p.add_argument("--out", required=True, help="folder of the trained model (written after every epoch)") p.add_argument("--base", default=DEFAULT_BASE, help="Hub repo or folder to start from (PyTorch weights)") p.add_argument("--epochs", type=int, default=3) p.add_argument("--lr", type=float, default=3e-5) p.add_argument("--batch-size", type=int, default=32, help="rows per optimizer step") p.add_argument("--micro-tokens", type=int, default=4096, help="padded tokens per forward/backward pass") p.add_argument("--eval-micro-tokens", type=int, default=16384, help="padded tokens per forward pass in evaluation") p.add_argument("--precision", choices=("fp32", "bf16"), default="fp32", help="bf16 = autocast, weights stay in fp32") p.add_argument("--device", default="auto", help="auto (cuda, then mps, then cpu), cuda, mps or cpu") p.add_argument("--seed", type=int, default=42) return p.parse_args(argv) def pick_device(wanted: str) -> str: if wanted != "auto": return wanted if torch.cuda.is_available(): return "cuda" return "mps" if torch.backends.mps.is_available() else "cpu" def synchronize(device: str) -> None: if device.startswith("cuda"): torch.cuda.synchronize() elif device == "mps": torch.mps.synchronize() def save(model, tokenizer, base_folder: str, out: str, vocab_map: np.ndarray | None) -> None: model.config.problem_type = "regression" # the text-classification pipeline then returns the raw outputs model.config.task_names = list(TASKS) model.config.output_layout = OUTPUT_LAYOUT model.save_pretrained(out) tokenizer.save_pretrained(out) if vocab_map is not None: np.save(os.path.join(out, VOCAB_MAP), vocab_map) def main(argv: list[str]) -> None: args = parse_args(argv) logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", stream=sys.stderr) device = pick_device(args.device) random.seed(args.seed) np.random.seed(args.seed) torch.manual_seed(args.seed) os.makedirs(args.out, exist_ok=True) base = local_folder(args.base) tokenizer = AutoTokenizer.from_pretrained(base) map_path = os.path.join(base, VOCAB_MAP) vocab_map = np.load(map_path) if os.path.isfile(map_path) else None pad_id = int(vocab_map[tokenizer.pad_token_id]) if vocab_map is not None else tokenizer.pad_token_id train = read_rows(args.train, tokenizer, vocab_map) valid = read_rows(args.validation, tokenizer, vocab_map) if args.validation else None log.info("device %s, train %d rows (%d with a complexity, %d needs_tools), base %s", device, len(train["input_ids"]), int((~np.isnan(train["complexity"])).sum()), int(train["needs_tools"].sum()), args.base) model = AutoModelForSequenceClassification.from_pretrained(base, num_labels=N_OUTPUTS, dtype=torch.float32).to(device) model.train() no_decay = ("bias", "norm") groups = [{"params": [p for n, p in model.named_parameters() if not any(k in n for k in no_decay)], "weight_decay": WEIGHT_DECAY}, {"params": [p for n, p in model.named_parameters() if any(k in n for k in no_decay)], "weight_decay": 0.0}] optimizer = torch.optim.AdamW(groups, lr=args.lr) lengths = [len(x) for x in train["input_ids"]] rng = random.Random(args.seed) total_steps = math.ceil(len(lengths) / args.batch_size) * args.epochs warmup = max(1, int(WARMUP_FRACTION * total_steps)) schedule = torch.optim.lr_scheduler.LambdaLR( optimizer, lambda s: min((s + 1) / warmup, max(0.0, (total_steps - s) / (total_steps - warmup)))) history, validation, step = [], [], 0 started = time.monotonic() for epoch in range(args.epochs): for batch in make_batches(lengths, args.batch_size, rng): denom = denominators(train, batch) batch_loss = collections.Counter() for rows in micro_batches(batch, lengths, args.micro_tokens): with torch.autocast(device, dtype=torch.bfloat16, enabled=args.precision == "bf16"): logits = model(**collate(train, rows, pad_id, device)).logits terms = {k: v / denom[k] for k, v in loss_terms(logits, train, rows, device).items()} sum(terms.values()).backward() for k, v in terms.items(): batch_loss[k] += v.item() grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), MAX_GRAD_NORM).item() if not math.isfinite(grad_norm) or not all(math.isfinite(v) for v in batch_loss.values()): raise RuntimeError(f"step {step}: non-finite loss or gradient ({dict(batch_loss)}, grad norm {grad_norm})") optimizer.step() schedule.step() optimizer.zero_grad(set_to_none=True) step += 1 if device == "mps" and step % EMPTY_CACHE_EVERY == 0: torch.mps.empty_cache() if step % 100 == 0 or step == total_steps: synchronize(device) history.append({"step": step, "loss": dict(batch_loss), "grad_norm": grad_norm, "lr": schedule.get_last_lr()[0], "seconds": round(time.monotonic() - started)}) log.info("step %d/%d loss %s grad %.2f %.0f s", step, total_steps, {k: round(v, 4) for k, v in batch_loss.items()}, grad_norm, time.monotonic() - started) if valid is not None: out = predict(model, valid, device, pad_id, args.precision, args.eval_micro_tokens) res = {"epoch": epoch + 1, "unweighted": metrics(valid, out, False)} if valid["has_weights"]: res["weighted"] = metrics(valid, out, True) validation.append(res) u = res["unweighted"] log.info("epoch %d: validation MAE %.3f, QWK %.3f, task accuracy %.3f, needs_tools F1 %s AUROC %s", epoch + 1, u["complexity"]["mae"], u["complexity"]["qwk_rounded"], u["task"]["accuracy"], round(u["needs_tools"]["f1"], 3), None if u["needs_tools"]["auroc"] is None else round(u["needs_tools"]["auroc"], 3)) save(model, tokenizer, base, args.out, vocab_map) with open(os.path.join(args.out, "training.json"), "w", encoding="utf-8") as f: json.dump({"args": vars(args), "device": device, "steps": step, "seconds": round(time.monotonic() - started), "history": history, "validation": validation}, f, indent=1) log.info("epoch %d saved to %s", epoch + 1, args.out) if __name__ == "__main__": main(sys.argv[1:])