Text Classification
Transformers
Safetensors
ONNX
multilingual
modernbert
routing
llm-routing
prompt-classification
complexity-estimation
text-embeddings-inference
Instructions to use scombat/s1d1-router with libraries, inference providers, notebooks, and local apps. Follow these links to get started.
- Libraries
- Transformers
How to use scombat/s1d1-router with Transformers:
# Use a pipeline as a high-level helper from transformers import pipeline pipe = pipeline("text-classification", model="scombat/s1d1-router")# pip install -U transformers accelerate # Load model directly from transformers import AutoTokenizer, AutoModelForSequenceClassification tokenizer = AutoTokenizer.from_pretrained("scombat/s1d1-router") model = AutoModelForSequenceClassification.from_pretrained("scombat/s1d1-router", device_map="auto") - Notebooks
- Google Colab
- Kaggle
Download train.py from scombat/s1d1-router: direct link, hf CLI and curl.
- Browser
- Download file 18.4 kB
-
https://huggingface.co/scombat/s1d1-router/resolve/main/train.py
- Command line
-
hf download hf://scombat/s1d1-router/train.py
-
curl -L -o train.py https://huggingface.co/scombat/s1d1-router/resolve/main/train.py
18.4 kB
| """Train the s1d1 router on your own labelled prompts: continue from the published weights, or start from mmBERT. | |
| Python >= 3.10. | |
| pip install "transformers>=5" torch pyarrow scikit-learn scipy | |
| python train.py --train my_train.parquet --validation my_validation.parquet --out my-router --device mps | |
| python train.py --train my_train.parquet --base jhu-clsp/mmBERT-base --epochs 3 --out my-router | |
| Parquet columns: `prompt` (string), `complexity_score` (float in 0.01..1.00, null when the prompt cannot be judged), | |
| `task` (one of the 19 names of s1d1_router.TASKS), `needs_tools` (bool). Optional `weight` (float, 1 by default): row | |
| weight in the loss and in the validation metrics. To label your own traffic with the judge that labelled the published | |
| model, see judge/. `--base` is a PyTorch repo or folder (this one, scombat/s1d1-router-6src-compact / -small / -tiny, | |
| or a mmBERT); a pruned-vocabulary base brings its vocab_map.npy along. | |
| Same recipe as the published model (pipeline/05_train.py in the project): full fine-tuning, AdamW (weight decay 0.01, | |
| none on biases and norms), linear warm-up over 5 % of the steps then linear decay, gradient clipping at 1.0, batches of | |
| --batch-size rows drawn from length-sorted buckets and cut into micro-batches of at most --micro-tokens padded tokens | |
| (exact gradient accumulation). Loss: squared error of output 0 against (complexity_score - 0.5) / 0.25 (null rows | |
| masked), cross-entropy of outputs 1..19 against task, binary cross-entropy of output 20 against needs_tools, each | |
| averaged over the batch with the row weights, summed without coefficients. Prompts longer than 1,024 tokens are | |
| truncated. The saved model is a plain AutoModelForSequenceClassification with the same 21 outputs, loadable with | |
| `Router(out_dir)`; it is written after every epoch (a crash keeps the last finished epoch). | |
| """ | |
| import argparse | |
| import collections | |
| import json | |
| import logging | |
| import math | |
| import os | |
| import random | |
| import sys | |
| import time | |
| # Hard cap on the Apple MPS pool (fractions of the recommended working set), set before torch loads: an allocation | |
| # above it fails loudly instead of pushing the machine into swap. Ignored on other devices. | |
| os.environ.setdefault("PYTORCH_MPS_HIGH_WATERMARK_RATIO", "0.4") | |
| os.environ.setdefault("PYTORCH_MPS_LOW_WATERMARK_RATIO", "0.3") | |
| import numpy as np | |
| import pyarrow.parquet as pq | |
| import torch | |
| import torch.nn.functional as F | |
| from transformers import AutoModelForSequenceClassification, AutoTokenizer | |
| sys.path.insert(0, os.path.dirname(os.path.abspath(__file__))) | |
| from s1d1_router import COMPLEXITY_CENTER, COMPLEXITY_SCALE, MAX_LENGTH, TASKS, TIERS, VOCAB_MAP, local_folder # noqa: E402 | |
| log = logging.getLogger("train") | |
| N_OUTPUTS = 1 + len(TASKS) + 1 | |
| TIER_SCORES = np.array([score for _, score in TIERS]) | |
| DEFAULT_BASE = "scombat/s1d1-router" | |
| OUTPUT_LAYOUT = ("0: complexity in tier units (score = 0.5 + 0.25 * logit, clipped to 0.01..1.00); " | |
| "1-19: task logits in task_names order; 20: needs_tools logit") | |
| LENGTH_BUCKET_BATCHES = 50 # batches per length-sorted bucket | |
| WARMUP_FRACTION = 0.05 | |
| WEIGHT_DECAY = 0.01 | |
| MAX_GRAD_NORM = 1.0 | |
| WEIGHT_EPS = 1e-12 | |
| PAD_MULTIPLE = 64 # padded widths are multiples of this: fewer distinct shapes, so a smaller allocator cache | |
| EMPTY_CACHE_EVERY = 25 # optimizer steps between two cache releases on MPS | |
| # --- data -------------------------------------------------------------------------------------- | |
| def read_rows(path: str, tokenizer, vocab_map: np.ndarray | None) -> dict: | |
| """The rows of a labelled parquet file, tokenized; `vocab_map` renumbers the token ids of a pruned vocabulary.""" | |
| t = pq.read_table(path).to_pydict() | |
| missing = [c for c in ("prompt", "complexity_score", "task", "needs_tools") if c not in t] | |
| if missing: | |
| sys.exit(f"{path}: missing columns {missing}") | |
| unknown = sorted({x for x in t["task"] if x not in TASKS}) | |
| if unknown: | |
| sys.exit(f"{path}: unknown task names {unknown[:10]} (expected one of {list(TASKS)})") | |
| complexity = np.array([np.nan if v is None else float(v) for v in t["complexity_score"]]) | |
| if np.nanmin(complexity) < 0 or np.nanmax(complexity) > 1: | |
| sys.exit(f"{path}: complexity_score must be in 0.01..1.00 or null") | |
| ids = tokenizer(t["prompt"], truncation=True, max_length=MAX_LENGTH)["input_ids"] | |
| if vocab_map is not None: | |
| ids = [vocab_map[np.asarray(x)].tolist() for x in ids] | |
| weight = np.array([1.0 if v is None else float(v) for v in t["weight"]]) if "weight" in t else np.ones(len(ids)) | |
| return { | |
| "input_ids": ids, "complexity": complexity, | |
| "task": np.array([TASKS.index(x) for x in t["task"]]), | |
| "needs_tools": np.array([bool(v) for v in t["needs_tools"]], dtype=np.float32), | |
| "weight": weight, "has_weights": "weight" in t, | |
| } | |
| def make_batches(lengths: list[int], batch_size: int, rng: random.Random) -> list[list[int]]: | |
| """Shuffle, sort buckets of LENGTH_BUCKET_BATCHES batches by length, cut into batches, shuffle the batches.""" | |
| order = list(range(len(lengths))) | |
| rng.shuffle(order) | |
| span = batch_size * LENGTH_BUCKET_BATCHES | |
| batches = [] | |
| for start in range(0, len(order), span): | |
| bucket = sorted(order[start:start + span], key=lambda i: lengths[i]) | |
| batches += [bucket[i:i + batch_size] for i in range(0, len(bucket), batch_size)] | |
| rng.shuffle(batches) | |
| return batches | |
| def micro_batches(batch: list[int], lengths: list[int], micro_tokens: int) -> list[list[int]]: | |
| """Cut a batch into length-sorted pieces of at most micro_tokens padded tokens (one row at least).""" | |
| batch = sorted(batch, key=lambda i: lengths[i]) | |
| out, current = [], [] | |
| for i in batch: | |
| if current and (len(current) + 1) * -(-lengths[i] // PAD_MULTIPLE) * PAD_MULTIPLE > micro_tokens: | |
| out.append(current) | |
| current = [] | |
| current.append(i) | |
| return out + [current] | |
| def collate(data: dict, rows: list[int], pad_id: int, device: str) -> dict: | |
| width = -(-max(len(data["input_ids"][i]) for i in rows) // PAD_MULTIPLE) * PAD_MULTIPLE | |
| ids = torch.full((len(rows), width), pad_id, dtype=torch.long) | |
| mask = torch.zeros((len(rows), width), dtype=torch.long) | |
| for k, i in enumerate(rows): | |
| n = len(data["input_ids"][i]) | |
| ids[k, :n] = torch.tensor(data["input_ids"][i]) | |
| mask[k, :n] = 1 | |
| return {"input_ids": ids.to(device), "attention_mask": mask.to(device)} | |
| # --- loss and metrics -------------------------------------------------------------------------- | |
| def loss_terms(logits: torch.Tensor, data: dict, rows: list[int], device: str) -> dict: | |
| """Weighted sums of the three losses over the rows (to be divided by the batch denominators).""" | |
| logits = logits.float() | |
| w = torch.tensor(data["weight"][rows], dtype=torch.float32, device=device) | |
| valid = torch.tensor(~np.isnan(data["complexity"][rows]), device=device) | |
| target = torch.tensor(np.nan_to_num((data["complexity"][rows] - COMPLEXITY_CENTER) / COMPLEXITY_SCALE), | |
| dtype=torch.float32, device=device) | |
| task = torch.tensor(data["task"][rows], device=device) | |
| tools = torch.tensor(data["needs_tools"][rows], device=device) | |
| return { | |
| "complexity": (w * valid * (logits[:, 0] - target) ** 2).sum(), | |
| "task": (w * F.cross_entropy(logits[:, 1:1 + len(TASKS)], task, reduction="none")).sum(), | |
| "needs_tools": (w * F.binary_cross_entropy_with_logits(logits[:, -1], tools, reduction="none")).sum(), | |
| } | |
| def denominators(data: dict, rows: list[int]) -> dict: | |
| w, valid = data["weight"][rows], ~np.isnan(data["complexity"][rows]) | |
| return {"complexity": max(float((w * valid).sum()), WEIGHT_EPS), "task": float(w.sum()), "needs_tools": float(w.sum())} | |
| def weighted_spearman(x: np.ndarray, y: np.ndarray, w: np.ndarray) -> float: | |
| """Weighted Pearson correlation of the weighted mid-ranks (equals Spearman when all weights are equal).""" | |
| def midranks(v): | |
| order = np.argsort(v, kind="stable") | |
| cum = np.cumsum(w[order]) - w[order] / 2 | |
| ranks = np.empty(len(v)) | |
| for value in np.unique(v): | |
| group = v[order] == value | |
| ranks[order[group]] = np.average(cum[group], weights=w[order][group]) | |
| return ranks | |
| rx, ry = midranks(x), midranks(y) | |
| mx, my = np.average(rx, weights=w), np.average(ry, weights=w) | |
| return float(np.sum(w * (rx - mx) * (ry - my)) / math.sqrt(np.sum(w * (rx - mx) ** 2) * np.sum(w * (ry - my) ** 2))) | |
| def metrics(data: dict, out: np.ndarray, weighted: bool) -> dict: | |
| """Agreement of the outputs `out` with the labels: complexity (MAE, Spearman, quadratic kappa on the nearest tier, | |
| confusion matrix), task (accuracy, macro-F1, F1 per class), needs_tools (F1 at 0.5, AUROC).""" | |
| from sklearn.metrics import cohen_kappa_score, confusion_matrix, f1_score, roc_auc_score | |
| w = data["weight"] if weighted else np.ones(len(out)) | |
| res = {"rows": len(out), "weighted": weighted} | |
| ok = ~np.isnan(data["complexity"]) | |
| score = np.clip(COMPLEXITY_CENTER + COMPLEXITY_SCALE * out[ok, 0], 0.01, 1.0) | |
| true, wc = data["complexity"][ok], w[ok] | |
| pred_tier = np.abs(score[:, None] - TIER_SCORES[None]).argmin(1) | |
| true_tier = np.abs(true[:, None] - TIER_SCORES[None]).argmin(1) | |
| tiers = [name for name, _ in TIERS] | |
| res["complexity"] = { | |
| "rows": int(ok.sum()), "mae": float(np.average(np.abs(score - true), weights=wc)), | |
| "spearman": weighted_spearman(score, true, wc), | |
| "qwk_rounded": float(cohen_kappa_score(true_tier, pred_tier, weights="quadratic", sample_weight=wc)), | |
| "confusion_true_by_pred": confusion_matrix(true_tier, pred_tier, labels=range(len(tiers)), sample_weight=wc).round(1).tolist(), | |
| "tiers": tiers, | |
| } | |
| pred_task, true_task = out[:, 1:1 + len(TASKS)].argmax(1), data["task"] | |
| f1 = f1_score(true_task, pred_task, labels=range(len(TASKS)), average=None, sample_weight=w, zero_division=0) | |
| present = np.bincount(true_task, minlength=len(TASKS)) > 0 | |
| res["task"] = { | |
| "accuracy": float(np.average(pred_task == true_task, weights=w)), | |
| "macro_f1_present_classes": float(f1[present].mean()), | |
| "per_class": {t: {"f1": float(f1[k]), "support": int((true_task == k).sum())} for k, t in enumerate(TASKS)}, | |
| } | |
| prob, true_tools = 1 / (1 + np.exp(-out[:, -1])), data["needs_tools"] | |
| res["needs_tools"] = { | |
| "f1": float(f1_score(true_tools, prob >= 0.5, sample_weight=w, zero_division=0)), | |
| "auroc": float(roc_auc_score(true_tools, prob, sample_weight=w)) if 0 < true_tools.sum() < len(true_tools) else None, | |
| "positives": int(true_tools.sum()), | |
| } | |
| return res | |
| def predict(model, data: dict, device: str, pad_id: int, precision: str, micro_tokens: int) -> np.ndarray: | |
| model.eval() | |
| lengths = [len(x) for x in data["input_ids"]] | |
| order = sorted(range(len(lengths)), key=lambda i: lengths[i]) | |
| out = np.zeros((len(lengths), N_OUTPUTS), dtype=np.float32) | |
| for rows in micro_batches(order, lengths, micro_tokens): | |
| with torch.autocast(device, dtype=torch.bfloat16, enabled=precision == "bf16"): | |
| logits = model(**collate(data, rows, pad_id, device)).logits | |
| out[rows] = logits.float().cpu().numpy() | |
| model.train() | |
| return out | |
| # --- training ---------------------------------------------------------------------------------- | |
| def parse_args(argv: list[str]) -> argparse.Namespace: | |
| p = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) | |
| p.add_argument("--train", required=True, help="parquet file of labelled prompts") | |
| p.add_argument("--validation", help="parquet file evaluated after every epoch (same columns)") | |
| p.add_argument("--out", required=True, help="folder of the trained model (written after every epoch)") | |
| p.add_argument("--base", default=DEFAULT_BASE, help="Hub repo or folder to start from (PyTorch weights)") | |
| p.add_argument("--epochs", type=int, default=3) | |
| p.add_argument("--lr", type=float, default=3e-5) | |
| p.add_argument("--batch-size", type=int, default=32, help="rows per optimizer step") | |
| p.add_argument("--micro-tokens", type=int, default=4096, help="padded tokens per forward/backward pass") | |
| p.add_argument("--eval-micro-tokens", type=int, default=16384, help="padded tokens per forward pass in evaluation") | |
| p.add_argument("--precision", choices=("fp32", "bf16"), default="fp32", help="bf16 = autocast, weights stay in fp32") | |
| p.add_argument("--device", default="auto", help="auto (cuda, then mps, then cpu), cuda, mps or cpu") | |
| p.add_argument("--seed", type=int, default=42) | |
| return p.parse_args(argv) | |
| def pick_device(wanted: str) -> str: | |
| if wanted != "auto": | |
| return wanted | |
| if torch.cuda.is_available(): | |
| return "cuda" | |
| return "mps" if torch.backends.mps.is_available() else "cpu" | |
| def synchronize(device: str) -> None: | |
| if device.startswith("cuda"): | |
| torch.cuda.synchronize() | |
| elif device == "mps": | |
| torch.mps.synchronize() | |
| def save(model, tokenizer, base_folder: str, out: str, vocab_map: np.ndarray | None) -> None: | |
| model.config.problem_type = "regression" # the text-classification pipeline then returns the raw outputs | |
| model.config.task_names = list(TASKS) | |
| model.config.output_layout = OUTPUT_LAYOUT | |
| model.save_pretrained(out) | |
| tokenizer.save_pretrained(out) | |
| if vocab_map is not None: | |
| np.save(os.path.join(out, VOCAB_MAP), vocab_map) | |
| def main(argv: list[str]) -> None: | |
| args = parse_args(argv) | |
| logging.basicConfig(level=logging.INFO, format="%(asctime)s %(levelname)s %(message)s", stream=sys.stderr) | |
| device = pick_device(args.device) | |
| random.seed(args.seed) | |
| np.random.seed(args.seed) | |
| torch.manual_seed(args.seed) | |
| os.makedirs(args.out, exist_ok=True) | |
| base = local_folder(args.base) | |
| tokenizer = AutoTokenizer.from_pretrained(base) | |
| map_path = os.path.join(base, VOCAB_MAP) | |
| vocab_map = np.load(map_path) if os.path.isfile(map_path) else None | |
| pad_id = int(vocab_map[tokenizer.pad_token_id]) if vocab_map is not None else tokenizer.pad_token_id | |
| train = read_rows(args.train, tokenizer, vocab_map) | |
| valid = read_rows(args.validation, tokenizer, vocab_map) if args.validation else None | |
| log.info("device %s, train %d rows (%d with a complexity, %d needs_tools), base %s", device, len(train["input_ids"]), | |
| int((~np.isnan(train["complexity"])).sum()), int(train["needs_tools"].sum()), args.base) | |
| model = AutoModelForSequenceClassification.from_pretrained(base, num_labels=N_OUTPUTS, dtype=torch.float32).to(device) | |
| model.train() | |
| no_decay = ("bias", "norm") | |
| groups = [{"params": [p for n, p in model.named_parameters() if not any(k in n for k in no_decay)], "weight_decay": WEIGHT_DECAY}, | |
| {"params": [p for n, p in model.named_parameters() if any(k in n for k in no_decay)], "weight_decay": 0.0}] | |
| optimizer = torch.optim.AdamW(groups, lr=args.lr) | |
| lengths = [len(x) for x in train["input_ids"]] | |
| rng = random.Random(args.seed) | |
| total_steps = math.ceil(len(lengths) / args.batch_size) * args.epochs | |
| warmup = max(1, int(WARMUP_FRACTION * total_steps)) | |
| schedule = torch.optim.lr_scheduler.LambdaLR( | |
| optimizer, lambda s: min((s + 1) / warmup, max(0.0, (total_steps - s) / (total_steps - warmup)))) | |
| history, validation, step = [], [], 0 | |
| started = time.monotonic() | |
| for epoch in range(args.epochs): | |
| for batch in make_batches(lengths, args.batch_size, rng): | |
| denom = denominators(train, batch) | |
| batch_loss = collections.Counter() | |
| for rows in micro_batches(batch, lengths, args.micro_tokens): | |
| with torch.autocast(device, dtype=torch.bfloat16, enabled=args.precision == "bf16"): | |
| logits = model(**collate(train, rows, pad_id, device)).logits | |
| terms = {k: v / denom[k] for k, v in loss_terms(logits, train, rows, device).items()} | |
| sum(terms.values()).backward() | |
| for k, v in terms.items(): | |
| batch_loss[k] += v.item() | |
| grad_norm = torch.nn.utils.clip_grad_norm_(model.parameters(), MAX_GRAD_NORM).item() | |
| if not math.isfinite(grad_norm) or not all(math.isfinite(v) for v in batch_loss.values()): | |
| raise RuntimeError(f"step {step}: non-finite loss or gradient ({dict(batch_loss)}, grad norm {grad_norm})") | |
| optimizer.step() | |
| schedule.step() | |
| optimizer.zero_grad(set_to_none=True) | |
| step += 1 | |
| if device == "mps" and step % EMPTY_CACHE_EVERY == 0: | |
| torch.mps.empty_cache() | |
| if step % 100 == 0 or step == total_steps: | |
| synchronize(device) | |
| history.append({"step": step, "loss": dict(batch_loss), "grad_norm": grad_norm, "lr": schedule.get_last_lr()[0], | |
| "seconds": round(time.monotonic() - started)}) | |
| log.info("step %d/%d loss %s grad %.2f %.0f s", step, total_steps, | |
| {k: round(v, 4) for k, v in batch_loss.items()}, grad_norm, time.monotonic() - started) | |
| if valid is not None: | |
| out = predict(model, valid, device, pad_id, args.precision, args.eval_micro_tokens) | |
| res = {"epoch": epoch + 1, "unweighted": metrics(valid, out, False)} | |
| if valid["has_weights"]: | |
| res["weighted"] = metrics(valid, out, True) | |
| validation.append(res) | |
| u = res["unweighted"] | |
| log.info("epoch %d: validation MAE %.3f, QWK %.3f, task accuracy %.3f, needs_tools F1 %s AUROC %s", epoch + 1, | |
| u["complexity"]["mae"], u["complexity"]["qwk_rounded"], u["task"]["accuracy"], | |
| round(u["needs_tools"]["f1"], 3), None if u["needs_tools"]["auroc"] is None else round(u["needs_tools"]["auroc"], 3)) | |
| save(model, tokenizer, base, args.out, vocab_map) | |
| with open(os.path.join(args.out, "training.json"), "w", encoding="utf-8") as f: | |
| json.dump({"args": vars(args), "device": device, "steps": step, "seconds": round(time.monotonic() - started), | |
| "history": history, "validation": validation}, f, indent=1) | |
| log.info("epoch %d saved to %s", epoch + 1, args.out) | |
| if __name__ == "__main__": | |
| main(sys.argv[1:]) | |