""" Alpha-S1: a System-One-style model. Unstructured / structured state in, typed probabilistic decisions out. * No text generation. Every question carries its own option list; the model can only score those options (masked softmax) -> a type error / out-of-schema answer is impossible by construction. * All questions of a query are answered in ONE parallel forward pass (no sequential decoding). * Options are embedded independently of the state (bi-encoder side) so a schema can be "compiled" once and cached; cardinality is limited only by memory (hundreds of options are cheap). * Slot Reasoner: a small set-transformer (squared-ReLU FFN) lets the questions of one query condition on each other. * Calibration: per-question learned temperature + post-hoc global temperature; outputs probs, confidence, certainty. """ import math, json, os from dataclasses import dataclass, asdict import torch import torch.nn as nn import torch.nn.functional as F from transformers import AutoModel, AutoConfig, AutoTokenizer @dataclass class AlphaConfig: backbone: str = "distilroberta-base" d_head: int = 256 slot_layers: int = 2 slot_heads: int = 8 act: str = "relu2" # "relu2" (squared ReLU, high-accessibility prior) | "gelu" max_len: int = 512 field_max: int = 96 opt_max: int = 16 class Tk: """Tokenizer wrapper: no truncation args (thread-safe), caches short strings.""" def __init__(self, tok): self.tok, self.c = tok, {} def __call__(self, text, maxlen=None): ids = self.c.get(text) if ids is None: ids = self.tok(text, add_special_tokens=False)["input_ids"] if len(text) < 200: self.c[text] = ids return ids[:maxlen] if maxlen else ids def opt_batch(texts, tk, tokz, cfg): cls, sep, pad = tokz.cls_token_id, tokz.sep_token_id, tokz.pad_token_id seqs = [[cls] + tk(" " + t)[:cfg.opt_max] + [sep] for t in texts] L = max(len(s) for s in seqs) ids = torch.full((len(seqs), L), pad, dtype=torch.long); att = torch.zeros((len(seqs), L), dtype=torch.long) for i, s in enumerate(seqs): ids[i, :len(s)] = torch.tensor(s); att[i, :len(s)] = 1 return ids, att def build_batch(exs, tk, tokz, cfg): """exs: [{"fields":[(name,text),..], "questions":[{"q":str,"options":[str],"gold":int,"ordinal":bool,"fam":str}]}]""" cls, sep, msk, pad = tokz.cls_token_id, tokz.sep_token_id, tokz.mask_token_id, tokz.pad_token_id seqs, qposs, rows = [], [], [] opt_index, opt_texts = {}, [] for bi, ex in enumerate(exs): qids = [tk(" " + q["q"]) for q in ex["questions"]] qtot = sum(len(x) + 1 for x in qids) + 3 nf = max(len(ex["fields"]), 1) per = max(8, min(cfg.field_max, (cfg.max_len - qtot) // nf - 8)) while True: ids = [cls] for name, text in ex["fields"]: ids += tk(f" {name}:") + tk(" " + text, per) ids.append(sep) pos = [] for x in qids: ids += x; ids.append(msk); pos.append(len(ids) - 1) ids.append(sep) if len(ids) <= cfg.max_len or per <= 8: break per -= 8 ids = ids[:cfg.max_len]; pos = [min(p, cfg.max_len - 1) for p in pos] seqs.append(ids); qposs.append(pos) for q in ex["questions"]: idxs = [] for o in q["options"]: if o not in opt_index: opt_index[o] = len(opt_texts); opt_texts.append(o) idxs.append(opt_index[o]) rows.append((idxs, q.get("gold", 0), bool(q.get("ordinal", False)), q.get("fam", ""), bi)) B, T = len(seqs), max(len(s) for s in seqs) ids = torch.full((B, T), pad, dtype=torch.long); att = torch.zeros((B, T), dtype=torch.long) kmax = max(len(p) for p in qposs) qpos = torch.zeros((B, kmax), dtype=torch.long); qvalid = torch.zeros((B, kmax), dtype=torch.bool) for i, (s, p) in enumerate(zip(seqs, qposs)): ids[i, :len(s)] = torch.tensor(s); att[i, :len(s)] = 1 qpos[i, :len(p)] = torch.tensor(p); qvalid[i, :len(p)] = True nQ, nO = len(rows), len(opt_texts) allowed = torch.zeros((nQ, nO), dtype=torch.bool); rank = torch.zeros((nQ, nO)) gold = torch.zeros(nQ, dtype=torch.long); ordn = torch.zeros(nQ, dtype=torch.bool) for r, (idxs, g, od, fam, bi) in enumerate(rows): allowed[r, idxs] = True rank[r, idxs] = torch.arange(len(idxs)).float() / max(len(idxs) - 1, 1) gold[r] = idxs[g]; ordn[r] = od oi, oa = opt_batch(opt_texts, tk, tokz, cfg) return dict(ids=ids, att=att, qpos=qpos, qvalid=qvalid, opt_ids=oi, opt_att=oa, allowed=allowed, rank=rank, gold=gold, ordinal=ordn, opt_texts=opt_texts, opt_lists=[r[0] for r in rows], gold_pos=[r[1] for r in rows], fams=[r[3] for r in rows], q_ex=[r[4] for r in rows]) def to_device(b, dev): return {k: (v.to(dev, non_blocking=True) if torch.is_tensor(v) else v) for k, v in b.items()} class EncWrap(nn.Module): """Backbone call with autocast INSIDE forward (required for nn.DataParallel replicas).""" def __init__(self, m): super().__init__(); self.m = m def forward(self, ids, att): with torch.autocast("cuda", dtype=torch.float16, enabled=ids.is_cuda): return self.m(input_ids=ids, attention_mask=att).last_hidden_state class SlotLayer(nn.Module): """Pre-LN transformer layer over the query slots of one example; squared-ReLU FFN; zero-init residual branches (starts as identity -> the generalizing 'read straight from the encoder' solution is the easiest to reach).""" def __init__(self, d, heads, act): super().__init__() self.n1, self.n2 = nn.LayerNorm(d), nn.LayerNorm(d) self.att = nn.MultiheadAttention(d, heads, batch_first=True) self.f1, self.f2, self.act = nn.Linear(d, 4 * d), nn.Linear(4 * d, d), act nn.init.zeros_(self.att.out_proj.weight); nn.init.zeros_(self.att.out_proj.bias) nn.init.zeros_(self.f2.weight); nn.init.zeros_(self.f2.bias) def forward(self, x, pad): y = self.n1(x) a, _ = self.att(y, y, y, key_padding_mask=pad, need_weights=False) x = x + a h = self.f1(self.n2(x)) h = F.relu(h) ** 2 if self.act == "relu2" else F.gelu(h) return x + self.f2(h) class Alpha(nn.Module): def __init__(self, cfg=None, tokenizer=None, pretrained=True, backbone_module=None): super().__init__() self.cfg = cfg or AlphaConfig() self.tokz = tokenizer or AutoTokenizer.from_pretrained(self.cfg.backbone) self.tk = Tk(self.tokz) if backbone_module is not None: self.backbone = backbone_module elif pretrained: try: self.backbone = AutoModel.from_pretrained(self.cfg.backbone, add_pooling_layer=False) except TypeError: self.backbone = AutoModel.from_pretrained(self.cfg.backbone) else: c = AutoConfig.from_pretrained(self.cfg.backbone) try: self.backbone = AutoModel.from_config(c, add_pooling_layer=False) except TypeError: self.backbone = AutoModel.from_config(c) d = self.backbone.config.hidden_size self.ln = nn.LayerNorm(d) self.slots = nn.ModuleList([SlotLayer(d, self.cfg.slot_heads, self.cfg.act) for _ in range(self.cfg.slot_layers)]) self.q_proj, self.o_proj = nn.Linear(d, self.cfg.d_head), nn.Linear(d, self.cfg.d_head) self.temp = nn.Linear(d, 1) # per-question (heteroscedastic) temperature nn.init.zeros_(self.temp.weight); nn.init.constant_(self.temp.bias, 0.11) self.log_scale = nn.Parameter(torch.tensor(math.log(10.0))) self.register_buffer("post_T", torch.tensor(1.0)) # post-hoc global temperature (fit on held-out calibration set) object.__setattr__(self, "_enc", EncWrap(self.backbone)) # not registered: shares backbone params object.__setattr__(self, "_dp", None) self._opt_cache = {} # ---- plumbing ---- def use_data_parallel(self): if torch.cuda.device_count() > 1: object.__setattr__(self, "_dp", nn.DataParallel(EncWrap(self.backbone))) return self def train(self, mode=True): self._opt_cache = {} return super().train(mode) def new_params(self): bb = {id(p) for p in self.backbone.parameters()} return [p for p in self.parameters() if id(p) not in bb] def encode(self, ids, att): enc = self._dp if (self._dp is not None and ids.size(0) >= 8) else self._enc return enc(ids, att) def embed_options(self, opt_ids, opt_att): H = self.encode(opt_ids, opt_att).float(); m = opt_att.unsqueeze(-1).float() return (H * m).sum(1) / m.sum(1).clamp(min=1) # ---- core: all questions, all options, one pass ---- def forward(self, b, opt_emb=None): H = self.encode(b["ids"], b["att"]) B = H.size(0) hq = H[torch.arange(B, device=H.device)[:, None], b["qpos"]].float() hq = self.ln(hq) pad = ~b["qvalid"] for layer in self.slots: hq = layer(hq, pad) hq = hq[b["qvalid"]] # [nQ, d] row-major == question order in batch if opt_emb is None: opt_emb = self.embed_options(b["opt_ids"], b["opt_att"]) q = F.normalize(self.q_proj(hq).float(), dim=-1); o = F.normalize(self.o_proj(opt_emb).float(), dim=-1) T = F.softplus(self.temp(hq).float()) + 0.25 logits = (q @ o.T) * self.log_scale.exp() / T return logits.masked_fill(~b["allowed"], -1e4) # type safety: only declared options exist # ---- public API ---- @torch.no_grad() def predict(self, state, questions): """state: {field: text}; questions: [{"id","question","options":[...],"ordinal":bool}] -> {id: {...}}""" dev = next(self.parameters()).device if self.training: self.eval() # (train() clears the option cache) ex = dict(fields=list(state.items()), questions=[dict(q=q["question"], options=list(q["options"]), gold=0, ordinal=q.get("ordinal", False)) for q in questions]) b = to_device(build_batch([ex], self.tk, self.tokz, self.cfg), dev) miss = [t for t in b["opt_texts"] if t not in self._opt_cache] if miss: # "schema compilation": option embeddings cached oi, oa = opt_batch(miss, self.tk, self.tokz, self.cfg) emb = self.embed_options(oi.to(dev), oa.to(dev)) for t, e in zip(miss, emb): self._opt_cache[t] = e opt_emb = torch.stack([self._opt_cache[t] for t in b["opt_texts"]]) logits = self.forward(b, opt_emb) / self.post_T out = {} for r, q in enumerate(questions): idx = b["opt_lists"][r] p = F.softmax(logits[r, idx], -1).cpu() K = len(idx); j = int(p.argmax()) ent = float(-(p * p.clamp_min(1e-9).log()).sum()) out[q["id"]] = dict(answer=q["options"][j], probs={o: float(x) for o, x in zip(q["options"], p)}, confidence=float(p[j]), certainty=(1 - ent / math.log(K)) if K > 1 else 1.0) return out def save_pretrained(self, path): os.makedirs(path, exist_ok=True) torch.save({k: v.detach().cpu() for k, v in self.state_dict().items()}, f"{path}/pytorch_model.bin") json.dump(asdict(self.cfg), open(f"{path}/alpha_config.json", "w"), indent=1) self.tokz.save_pretrained(path) @classmethod def from_pretrained(cls, path_or_repo, device=None): if not os.path.isdir(path_or_repo): from huggingface_hub import snapshot_download path_or_repo = snapshot_download(path_or_repo) cfg = AlphaConfig(**json.load(open(f"{path_or_repo}/alpha_config.json"))) m = cls(cfg, AutoTokenizer.from_pretrained(path_or_repo), pretrained=False) missing, unexpected = m.load_state_dict(torch.load(f"{path_or_repo}/pytorch_model.bin", map_location="cpu"), strict=False) missing = [k for k in missing if "pooler" not in k] assert not missing, f"missing keys: {missing[:5]}" return m.to(device) if device else m