# Têtu Code 2 — inference code (tetu-AI). License: Apache 2.0, see README.md. from pathlib import Path import math import numpy as np import torch import torch.nn.functional as F from torch.nn.attention import SDPBackend, sdpa_kernel from tokenizers import Tokenizer from tetu_code.model import TetuCode, TetuCodeConfig, apply_rope, rms, rope_table, unpack_struct from tetu_code.structure import Tables, features ROOT = Path(__file__).resolve().parents[1] def attend(q, k, v, mask): B, H, T, hd = q.shape G = H // k.size(1) if T == 1: with sdpa_kernel(SDPBackend.MATH): y = F.scaled_dot_product_attention(q.reshape(B, H // G, G, hd), k, v, attn_mask=mask) return y.reshape(B, H, 1, hd) return F.scaled_dot_product_attention(q, k.repeat_interleave(G, 1), v.repeat_interleave(G, 1), attn_mask=mask) def load(path, device='cuda', dtype=torch.bfloat16, tokenizer=None): if Path(path).is_dir(): import json from safetensors.torch import load_file meta = json.loads((Path(path) / 'config.json').read_text(encoding='utf-8')) ck = dict(meta, model=load_file(str(Path(path) / 'model.safetensors'))) else: ck = torch.load(path, map_location='cpu', weights_only=False) cfg = TetuCodeConfig(**ck['config']) with torch.device(device): model = TetuCode(cfg) model.to(device) model.load_state_dict(ck['model']) if torch.device(device).type != 'cpu': with torch.device('cpu'): for name, theta in (('loc', cfg.rope_local), ('glo', cfg.rope_global)): cosine, sine = rope_table(cfg.max_seq_len, cfg.d_model // cfg.n_heads, theta) setattr(model, f'cos_{name}', cosine.to(device)) setattr(model, f'sin_{name}', sine.to(device)) for name, p in model.named_parameters(): if not name.startswith('doubt_head'): p.data = p.data.to(dtype) model.to(device).eval() model.meta = {k: ck.get(k) for k in ('step', 'tokens', 'val')} return (model, Tokenizer.from_file(str(tokenizer or ROOT / 'tokenizer.json'))) class Decoder: def __init__(self, model, tok, max_len=32768, ring=4096, chunk=2048, graph=True, compile=False, memo=None): self.memo = memo cfg = model.cfg self.model, self.tok, self.cfg = (model, tok, cfg) self.dev = next(model.parameters()).device dt = model.tok_emb.weight.dtype hd = cfg.d_model // cfg.n_heads self.max_len, self.ring, self.chunk, self.sw = (max_len, ring, chunk, cfg.window // 128) assert ring >= chunk + cfg.window, 'anneau trop petit pour le pré-remplissage par morceaux' self.S = [max_len if b.attn.is_global else ring for b in model.blocks] self.K = [torch.zeros(1, cfg.n_kv_heads, s, hd, device=self.dev, dtype=dt) for s in self.S] self.V = [torch.zeros_like(k) for k in self.K] self.ring_pos = torch.full((ring,), -(1 << 40), dtype=torch.long, device=self.dev) self.tok_ring = torch.full((ring,), cfg.vocab_size, dtype=torch.long, device=self.dev) self.eng = [i for i, b in enumerate(model.blocks) if b.engram is not None] self.order = max((max(o) for _, o in cfg.engram)) self.s_ring = {i: torch.zeros(1, ring, cfg.d_model, device=self.dev, dtype=dt) for i in self.eng} self.gidx = torch.arange(max_len, device=self.dev) self.tab = Tables(tok).arrays self.flag = self.tab[2] self.ids, self.st, self.reset_at = ([], [], 0) self.t_in = torch.zeros(1, 1, dtype=torch.long, device=self.dev) self.s_in = torch.zeros(1, 1, dtype=torch.int16, device=self.dev) self.p_in = torch.zeros(1, dtype=torch.long, device=self.dev) self.temp_t = torch.zeros((), device=self.dev) self.top_p_t = torch.full((), 0.95, device=self.dev) self.noise = torch.ones(50, device=self.dev) self.nxt = torch.zeros((), dtype=torch.long, device=self.dev) self._doubt = None self.graphs = {} if graph and self.dev.type == 'cuda': step = self._step if compile: import torch._dynamo as dynamo dynamo.config.cache_size_limit = max(dynamo.config.cache_size_limit, 16) step = torch.compile(self._step, dynamic=False) for n in sorted({min(4096 << i, max_len) for i in range(8) if (4096 << i) // 2 < max_len}): s = torch.cuda.Stream() s.wait_stream(torch.cuda.current_stream()) with torch.cuda.stream(s): for _ in range(3): self.noise.exponential_() step(n) torch.cuda.current_stream().wait_stream(s) g = torch.cuda.CUDAGraph() with torch.cuda.graph(g): self.noise.exponential_() self.graphs[n] = (g, step(n)) self.reset() @property def last_doubt(self): return float(self._doubt[0, -1]) if self._doubt is not None else 0.0 def reset(self): self.ring_pos.fill_(-(1 << 40)) self.tok_ring.fill_(self.cfg.vocab_size) for r in self.s_ring.values(): r.zero_() self.ids, self.st, self.reset_at = ([], [], 0) def _struct(self, new): s0 = self.reset_at st = features(np.asarray(self.ids[s0:] + list(new), dtype=np.int64), *self.tab) return [int(v) for v in st[len(self.ids) - s0:]] def _append(self, toks, st): for t in toks: if self.flag[t] == 1: self.reset_at = len(self.ids) self.ids.append(t) self.st += st def rollback(self, n): if n <= 0: return True if n > self.ring - self.cfg.window - 128: return False p = len(self.ids) - n self.ring_pos.masked_fill_(self.ring_pos >= p, -(1 << 40)) del self.ids[p:], self.st[p:] r = np.flatnonzero(self.flag[np.asarray(self.ids, dtype=np.int64)] == 1) if self.ids else [] self.reset_at = int(r[-1]) if len(r) else 0 return True def sync(self, ids): ids = list(ids) k = 0 n = min(len(ids), len(self.ids)) while k < n and ids[k] == self.ids[k]: k += 1 if k == len(ids) and k > 0: k -= 1 if not self.rollback(len(self.ids) - k): self.reset() k = 0 return self.feed(ids[k:]) @torch.no_grad() def feed(self, new): assert len(self.ids) + len(new) < self.max_len, 'contexte trop long' out = None for a in range(0, len(new), self.chunk): part = new[a:a + self.chunk] if len(part) == 1 and self.graphs: out = self.step(part[0]) else: out = self._prefill(part) return out def step(self, t): if not self.graphs: return self._prefill([t]) st = self._struct([t])[0] n = len(self.ids) g, out = next((v for k, v in self.graphs.items() if k > n)) self.t_in.fill_(t) self.s_in.fill_(st) self.p_in.fill_(n) g.replay() self._append([t], [st]) return out def _rope(self, pos): m = self.model qs = (torch.log(pos.float() + 1) / math.log(self.cfg.log_base)).clamp_min(1.0) return ((m.cos_loc[pos], m.sin_loc[pos]), (m.cos_glo[pos], m.sin_glo[pos]), qs) def _embed(self, idx, st): m = self.model x = m.tok_emb(idx) for e, f in zip(m.struct_emb, unpack_struct(st)): x = x + e(f) return rms(x) def _layers(self, x, idx, pos, single): m, cfg = (self.model, self.cfg) T = idx.size(1) loc, glo, qs = self._rope(pos) L, skips = (cfg.n_layers, []) slot = pos % self.ring self.tok_ring.index_copy_(0, slot, idx[0]) self.ring_pos.index_copy_(0, slot, pos) for i, blk in enumerate(m.blocks): if i >= L - L // 2: x = x + m.skip_w[L - 1 - i] * skips.pop() if blk.engram is not None: eng, xe = (blk.engram, rms(x)) back = pos[None, :] - torch.arange(self.order - 1, -1, -1, device=self.dev)[:, None] ctx = torch.where(back >= 0, self.tok_ring[back % self.ring], torch.full_like(back, cfg.eos_id)) e = eng.table(eng.slots(ctx.t().contiguous())[:, -1]).reshape(1, T, -1).to(x.dtype) k, v = (eng.wk(e), eng.wv(e)) alpha = torch.sigmoid((rms(xe) * rms(k)).sum(-1, keepdim=True) / math.sqrt(cfg.d_model)) s = alpha * v self.s_ring[i][0].index_copy_(0, slot, s[0]) prev = pos[None, :] - torch.arange(3, 0, -1, device=self.dev)[:, None] ps = torch.where((prev >= 0)[..., None], self.s_ring[i][0][prev % self.ring], torch.zeros_like(s[0])) win = torch.cat([ps, s], dim=0).permute(1, 2, 0) c = (win * eng.conv.weight[:, 0, :]).sum(-1) x = x + s + F.silu(c[None]) att, xa = (blk.attn, rms(x)) q = att.wq(xa).view(1, T, att.H, att.hd).transpose(1, 2) k = att.wk(xa).view(1, T, att.Hk, att.hd).transpose(1, 2) v = att.wv(xa).view(1, T, att.Hk, att.hd).transpose(1, 2) j = m.ve_layer.get(i) if j is not None: v = v + m.ve_lambda[i] * m.value_embeds[j](idx).to(v.dtype).view(1, T, att.Hk, att.hd).transpose(1, 2) c_, s_ = glo if att.is_global else loc q, k = (apply_rope(rms(q), c_, s_), apply_rope(rms(k), c_, s_)) if att.is_global: q = q * qs.to(q.dtype)[:, None] self.K[i].index_copy_(2, pos, k) self.V[i].index_copy_(2, pos, v) if single: Kc, Vc = (self.K[i][:, :, :single], self.V[i][:, :, :single]) mask = (self.gidx[:single] <= pos[0])[None, None, None] else: end = int(pos[-1]) + 1 Kc, Vc = (self.K[i][:, :, :end], self.V[i][:, :, :end]) kp = self.gidx[:end] mask = (kp[None] <= pos[:, None])[None, None] else: self.K[i].index_copy_(2, slot, k) self.V[i].index_copy_(2, slot, v) if single: rp = self.ring_pos Kc, Vc = (self.K[i], self.V[i]) mask = ((rp <= pos[0]) & (rp // 128 > pos[0] // 128 - self.sw))[None, None, None] else: k0 = max(0, (int(pos[0]) // 128 - self.sw + 1) * 128) kp = torch.arange(k0, int(pos[-1]) + 1, device=self.dev) Kc, Vc = (self.K[i][:, :, kp % self.ring], self.V[i][:, :, kp % self.ring]) mask = ((kp[None] <= pos[:, None]) & (pos[:, None] // 128 - kp[None] // 128 < self.sw))[None, None] y = attend(q, Kc, Vc, mask) x = x + att.wo(y.transpose(1, 2).reshape(1, T, -1) * torch.sigmoid(att.wg(xa))) x = x + blk.mlp(rms(x)) if self.memo is not None and i == self.memo.L: if getattr(self.memo, 'exact', False): back = pos[None, :] - torch.arange(self.memo.K - 1, -1, -1, device=self.dev)[:, None] ctx = torch.where(back >= 0, self.tok_ring[back % self.ring], torch.full_like(back, -1)) x = self.memo(x, ctx.t()[None]) else: x = self.memo(x) if i < L // 2: skips.append(x) return rms(x) @torch.no_grad() def _step(self, n): h = self._layers(self._embed(self.t_in, self.s_in), self.t_in, self.p_in, n) logits = self.model.logits(h) z = logits[0, -1].float() v, i = torch.topk(z / self.temp_t.clamp_min(1e-06), 50) p = torch.softmax(v, -1) p = torch.where(torch.cumsum(p, -1) - p < self.top_p_t, p, torch.zeros_like(p)) pick = i.gather(0, torch.argmax(p / self.noise).view(1))[0] self.nxt.copy_(torch.where(self.temp_t > 0, pick, z.argmax())) return (logits, torch.sigmoid(self.model.doubt_head(h, logits))) @torch.no_grad() def _prefill(self, part): st = self._struct(part) idx = torch.tensor([part], device=self.dev) sx = torch.tensor([st], device=self.dev, dtype=torch.int16) pos = torch.arange(len(self.ids), len(self.ids) + len(part), device=self.dev) h = self._layers(self._embed(idx, sx), idx, pos, 0)[:, -1:] self._append(list(part), st) logits = self.model.logits(h) return (logits, torch.sigmoid(self.model.doubt_head(h, logits))) @torch.no_grad() def generate(self, ids, max_new=512, temp=0.0, top_p=0.95, top_k=50, stop=(), seed=0): g = torch.Generator(device=self.dev).manual_seed(seed) self.temp_t.fill_(max(temp, 0.0)) self.top_p_t.fill_(top_p) in_graph = bool(self.graphs) and top_k == 50 logits, doubt = self.sync(ids) stop = set(stop) t = None for _ in range(max_new): if t is None: z = logits[0, -1].float() if temp <= 0: t = int(z.argmax()) else: v, i = torch.topk(z / temp, top_k) p = torch.softmax(v, -1) keep = torch.cumsum(p, -1) - p < top_p p = torch.where(keep, p, torch.zeros_like(p)) t = int(i[torch.multinomial(p / p.sum(), 1, generator=g)]) self._doubt = doubt yield t if t in stop: return logits, doubt = self.step(t) t = int(self.nxt) if in_graph else None