Download tetu_code/generate.py from tetu-AI/Tetu-Code-2: direct link, hf CLI and curl.
- Browser
- Download file 13.9 kB
-
https://huggingface.co/tetu-AI/Tetu-Code-2/resolve/main/tetu_code/generate.py
- Command line
-
hf download hf://tetu-AI/Tetu-Code-2/tetu_code/generate.py
-
curl -L -o generate.py https://huggingface.co/tetu-AI/Tetu-Code-2/resolve/main/tetu_code/generate.py
13.9 kB
| # Têtu Code 2 — inference code (tetu-AI). License: Apache 2.0, see README.md. | |
| from pathlib import Path | |
| import math | |
| import numpy as np | |
| import torch | |
| import torch.nn.functional as F | |
| from torch.nn.attention import SDPBackend, sdpa_kernel | |
| from tokenizers import Tokenizer | |
| from tetu_code.model import TetuCode, TetuCodeConfig, apply_rope, rms, rope_table, unpack_struct | |
| from tetu_code.structure import Tables, features | |
| ROOT = Path(__file__).resolve().parents[1] | |
| def attend(q, k, v, mask): | |
| B, H, T, hd = q.shape | |
| G = H // k.size(1) | |
| if T == 1: | |
| with sdpa_kernel(SDPBackend.MATH): | |
| y = F.scaled_dot_product_attention(q.reshape(B, H // G, G, hd), k, v, attn_mask=mask) | |
| return y.reshape(B, H, 1, hd) | |
| return F.scaled_dot_product_attention(q, k.repeat_interleave(G, 1), v.repeat_interleave(G, 1), attn_mask=mask) | |
| def load(path, device='cuda', dtype=torch.bfloat16, tokenizer=None): | |
| if Path(path).is_dir(): | |
| import json | |
| from safetensors.torch import load_file | |
| meta = json.loads((Path(path) / 'config.json').read_text(encoding='utf-8')) | |
| ck = dict(meta, model=load_file(str(Path(path) / 'model.safetensors'))) | |
| else: | |
| ck = torch.load(path, map_location='cpu', weights_only=False) | |
| cfg = TetuCodeConfig(**ck['config']) | |
| with torch.device(device): | |
| model = TetuCode(cfg) | |
| model.to(device) | |
| model.load_state_dict(ck['model']) | |
| if torch.device(device).type != 'cpu': | |
| with torch.device('cpu'): | |
| for name, theta in (('loc', cfg.rope_local), ('glo', cfg.rope_global)): | |
| cosine, sine = rope_table(cfg.max_seq_len, cfg.d_model // cfg.n_heads, theta) | |
| setattr(model, f'cos_{name}', cosine.to(device)) | |
| setattr(model, f'sin_{name}', sine.to(device)) | |
| for name, p in model.named_parameters(): | |
| if not name.startswith('doubt_head'): | |
| p.data = p.data.to(dtype) | |
| model.to(device).eval() | |
| model.meta = {k: ck.get(k) for k in ('step', 'tokens', 'val')} | |
| return (model, Tokenizer.from_file(str(tokenizer or ROOT / 'tokenizer.json'))) | |
| class Decoder: | |
| def __init__(self, model, tok, max_len=32768, ring=4096, chunk=2048, graph=True, compile=False, memo=None): | |
| self.memo = memo | |
| cfg = model.cfg | |
| self.model, self.tok, self.cfg = (model, tok, cfg) | |
| self.dev = next(model.parameters()).device | |
| dt = model.tok_emb.weight.dtype | |
| hd = cfg.d_model // cfg.n_heads | |
| self.max_len, self.ring, self.chunk, self.sw = (max_len, ring, chunk, cfg.window // 128) | |
| assert ring >= chunk + cfg.window, 'anneau trop petit pour le pré-remplissage par morceaux' | |
| self.S = [max_len if b.attn.is_global else ring for b in model.blocks] | |
| self.K = [torch.zeros(1, cfg.n_kv_heads, s, hd, device=self.dev, dtype=dt) for s in self.S] | |
| self.V = [torch.zeros_like(k) for k in self.K] | |
| self.ring_pos = torch.full((ring,), -(1 << 40), dtype=torch.long, device=self.dev) | |
| self.tok_ring = torch.full((ring,), cfg.vocab_size, dtype=torch.long, device=self.dev) | |
| self.eng = [i for i, b in enumerate(model.blocks) if b.engram is not None] | |
| self.order = max((max(o) for _, o in cfg.engram)) | |
| self.s_ring = {i: torch.zeros(1, ring, cfg.d_model, device=self.dev, dtype=dt) for i in self.eng} | |
| self.gidx = torch.arange(max_len, device=self.dev) | |
| self.tab = Tables(tok).arrays | |
| self.flag = self.tab[2] | |
| self.ids, self.st, self.reset_at = ([], [], 0) | |
| self.t_in = torch.zeros(1, 1, dtype=torch.long, device=self.dev) | |
| self.s_in = torch.zeros(1, 1, dtype=torch.int16, device=self.dev) | |
| self.p_in = torch.zeros(1, dtype=torch.long, device=self.dev) | |
| self.temp_t = torch.zeros((), device=self.dev) | |
| self.top_p_t = torch.full((), 0.95, device=self.dev) | |
| self.noise = torch.ones(50, device=self.dev) | |
| self.nxt = torch.zeros((), dtype=torch.long, device=self.dev) | |
| self._doubt = None | |
| self.graphs = {} | |
| if graph and self.dev.type == 'cuda': | |
| step = self._step | |
| if compile: | |
| import torch._dynamo as dynamo | |
| dynamo.config.cache_size_limit = max(dynamo.config.cache_size_limit, 16) | |
| step = torch.compile(self._step, dynamic=False) | |
| for n in sorted({min(4096 << i, max_len) for i in range(8) if (4096 << i) // 2 < max_len}): | |
| s = torch.cuda.Stream() | |
| s.wait_stream(torch.cuda.current_stream()) | |
| with torch.cuda.stream(s): | |
| for _ in range(3): | |
| self.noise.exponential_() | |
| step(n) | |
| torch.cuda.current_stream().wait_stream(s) | |
| g = torch.cuda.CUDAGraph() | |
| with torch.cuda.graph(g): | |
| self.noise.exponential_() | |
| self.graphs[n] = (g, step(n)) | |
| self.reset() | |
| def last_doubt(self): | |
| return float(self._doubt[0, -1]) if self._doubt is not None else 0.0 | |
| def reset(self): | |
| self.ring_pos.fill_(-(1 << 40)) | |
| self.tok_ring.fill_(self.cfg.vocab_size) | |
| for r in self.s_ring.values(): | |
| r.zero_() | |
| self.ids, self.st, self.reset_at = ([], [], 0) | |
| def _struct(self, new): | |
| s0 = self.reset_at | |
| st = features(np.asarray(self.ids[s0:] + list(new), dtype=np.int64), *self.tab) | |
| return [int(v) for v in st[len(self.ids) - s0:]] | |
| def _append(self, toks, st): | |
| for t in toks: | |
| if self.flag[t] == 1: | |
| self.reset_at = len(self.ids) | |
| self.ids.append(t) | |
| self.st += st | |
| def rollback(self, n): | |
| if n <= 0: | |
| return True | |
| if n > self.ring - self.cfg.window - 128: | |
| return False | |
| p = len(self.ids) - n | |
| self.ring_pos.masked_fill_(self.ring_pos >= p, -(1 << 40)) | |
| del self.ids[p:], self.st[p:] | |
| r = np.flatnonzero(self.flag[np.asarray(self.ids, dtype=np.int64)] == 1) if self.ids else [] | |
| self.reset_at = int(r[-1]) if len(r) else 0 | |
| return True | |
| def sync(self, ids): | |
| ids = list(ids) | |
| k = 0 | |
| n = min(len(ids), len(self.ids)) | |
| while k < n and ids[k] == self.ids[k]: | |
| k += 1 | |
| if k == len(ids) and k > 0: | |
| k -= 1 | |
| if not self.rollback(len(self.ids) - k): | |
| self.reset() | |
| k = 0 | |
| return self.feed(ids[k:]) | |
| def feed(self, new): | |
| assert len(self.ids) + len(new) < self.max_len, 'contexte trop long' | |
| out = None | |
| for a in range(0, len(new), self.chunk): | |
| part = new[a:a + self.chunk] | |
| if len(part) == 1 and self.graphs: | |
| out = self.step(part[0]) | |
| else: | |
| out = self._prefill(part) | |
| return out | |
| def step(self, t): | |
| if not self.graphs: | |
| return self._prefill([t]) | |
| st = self._struct([t])[0] | |
| n = len(self.ids) | |
| g, out = next((v for k, v in self.graphs.items() if k > n)) | |
| self.t_in.fill_(t) | |
| self.s_in.fill_(st) | |
| self.p_in.fill_(n) | |
| g.replay() | |
| self._append([t], [st]) | |
| return out | |
| def _rope(self, pos): | |
| m = self.model | |
| qs = (torch.log(pos.float() + 1) / math.log(self.cfg.log_base)).clamp_min(1.0) | |
| return ((m.cos_loc[pos], m.sin_loc[pos]), (m.cos_glo[pos], m.sin_glo[pos]), qs) | |
| def _embed(self, idx, st): | |
| m = self.model | |
| x = m.tok_emb(idx) | |
| for e, f in zip(m.struct_emb, unpack_struct(st)): | |
| x = x + e(f) | |
| return rms(x) | |
| def _layers(self, x, idx, pos, single): | |
| m, cfg = (self.model, self.cfg) | |
| T = idx.size(1) | |
| loc, glo, qs = self._rope(pos) | |
| L, skips = (cfg.n_layers, []) | |
| slot = pos % self.ring | |
| self.tok_ring.index_copy_(0, slot, idx[0]) | |
| self.ring_pos.index_copy_(0, slot, pos) | |
| for i, blk in enumerate(m.blocks): | |
| if i >= L - L // 2: | |
| x = x + m.skip_w[L - 1 - i] * skips.pop() | |
| if blk.engram is not None: | |
| eng, xe = (blk.engram, rms(x)) | |
| back = pos[None, :] - torch.arange(self.order - 1, -1, -1, device=self.dev)[:, None] | |
| ctx = torch.where(back >= 0, self.tok_ring[back % self.ring], torch.full_like(back, cfg.eos_id)) | |
| e = eng.table(eng.slots(ctx.t().contiguous())[:, -1]).reshape(1, T, -1).to(x.dtype) | |
| k, v = (eng.wk(e), eng.wv(e)) | |
| alpha = torch.sigmoid((rms(xe) * rms(k)).sum(-1, keepdim=True) / math.sqrt(cfg.d_model)) | |
| s = alpha * v | |
| self.s_ring[i][0].index_copy_(0, slot, s[0]) | |
| prev = pos[None, :] - torch.arange(3, 0, -1, device=self.dev)[:, None] | |
| ps = torch.where((prev >= 0)[..., None], self.s_ring[i][0][prev % self.ring], torch.zeros_like(s[0])) | |
| win = torch.cat([ps, s], dim=0).permute(1, 2, 0) | |
| c = (win * eng.conv.weight[:, 0, :]).sum(-1) | |
| x = x + s + F.silu(c[None]) | |
| att, xa = (blk.attn, rms(x)) | |
| q = att.wq(xa).view(1, T, att.H, att.hd).transpose(1, 2) | |
| k = att.wk(xa).view(1, T, att.Hk, att.hd).transpose(1, 2) | |
| v = att.wv(xa).view(1, T, att.Hk, att.hd).transpose(1, 2) | |
| j = m.ve_layer.get(i) | |
| if j is not None: | |
| v = v + m.ve_lambda[i] * m.value_embeds[j](idx).to(v.dtype).view(1, T, att.Hk, att.hd).transpose(1, 2) | |
| c_, s_ = glo if att.is_global else loc | |
| q, k = (apply_rope(rms(q), c_, s_), apply_rope(rms(k), c_, s_)) | |
| if att.is_global: | |
| q = q * qs.to(q.dtype)[:, None] | |
| self.K[i].index_copy_(2, pos, k) | |
| self.V[i].index_copy_(2, pos, v) | |
| if single: | |
| Kc, Vc = (self.K[i][:, :, :single], self.V[i][:, :, :single]) | |
| mask = (self.gidx[:single] <= pos[0])[None, None, None] | |
| else: | |
| end = int(pos[-1]) + 1 | |
| Kc, Vc = (self.K[i][:, :, :end], self.V[i][:, :, :end]) | |
| kp = self.gidx[:end] | |
| mask = (kp[None] <= pos[:, None])[None, None] | |
| else: | |
| self.K[i].index_copy_(2, slot, k) | |
| self.V[i].index_copy_(2, slot, v) | |
| if single: | |
| rp = self.ring_pos | |
| Kc, Vc = (self.K[i], self.V[i]) | |
| mask = ((rp <= pos[0]) & (rp // 128 > pos[0] // 128 - self.sw))[None, None, None] | |
| else: | |
| k0 = max(0, (int(pos[0]) // 128 - self.sw + 1) * 128) | |
| kp = torch.arange(k0, int(pos[-1]) + 1, device=self.dev) | |
| Kc, Vc = (self.K[i][:, :, kp % self.ring], self.V[i][:, :, kp % self.ring]) | |
| mask = ((kp[None] <= pos[:, None]) & (pos[:, None] // 128 - kp[None] // 128 < self.sw))[None, None] | |
| y = attend(q, Kc, Vc, mask) | |
| x = x + att.wo(y.transpose(1, 2).reshape(1, T, -1) * torch.sigmoid(att.wg(xa))) | |
| x = x + blk.mlp(rms(x)) | |
| if self.memo is not None and i == self.memo.L: | |
| if getattr(self.memo, 'exact', False): | |
| back = pos[None, :] - torch.arange(self.memo.K - 1, -1, -1, device=self.dev)[:, None] | |
| ctx = torch.where(back >= 0, self.tok_ring[back % self.ring], torch.full_like(back, -1)) | |
| x = self.memo(x, ctx.t()[None]) | |
| else: | |
| x = self.memo(x) | |
| if i < L // 2: | |
| skips.append(x) | |
| return rms(x) | |
| def _step(self, n): | |
| h = self._layers(self._embed(self.t_in, self.s_in), self.t_in, self.p_in, n) | |
| logits = self.model.logits(h) | |
| z = logits[0, -1].float() | |
| v, i = torch.topk(z / self.temp_t.clamp_min(1e-06), 50) | |
| p = torch.softmax(v, -1) | |
| p = torch.where(torch.cumsum(p, -1) - p < self.top_p_t, p, torch.zeros_like(p)) | |
| pick = i.gather(0, torch.argmax(p / self.noise).view(1))[0] | |
| self.nxt.copy_(torch.where(self.temp_t > 0, pick, z.argmax())) | |
| return (logits, torch.sigmoid(self.model.doubt_head(h, logits))) | |
| def _prefill(self, part): | |
| st = self._struct(part) | |
| idx = torch.tensor([part], device=self.dev) | |
| sx = torch.tensor([st], device=self.dev, dtype=torch.int16) | |
| pos = torch.arange(len(self.ids), len(self.ids) + len(part), device=self.dev) | |
| h = self._layers(self._embed(idx, sx), idx, pos, 0)[:, -1:] | |
| self._append(list(part), st) | |
| logits = self.model.logits(h) | |
| return (logits, torch.sigmoid(self.model.doubt_head(h, logits))) | |
| def generate(self, ids, max_new=512, temp=0.0, top_p=0.95, top_k=50, stop=(), seed=0): | |
| g = torch.Generator(device=self.dev).manual_seed(seed) | |
| self.temp_t.fill_(max(temp, 0.0)) | |
| self.top_p_t.fill_(top_p) | |
| in_graph = bool(self.graphs) and top_k == 50 | |
| logits, doubt = self.sync(ids) | |
| stop = set(stop) | |
| t = None | |
| for _ in range(max_new): | |
| if t is None: | |
| z = logits[0, -1].float() | |
| if temp <= 0: | |
| t = int(z.argmax()) | |
| else: | |
| v, i = torch.topk(z / temp, top_k) | |
| p = torch.softmax(v, -1) | |
| keep = torch.cumsum(p, -1) - p < top_p | |
| p = torch.where(keep, p, torch.zeros_like(p)) | |
| t = int(i[torch.multinomial(p / p.sum(), 1, generator=g)]) | |
| self._doubt = doubt | |
| yield t | |
| if t in stop: | |
| return | |
| logits, doubt = self.step(t) | |
| t = int(self.nxt) if in_graph else None | |