Tetu-Code-2 / tetu_code /generate.py
tetu-AI's picture
Initial release
2309cb1 verified
Raw History Blame Contribute Delete
13.9 kB
# Têtu Code 2 — inference code (tetu-AI). License: Apache 2.0, see README.md.
from pathlib import Path
import math
import numpy as np
import torch
import torch.nn.functional as F
from torch.nn.attention import SDPBackend, sdpa_kernel
from tokenizers import Tokenizer
from tetu_code.model import TetuCode, TetuCodeConfig, apply_rope, rms, rope_table, unpack_struct
from tetu_code.structure import Tables, features
ROOT = Path(__file__).resolve().parents[1]
def attend(q, k, v, mask):
B, H, T, hd = q.shape
G = H // k.size(1)
if T == 1:
with sdpa_kernel(SDPBackend.MATH):
y = F.scaled_dot_product_attention(q.reshape(B, H // G, G, hd), k, v, attn_mask=mask)
return y.reshape(B, H, 1, hd)
return F.scaled_dot_product_attention(q, k.repeat_interleave(G, 1), v.repeat_interleave(G, 1), attn_mask=mask)
def load(path, device='cuda', dtype=torch.bfloat16, tokenizer=None):
if Path(path).is_dir():
import json
from safetensors.torch import load_file
meta = json.loads((Path(path) / 'config.json').read_text(encoding='utf-8'))
ck = dict(meta, model=load_file(str(Path(path) / 'model.safetensors')))
else:
ck = torch.load(path, map_location='cpu', weights_only=False)
cfg = TetuCodeConfig(**ck['config'])
with torch.device(device):
model = TetuCode(cfg)
model.to(device)
model.load_state_dict(ck['model'])
if torch.device(device).type != 'cpu':
with torch.device('cpu'):
for name, theta in (('loc', cfg.rope_local), ('glo', cfg.rope_global)):
cosine, sine = rope_table(cfg.max_seq_len, cfg.d_model // cfg.n_heads, theta)
setattr(model, f'cos_{name}', cosine.to(device))
setattr(model, f'sin_{name}', sine.to(device))
for name, p in model.named_parameters():
if not name.startswith('doubt_head'):
p.data = p.data.to(dtype)
model.to(device).eval()
model.meta = {k: ck.get(k) for k in ('step', 'tokens', 'val')}
return (model, Tokenizer.from_file(str(tokenizer or ROOT / 'tokenizer.json')))
class Decoder:
def __init__(self, model, tok, max_len=32768, ring=4096, chunk=2048, graph=True, compile=False, memo=None):
self.memo = memo
cfg = model.cfg
self.model, self.tok, self.cfg = (model, tok, cfg)
self.dev = next(model.parameters()).device
dt = model.tok_emb.weight.dtype
hd = cfg.d_model // cfg.n_heads
self.max_len, self.ring, self.chunk, self.sw = (max_len, ring, chunk, cfg.window // 128)
assert ring >= chunk + cfg.window, 'anneau trop petit pour le pré-remplissage par morceaux'
self.S = [max_len if b.attn.is_global else ring for b in model.blocks]
self.K = [torch.zeros(1, cfg.n_kv_heads, s, hd, device=self.dev, dtype=dt) for s in self.S]
self.V = [torch.zeros_like(k) for k in self.K]
self.ring_pos = torch.full((ring,), -(1 << 40), dtype=torch.long, device=self.dev)
self.tok_ring = torch.full((ring,), cfg.vocab_size, dtype=torch.long, device=self.dev)
self.eng = [i for i, b in enumerate(model.blocks) if b.engram is not None]
self.order = max((max(o) for _, o in cfg.engram))
self.s_ring = {i: torch.zeros(1, ring, cfg.d_model, device=self.dev, dtype=dt) for i in self.eng}
self.gidx = torch.arange(max_len, device=self.dev)
self.tab = Tables(tok).arrays
self.flag = self.tab[2]
self.ids, self.st, self.reset_at = ([], [], 0)
self.t_in = torch.zeros(1, 1, dtype=torch.long, device=self.dev)
self.s_in = torch.zeros(1, 1, dtype=torch.int16, device=self.dev)
self.p_in = torch.zeros(1, dtype=torch.long, device=self.dev)
self.temp_t = torch.zeros((), device=self.dev)
self.top_p_t = torch.full((), 0.95, device=self.dev)
self.noise = torch.ones(50, device=self.dev)
self.nxt = torch.zeros((), dtype=torch.long, device=self.dev)
self._doubt = None
self.graphs = {}
if graph and self.dev.type == 'cuda':
step = self._step
if compile:
import torch._dynamo as dynamo
dynamo.config.cache_size_limit = max(dynamo.config.cache_size_limit, 16)
step = torch.compile(self._step, dynamic=False)
for n in sorted({min(4096 << i, max_len) for i in range(8) if (4096 << i) // 2 < max_len}):
s = torch.cuda.Stream()
s.wait_stream(torch.cuda.current_stream())
with torch.cuda.stream(s):
for _ in range(3):
self.noise.exponential_()
step(n)
torch.cuda.current_stream().wait_stream(s)
g = torch.cuda.CUDAGraph()
with torch.cuda.graph(g):
self.noise.exponential_()
self.graphs[n] = (g, step(n))
self.reset()
@property
def last_doubt(self):
return float(self._doubt[0, -1]) if self._doubt is not None else 0.0
def reset(self):
self.ring_pos.fill_(-(1 << 40))
self.tok_ring.fill_(self.cfg.vocab_size)
for r in self.s_ring.values():
r.zero_()
self.ids, self.st, self.reset_at = ([], [], 0)
def _struct(self, new):
s0 = self.reset_at
st = features(np.asarray(self.ids[s0:] + list(new), dtype=np.int64), *self.tab)
return [int(v) for v in st[len(self.ids) - s0:]]
def _append(self, toks, st):
for t in toks:
if self.flag[t] == 1:
self.reset_at = len(self.ids)
self.ids.append(t)
self.st += st
def rollback(self, n):
if n <= 0:
return True
if n > self.ring - self.cfg.window - 128:
return False
p = len(self.ids) - n
self.ring_pos.masked_fill_(self.ring_pos >= p, -(1 << 40))
del self.ids[p:], self.st[p:]
r = np.flatnonzero(self.flag[np.asarray(self.ids, dtype=np.int64)] == 1) if self.ids else []
self.reset_at = int(r[-1]) if len(r) else 0
return True
def sync(self, ids):
ids = list(ids)
k = 0
n = min(len(ids), len(self.ids))
while k < n and ids[k] == self.ids[k]:
k += 1
if k == len(ids) and k > 0:
k -= 1
if not self.rollback(len(self.ids) - k):
self.reset()
k = 0
return self.feed(ids[k:])
@torch.no_grad()
def feed(self, new):
assert len(self.ids) + len(new) < self.max_len, 'contexte trop long'
out = None
for a in range(0, len(new), self.chunk):
part = new[a:a + self.chunk]
if len(part) == 1 and self.graphs:
out = self.step(part[0])
else:
out = self._prefill(part)
return out
def step(self, t):
if not self.graphs:
return self._prefill([t])
st = self._struct([t])[0]
n = len(self.ids)
g, out = next((v for k, v in self.graphs.items() if k > n))
self.t_in.fill_(t)
self.s_in.fill_(st)
self.p_in.fill_(n)
g.replay()
self._append([t], [st])
return out
def _rope(self, pos):
m = self.model
qs = (torch.log(pos.float() + 1) / math.log(self.cfg.log_base)).clamp_min(1.0)
return ((m.cos_loc[pos], m.sin_loc[pos]), (m.cos_glo[pos], m.sin_glo[pos]), qs)
def _embed(self, idx, st):
m = self.model
x = m.tok_emb(idx)
for e, f in zip(m.struct_emb, unpack_struct(st)):
x = x + e(f)
return rms(x)
def _layers(self, x, idx, pos, single):
m, cfg = (self.model, self.cfg)
T = idx.size(1)
loc, glo, qs = self._rope(pos)
L, skips = (cfg.n_layers, [])
slot = pos % self.ring
self.tok_ring.index_copy_(0, slot, idx[0])
self.ring_pos.index_copy_(0, slot, pos)
for i, blk in enumerate(m.blocks):
if i >= L - L // 2:
x = x + m.skip_w[L - 1 - i] * skips.pop()
if blk.engram is not None:
eng, xe = (blk.engram, rms(x))
back = pos[None, :] - torch.arange(self.order - 1, -1, -1, device=self.dev)[:, None]
ctx = torch.where(back >= 0, self.tok_ring[back % self.ring], torch.full_like(back, cfg.eos_id))
e = eng.table(eng.slots(ctx.t().contiguous())[:, -1]).reshape(1, T, -1).to(x.dtype)
k, v = (eng.wk(e), eng.wv(e))
alpha = torch.sigmoid((rms(xe) * rms(k)).sum(-1, keepdim=True) / math.sqrt(cfg.d_model))
s = alpha * v
self.s_ring[i][0].index_copy_(0, slot, s[0])
prev = pos[None, :] - torch.arange(3, 0, -1, device=self.dev)[:, None]
ps = torch.where((prev >= 0)[..., None], self.s_ring[i][0][prev % self.ring], torch.zeros_like(s[0]))
win = torch.cat([ps, s], dim=0).permute(1, 2, 0)
c = (win * eng.conv.weight[:, 0, :]).sum(-1)
x = x + s + F.silu(c[None])
att, xa = (blk.attn, rms(x))
q = att.wq(xa).view(1, T, att.H, att.hd).transpose(1, 2)
k = att.wk(xa).view(1, T, att.Hk, att.hd).transpose(1, 2)
v = att.wv(xa).view(1, T, att.Hk, att.hd).transpose(1, 2)
j = m.ve_layer.get(i)
if j is not None:
v = v + m.ve_lambda[i] * m.value_embeds[j](idx).to(v.dtype).view(1, T, att.Hk, att.hd).transpose(1, 2)
c_, s_ = glo if att.is_global else loc
q, k = (apply_rope(rms(q), c_, s_), apply_rope(rms(k), c_, s_))
if att.is_global:
q = q * qs.to(q.dtype)[:, None]
self.K[i].index_copy_(2, pos, k)
self.V[i].index_copy_(2, pos, v)
if single:
Kc, Vc = (self.K[i][:, :, :single], self.V[i][:, :, :single])
mask = (self.gidx[:single] <= pos[0])[None, None, None]
else:
end = int(pos[-1]) + 1
Kc, Vc = (self.K[i][:, :, :end], self.V[i][:, :, :end])
kp = self.gidx[:end]
mask = (kp[None] <= pos[:, None])[None, None]
else:
self.K[i].index_copy_(2, slot, k)
self.V[i].index_copy_(2, slot, v)
if single:
rp = self.ring_pos
Kc, Vc = (self.K[i], self.V[i])
mask = ((rp <= pos[0]) & (rp // 128 > pos[0] // 128 - self.sw))[None, None, None]
else:
k0 = max(0, (int(pos[0]) // 128 - self.sw + 1) * 128)
kp = torch.arange(k0, int(pos[-1]) + 1, device=self.dev)
Kc, Vc = (self.K[i][:, :, kp % self.ring], self.V[i][:, :, kp % self.ring])
mask = ((kp[None] <= pos[:, None]) & (pos[:, None] // 128 - kp[None] // 128 < self.sw))[None, None]
y = attend(q, Kc, Vc, mask)
x = x + att.wo(y.transpose(1, 2).reshape(1, T, -1) * torch.sigmoid(att.wg(xa)))
x = x + blk.mlp(rms(x))
if self.memo is not None and i == self.memo.L:
if getattr(self.memo, 'exact', False):
back = pos[None, :] - torch.arange(self.memo.K - 1, -1, -1, device=self.dev)[:, None]
ctx = torch.where(back >= 0, self.tok_ring[back % self.ring], torch.full_like(back, -1))
x = self.memo(x, ctx.t()[None])
else:
x = self.memo(x)
if i < L // 2:
skips.append(x)
return rms(x)
@torch.no_grad()
def _step(self, n):
h = self._layers(self._embed(self.t_in, self.s_in), self.t_in, self.p_in, n)
logits = self.model.logits(h)
z = logits[0, -1].float()
v, i = torch.topk(z / self.temp_t.clamp_min(1e-06), 50)
p = torch.softmax(v, -1)
p = torch.where(torch.cumsum(p, -1) - p < self.top_p_t, p, torch.zeros_like(p))
pick = i.gather(0, torch.argmax(p / self.noise).view(1))[0]
self.nxt.copy_(torch.where(self.temp_t > 0, pick, z.argmax()))
return (logits, torch.sigmoid(self.model.doubt_head(h, logits)))
@torch.no_grad()
def _prefill(self, part):
st = self._struct(part)
idx = torch.tensor([part], device=self.dev)
sx = torch.tensor([st], device=self.dev, dtype=torch.int16)
pos = torch.arange(len(self.ids), len(self.ids) + len(part), device=self.dev)
h = self._layers(self._embed(idx, sx), idx, pos, 0)[:, -1:]
self._append(list(part), st)
logits = self.model.logits(h)
return (logits, torch.sigmoid(self.model.doubt_head(h, logits)))
@torch.no_grad()
def generate(self, ids, max_new=512, temp=0.0, top_p=0.95, top_k=50, stop=(), seed=0):
g = torch.Generator(device=self.dev).manual_seed(seed)
self.temp_t.fill_(max(temp, 0.0))
self.top_p_t.fill_(top_p)
in_graph = bool(self.graphs) and top_k == 50
logits, doubt = self.sync(ids)
stop = set(stop)
t = None
for _ in range(max_new):
if t is None:
z = logits[0, -1].float()
if temp <= 0:
t = int(z.argmax())
else:
v, i = torch.topk(z / temp, top_k)
p = torch.softmax(v, -1)
keep = torch.cumsum(p, -1) - p < top_p
p = torch.where(keep, p, torch.zeros_like(p))
t = int(i[torch.multinomial(p / p.sum(), 1, generator=g)])
self._doubt = doubt
yield t
if t in stop:
return
logits, doubt = self.step(t)
t = int(self.nxt) if in_graph else None