dilmac-en-tr-28m / modeling_dilmac.py
edededdy's picture
Upload folder using huggingface_hub
b72a467 verified
Raw History Blame Contribute Delete
15.1 kB
"""Dilmaç: a small English-to-Turkish encoder-decoder translation model.
from transformers import AutoModel, AutoTokenizer
tok = AutoTokenizer.from_pretrained(repo, trust_remote_code=True)
model = AutoModel.from_pretrained(repo, trust_remote_code=True)
model.translate(["How are you?"], tok)
Pre-norm RMSNorm Transformer, SwiGLU feed-forward, rotary positions in self-attention,
no biases, and one embedding matrix shared by source, target and output. Decoding
(greedy or beam search, with optional n-gram blocking and a source-relative length cap)
is implemented here with its own KV cache rather than through transformers' generate().
"""
import torch
import torch.nn as nn
import torch.nn.functional as F
from transformers import PreTrainedModel
from transformers.modeling_outputs import Seq2SeqLMOutput
from .configuration_dilmac import DilmacConfig
def rotate(x, pos):
"""Rotary position embedding. x: [B, H, T, hd]; pos: [B, T] position ids."""
hd = x.size(-1)
inv = 1.0 / (10000 ** (torch.arange(0, hd, 2, device=x.device).float() / hd))
freqs = pos.float()[..., None] * inv
cos, sin = freqs.cos().unsqueeze(1).to(x.dtype), freqs.sin().unsqueeze(1).to(x.dtype)
x1, x2 = x[..., ::2], x[..., 1::2]
return torch.stack((x1 * cos - x2 * sin, x1 * sin + x2 * cos), dim=-1).flatten(-2)
class Attention(nn.Module):
def __init__(self, cfg, rotary):
super().__init__()
self.h, self.hd = cfg.num_heads, cfg.d_model // cfg.num_heads
self.q = nn.Linear(cfg.d_model, cfg.d_model, bias=False)
self.kv = nn.Linear(cfg.d_model, 2 * cfg.d_model, bias=False)
self.o = nn.Linear(cfg.d_model, cfg.d_model, bias=False)
self.rotary = rotary
def split(self, x):
b, t, _ = x.shape
return x.view(b, t, self.h, self.hd).transpose(1, 2)
def forward(self, x, kv_in=None, pos=None, mask=None, causal=False, cache=None):
"""Self-attention when kv_in is None, else cross-attention over kv_in. cache (a
dict) keeps keys/values between decoding steps: appended for self-attention,
computed once for cross-attention."""
q = self.split(self.q(x))
if cache is not None and "k" in cache and kv_in is not None:
k, v = cache["k"], cache["v"]
else:
k, v = (self.split(t) for t in self.kv(x if kv_in is None else kv_in).chunk(2, dim=-1))
if self.rotary:
q, k = rotate(q, pos), rotate(k, pos)
if cache is not None:
if "k" in cache and kv_in is None:
k, v = torch.cat([cache["k"], k], 2), torch.cat([cache["v"], v], 2)
cache["k"], cache["v"] = k, v
y = F.scaled_dot_product_attention(q, k, v, attn_mask=mask, is_causal=causal and mask is None)
return self.o(y.transpose(1, 2).flatten(2))
class SwiGLU(nn.Module):
def __init__(self, cfg):
super().__init__()
self.wi = nn.Linear(cfg.d_model, 2 * cfg.ffn_dim, bias=False)
self.wo = nn.Linear(cfg.ffn_dim, cfg.d_model, bias=False)
def forward(self, x):
a, b = self.wi(x).chunk(2, dim=-1)
return self.wo(F.silu(a) * b)
class Block(nn.Module):
def __init__(self, cfg, cross=False):
super().__init__()
self.n1, self.attn = nn.RMSNorm(cfg.d_model), Attention(cfg, rotary=True)
self.cross = cross
if cross:
self.n2, self.xattn = nn.RMSNorm(cfg.d_model), Attention(cfg, rotary=False)
self.n3, self.ff = nn.RMSNorm(cfg.d_model), SwiGLU(cfg)
def forward(self, x, pos, mask=None, causal=False, mem=None, mem_mask=None, cache=None):
sc = xc = None
if cache is not None:
sc, xc = cache.setdefault("self", {}), cache.setdefault("cross", {})
x = x + self.attn(self.n1(x), pos=pos, mask=mask, causal=causal, cache=sc)
if self.cross:
x = x + self.xattn(self.n2(x), kv_in=mem, mask=mem_mask, cache=xc)
return x + self.ff(self.n3(x))
class DilmacForTranslation(PreTrainedModel):
config_class = DilmacConfig
base_model_prefix = "dilmac"
main_input_name = "input_ids"
_no_split_modules = ["Block"]
def __init__(self, config: DilmacConfig):
super().__init__(config)
self.emb = nn.Embedding(config.vocab_size, config.d_model)
self.enc = nn.ModuleList(Block(config) for _ in range(config.encoder_layers))
self.enc_norm = nn.RMSNorm(config.d_model)
self.dec = nn.ModuleList(Block(config, cross=True) for _ in range(config.decoder_layers))
self.dec_norm = nn.RMSNorm(config.d_model)
self.post_init()
def _init_weights(self, module):
if isinstance(module, (nn.Linear, nn.Embedding)):
nn.init.normal_(module.weight, std=0.02)
elif isinstance(module, nn.RMSNorm):
nn.init.ones_(module.weight)
def get_input_embeddings(self):
return self.emb
def set_input_embeddings(self, value):
self.emb = value
# --- model ---------------------------------------------------------------------------
def _pad_mask(self, ids):
return (ids != self.config.pad_token_id)[:, None, None, :]
def _positions(self, ids):
real = (ids != self.config.pad_token_id).long()
return (real.cumsum(1) - 1).clamp(min=0)
def _logits(self, h):
return self.dec_norm(h) @ self.emb.weight.T
def encode(self, input_ids):
x = self.emb(input_ids)
pos, mask = self._positions(input_ids), self._pad_mask(input_ids)
for blk in self.enc:
x = blk(x, pos, mask=mask)
return self.enc_norm(x)
def forward(self, input_ids, decoder_input_ids=None, labels=None, attention_mask=None, **kwargs):
"""Teacher-forced forward pass. input_ids: [B, S] source ids, right-padded with
pad_token_id (attention_mask is accepted but padding is read from the ids).
Pass labels ([B, T] target ids followed by eos, padded with pad_token_id or -100)
to get the loss; decoder_input_ids default to bos + labels[:, :-1]."""
cfg = self.config
if labels is not None:
labels = labels.masked_fill(labels == -100, cfg.pad_token_id)
if decoder_input_ids is None:
bos = labels.new_full((labels.size(0), 1), cfg.bos_token_id)
decoder_input_ids = torch.cat([bos, labels[:, :-1]], 1)
mem, mem_mask = self.encode(input_ids), self._pad_mask(input_ids)
x, pos = self.emb(decoder_input_ids), self._positions(decoder_input_ids)
for blk in self.dec:
x = blk(x, pos, causal=True, mem=mem, mem_mask=mem_mask)
logits = self._logits(x)
loss = None
if labels is not None:
loss = F.cross_entropy(logits.float().flatten(0, 1), labels.flatten(), ignore_index=cfg.pad_token_id)
return Seq2SeqLMOutput(loss=loss, logits=logits, encoder_last_hidden_state=mem)
# --- translation -------------------------------------------------------------------
@torch.no_grad()
def translate(self, texts, tokenizer, num_beams=None, no_repeat_ngram_size=None,
max_length_ratio=None, length_penalty=None, batch_size=32):
"""Translate English text (a string or a list of strings) to Turkish.
num_beams: 1 = greedy, >1 = beam search (default from the config: 4).
no_repeat_ngram_size: never repeat an n-gram of this size in one output (0 = off).
max_length_ratio: stop after max_length_ratio * source tokens + 5 tokens.
length_penalty: beam scores are log p / ((5 + length) / 6) ** length_penalty.
Sources longer than config.max_source_tokens tokens are truncated.
"""
cfg = self.config
single = isinstance(texts, str)
texts = [texts] if single else list(texts)
opts = dict(
beam=cfg.beam_size if num_beams is None else num_beams,
no_repeat=cfg.no_repeat_ngram if no_repeat_ngram_size is None else no_repeat_ngram_size,
max_ratio=cfg.max_length_ratio if max_length_ratio is None else max_length_ratio,
len_pen=cfg.len_penalty if length_penalty is None else length_penalty,
)
self.eval()
enc = [ids[:cfg.max_source_tokens] or [cfg.eos_token_id]
for ids in tokenizer(texts, add_special_tokens=False)["input_ids"]]
order = sorted(range(len(enc)), key=lambda i: len(enc[i]))
out = [None] * len(enc)
for k in range(0, len(order), batch_size):
idx = order[k:k + batch_size]
width = max(len(enc[i]) for i in idx)
src = torch.full((len(idx), width), cfg.pad_token_id, dtype=torch.long)
for r, i in enumerate(idx):
src[r, :len(enc[i])] = torch.tensor(enc[i])
for i, ids in zip(idx, self._decode(src.to(self.device), **opts)):
out[i] = tokenizer.decode(ids, skip_special_tokens=True)
return out[0] if single else out
def _decode(self, src, beam, no_repeat, max_ratio, len_pen):
if beam > 1:
return self._beam(src, beam, len_pen, no_repeat, max_ratio)
cfg, b = self.config, src.size(0)
lens = (src != cfg.pad_token_id).sum(1)
limit = (lens.float() * max_ratio).long() + 5 if max_ratio else lens * 2 + 10
mem, mem_mask = self.encode(src), self._pad_mask(src)
caches = [{} for _ in self.dec]
cur = torch.full((b, 1), cfg.bos_token_id, device=src.device)
gen = [[] for _ in range(b)]
done = torch.zeros(b, dtype=torch.bool, device=src.device)
for step in range(int(limit.max()) + 1):
x = self.emb(cur)
pos = torch.full((b, 1), step, dtype=torch.long, device=src.device)
for blk, c in zip(self.dec, caches):
x = blk(x, pos, mem=mem, mem_mask=mem_mask, cache=c)
logits = self._logits(x)[:, 0].float()
if no_repeat:
_block_ngrams(logits, gen, no_repeat)
_force_eos(logits, step >= limit, cfg.eos_token_id)
cur = logits.argmax(-1, keepdim=True)
for g, t, d in zip(gen, cur[:, 0].tolist(), done.tolist()):
if not d:
g.append(t)
done |= cur[:, 0] == cfg.eos_token_id
if done.all():
break
return [g[:g.index(cfg.eos_token_id)] if cfg.eos_token_id in g else g for g in gen]
def _beam(self, src, k, len_pen, no_repeat, max_ratio):
cfg, b, dev = self.config, src.size(0), src.device
eos, pad = cfg.eos_token_id, cfg.pad_token_id
lens = (src != pad).sum(1)
limit = ((lens.float() * max_ratio).long() + 5 if max_ratio else lens * 2 + 10).tolist()
mem = self.encode(src).repeat_interleave(k, 0)
mem_mask = self._pad_mask(src).repeat_interleave(k, 0)
caches = [{} for _ in self.dec]
cur = torch.full((b * k, 1), cfg.bos_token_id, device=dev)
scores = torch.zeros(b, k, device=dev)
scores[:, 1:] = float("-inf") # all beams start identical; keep only the first
seqs = [[] for _ in range(b * k)]
finished = [[] for _ in range(b)] # (normalised score, ids)
open_ = [True] * b
def norm(score, n):
return score / ((5 + n) / 6) ** len_pen
for step in range(max(limit) + 1):
pos = torch.full((b * k, 1), step, dtype=torch.long, device=dev)
x = self.emb(cur)
for blk, c in zip(self.dec, caches):
x = blk(x, pos, mem=mem, mem_mask=mem_mask, cache=c)
logp = torch.log_softmax(self._logits(x)[:, 0].float(), -1)
if no_repeat:
_block_ngrams(logp, seqs, no_repeat)
_force_eos(logp, torch.tensor([step >= limit[i // k] for i in range(b * k)], device=dev), eos)
cand = (scores.view(-1, 1) + logp).view(b, -1)
top_s, top_i = cand.topk(2 * k, dim=1)
top_s, top_i = top_s.tolist(), top_i.tolist()
vocab = logp.size(1)
new_scores, origin, tokens = [], [], []
for i in range(b):
keep = []
if open_[i]:
for s, j in zip(top_s[i], top_i[i]):
if s == float("-inf"):
break
beam_i, tok = divmod(j, vocab)
if tok == eos:
if len(finished[i]) < k:
finished[i].append((norm(s, step + 1), seqs[i * k + beam_i]))
else:
keep.append((s, i * k + beam_i, tok))
if len(keep) == k:
break
# Stop once k hypotheses have finished or no open beam can beat the best.
if finished[i] and (len(finished[i]) >= k or not keep or (
len_pen >= 0 and norm(keep[0][0], limit[i] + 1) <= max(f[0] for f in finished[i]))):
open_[i] = False
keep = []
while len(keep) < k: # pad closed or short groups with dead beams
keep.append((float("-inf"), i * k, pad))
for s, row, tok in keep:
new_scores.append(s)
origin.append(row)
tokens.append(tok)
if not any(open_):
break
idx = torch.tensor(origin, device=dev)
for c in caches:
c["self"]["k"] = c["self"]["k"].index_select(0, idx)
c["self"]["v"] = c["self"]["v"].index_select(0, idx)
seqs = [seqs[o] + [t] for o, t in zip(origin, tokens)]
scores = torch.tensor(new_scores, device=dev).view(b, k)
cur = torch.tensor(tokens, device=dev).view(-1, 1)
out = []
for i in range(b):
if not finished[i]: # ran out of steps: take the best open beam
row = i * k + int(scores[i].argmax())
finished[i].append((norm(float(scores[i].max()), len(seqs[row])), seqs[row]))
out.append(max(finished[i], key=lambda f: f[0])[1])
return out
def _block_ngrams(logits, gen, n):
"""Set to -inf every token that would repeat an n-gram already in that row of gen."""
rows, cols = [], []
for r, g in enumerate(gen):
if len(g) < n:
continue
prefix = g[len(g) - n + 1:] if n > 1 else []
for i in range(len(g) - n + 1):
if g[i:i + n - 1] == prefix:
rows.append(r)
cols.append(g[i + n - 1])
if rows:
logits[rows, cols] = float("-inf")
def _force_eos(logits, rows, eos):
"""Make eos the only choice in the rows where `rows` (a bool tensor) is True."""
if rows.any():
e = logits[rows, eos].clone()
logits[rows] = float("-inf")
logits[rows, eos] = torch.where(torch.isinf(e), torch.zeros_like(e), e)