"""DilmaƧ: a small English-to-Turkish encoder-decoder translation model. from transformers import AutoModel, AutoTokenizer tok = AutoTokenizer.from_pretrained(repo, trust_remote_code=True) model = AutoModel.from_pretrained(repo, trust_remote_code=True) model.translate(["How are you?"], tok) Pre-norm RMSNorm Transformer, SwiGLU feed-forward, rotary positions in self-attention, no biases, and one embedding matrix shared by source, target and output. Decoding (greedy or beam search, with optional n-gram blocking and a source-relative length cap) is implemented here with its own KV cache rather than through transformers' generate(). """ import torch import torch.nn as nn import torch.nn.functional as F from transformers import PreTrainedModel from transformers.modeling_outputs import Seq2SeqLMOutput from .configuration_dilmac import DilmacConfig def rotate(x, pos): """Rotary position embedding. x: [B, H, T, hd]; pos: [B, T] position ids.""" hd = x.size(-1) inv = 1.0 / (10000 ** (torch.arange(0, hd, 2, device=x.device).float() / hd)) freqs = pos.float()[..., None] * inv cos, sin = freqs.cos().unsqueeze(1).to(x.dtype), freqs.sin().unsqueeze(1).to(x.dtype) x1, x2 = x[..., ::2], x[..., 1::2] return torch.stack((x1 * cos - x2 * sin, x1 * sin + x2 * cos), dim=-1).flatten(-2) class Attention(nn.Module): def __init__(self, cfg, rotary): super().__init__() self.h, self.hd = cfg.num_heads, cfg.d_model // cfg.num_heads self.q = nn.Linear(cfg.d_model, cfg.d_model, bias=False) self.kv = nn.Linear(cfg.d_model, 2 * cfg.d_model, bias=False) self.o = nn.Linear(cfg.d_model, cfg.d_model, bias=False) self.rotary = rotary def split(self, x): b, t, _ = x.shape return x.view(b, t, self.h, self.hd).transpose(1, 2) def forward(self, x, kv_in=None, pos=None, mask=None, causal=False, cache=None): """Self-attention when kv_in is None, else cross-attention over kv_in. cache (a dict) keeps keys/values between decoding steps: appended for self-attention, computed once for cross-attention.""" q = self.split(self.q(x)) if cache is not None and "k" in cache and kv_in is not None: k, v = cache["k"], cache["v"] else: k, v = (self.split(t) for t in self.kv(x if kv_in is None else kv_in).chunk(2, dim=-1)) if self.rotary: q, k = rotate(q, pos), rotate(k, pos) if cache is not None: if "k" in cache and kv_in is None: k, v = torch.cat([cache["k"], k], 2), torch.cat([cache["v"], v], 2) cache["k"], cache["v"] = k, v y = F.scaled_dot_product_attention(q, k, v, attn_mask=mask, is_causal=causal and mask is None) return self.o(y.transpose(1, 2).flatten(2)) class SwiGLU(nn.Module): def __init__(self, cfg): super().__init__() self.wi = nn.Linear(cfg.d_model, 2 * cfg.ffn_dim, bias=False) self.wo = nn.Linear(cfg.ffn_dim, cfg.d_model, bias=False) def forward(self, x): a, b = self.wi(x).chunk(2, dim=-1) return self.wo(F.silu(a) * b) class Block(nn.Module): def __init__(self, cfg, cross=False): super().__init__() self.n1, self.attn = nn.RMSNorm(cfg.d_model), Attention(cfg, rotary=True) self.cross = cross if cross: self.n2, self.xattn = nn.RMSNorm(cfg.d_model), Attention(cfg, rotary=False) self.n3, self.ff = nn.RMSNorm(cfg.d_model), SwiGLU(cfg) def forward(self, x, pos, mask=None, causal=False, mem=None, mem_mask=None, cache=None): sc = xc = None if cache is not None: sc, xc = cache.setdefault("self", {}), cache.setdefault("cross", {}) x = x + self.attn(self.n1(x), pos=pos, mask=mask, causal=causal, cache=sc) if self.cross: x = x + self.xattn(self.n2(x), kv_in=mem, mask=mem_mask, cache=xc) return x + self.ff(self.n3(x)) class DilmacForTranslation(PreTrainedModel): config_class = DilmacConfig base_model_prefix = "dilmac" main_input_name = "input_ids" _no_split_modules = ["Block"] def __init__(self, config: DilmacConfig): super().__init__(config) self.emb = nn.Embedding(config.vocab_size, config.d_model) self.enc = nn.ModuleList(Block(config) for _ in range(config.encoder_layers)) self.enc_norm = nn.RMSNorm(config.d_model) self.dec = nn.ModuleList(Block(config, cross=True) for _ in range(config.decoder_layers)) self.dec_norm = nn.RMSNorm(config.d_model) self.post_init() def _init_weights(self, module): if isinstance(module, (nn.Linear, nn.Embedding)): nn.init.normal_(module.weight, std=0.02) elif isinstance(module, nn.RMSNorm): nn.init.ones_(module.weight) def get_input_embeddings(self): return self.emb def set_input_embeddings(self, value): self.emb = value # --- model --------------------------------------------------------------------------- def _pad_mask(self, ids): return (ids != self.config.pad_token_id)[:, None, None, :] def _positions(self, ids): real = (ids != self.config.pad_token_id).long() return (real.cumsum(1) - 1).clamp(min=0) def _logits(self, h): return self.dec_norm(h) @ self.emb.weight.T def encode(self, input_ids): x = self.emb(input_ids) pos, mask = self._positions(input_ids), self._pad_mask(input_ids) for blk in self.enc: x = blk(x, pos, mask=mask) return self.enc_norm(x) def forward(self, input_ids, decoder_input_ids=None, labels=None, attention_mask=None, **kwargs): """Teacher-forced forward pass. input_ids: [B, S] source ids, right-padded with pad_token_id (attention_mask is accepted but padding is read from the ids). Pass labels ([B, T] target ids followed by eos, padded with pad_token_id or -100) to get the loss; decoder_input_ids default to bos + labels[:, :-1].""" cfg = self.config if labels is not None: labels = labels.masked_fill(labels == -100, cfg.pad_token_id) if decoder_input_ids is None: bos = labels.new_full((labels.size(0), 1), cfg.bos_token_id) decoder_input_ids = torch.cat([bos, labels[:, :-1]], 1) mem, mem_mask = self.encode(input_ids), self._pad_mask(input_ids) x, pos = self.emb(decoder_input_ids), self._positions(decoder_input_ids) for blk in self.dec: x = blk(x, pos, causal=True, mem=mem, mem_mask=mem_mask) logits = self._logits(x) loss = None if labels is not None: loss = F.cross_entropy(logits.float().flatten(0, 1), labels.flatten(), ignore_index=cfg.pad_token_id) return Seq2SeqLMOutput(loss=loss, logits=logits, encoder_last_hidden_state=mem) # --- translation ------------------------------------------------------------------- @torch.no_grad() def translate(self, texts, tokenizer, num_beams=None, no_repeat_ngram_size=None, max_length_ratio=None, length_penalty=None, batch_size=32): """Translate English text (a string or a list of strings) to Turkish. num_beams: 1 = greedy, >1 = beam search (default from the config: 4). no_repeat_ngram_size: never repeat an n-gram of this size in one output (0 = off). max_length_ratio: stop after max_length_ratio * source tokens + 5 tokens. length_penalty: beam scores are log p / ((5 + length) / 6) ** length_penalty. Sources longer than config.max_source_tokens tokens are truncated. """ cfg = self.config single = isinstance(texts, str) texts = [texts] if single else list(texts) opts = dict( beam=cfg.beam_size if num_beams is None else num_beams, no_repeat=cfg.no_repeat_ngram if no_repeat_ngram_size is None else no_repeat_ngram_size, max_ratio=cfg.max_length_ratio if max_length_ratio is None else max_length_ratio, len_pen=cfg.len_penalty if length_penalty is None else length_penalty, ) self.eval() enc = [ids[:cfg.max_source_tokens] or [cfg.eos_token_id] for ids in tokenizer(texts, add_special_tokens=False)["input_ids"]] order = sorted(range(len(enc)), key=lambda i: len(enc[i])) out = [None] * len(enc) for k in range(0, len(order), batch_size): idx = order[k:k + batch_size] width = max(len(enc[i]) for i in idx) src = torch.full((len(idx), width), cfg.pad_token_id, dtype=torch.long) for r, i in enumerate(idx): src[r, :len(enc[i])] = torch.tensor(enc[i]) for i, ids in zip(idx, self._decode(src.to(self.device), **opts)): out[i] = tokenizer.decode(ids, skip_special_tokens=True) return out[0] if single else out def _decode(self, src, beam, no_repeat, max_ratio, len_pen): if beam > 1: return self._beam(src, beam, len_pen, no_repeat, max_ratio) cfg, b = self.config, src.size(0) lens = (src != cfg.pad_token_id).sum(1) limit = (lens.float() * max_ratio).long() + 5 if max_ratio else lens * 2 + 10 mem, mem_mask = self.encode(src), self._pad_mask(src) caches = [{} for _ in self.dec] cur = torch.full((b, 1), cfg.bos_token_id, device=src.device) gen = [[] for _ in range(b)] done = torch.zeros(b, dtype=torch.bool, device=src.device) for step in range(int(limit.max()) + 1): x = self.emb(cur) pos = torch.full((b, 1), step, dtype=torch.long, device=src.device) for blk, c in zip(self.dec, caches): x = blk(x, pos, mem=mem, mem_mask=mem_mask, cache=c) logits = self._logits(x)[:, 0].float() if no_repeat: _block_ngrams(logits, gen, no_repeat) _force_eos(logits, step >= limit, cfg.eos_token_id) cur = logits.argmax(-1, keepdim=True) for g, t, d in zip(gen, cur[:, 0].tolist(), done.tolist()): if not d: g.append(t) done |= cur[:, 0] == cfg.eos_token_id if done.all(): break return [g[:g.index(cfg.eos_token_id)] if cfg.eos_token_id in g else g for g in gen] def _beam(self, src, k, len_pen, no_repeat, max_ratio): cfg, b, dev = self.config, src.size(0), src.device eos, pad = cfg.eos_token_id, cfg.pad_token_id lens = (src != pad).sum(1) limit = ((lens.float() * max_ratio).long() + 5 if max_ratio else lens * 2 + 10).tolist() mem = self.encode(src).repeat_interleave(k, 0) mem_mask = self._pad_mask(src).repeat_interleave(k, 0) caches = [{} for _ in self.dec] cur = torch.full((b * k, 1), cfg.bos_token_id, device=dev) scores = torch.zeros(b, k, device=dev) scores[:, 1:] = float("-inf") # all beams start identical; keep only the first seqs = [[] for _ in range(b * k)] finished = [[] for _ in range(b)] # (normalised score, ids) open_ = [True] * b def norm(score, n): return score / ((5 + n) / 6) ** len_pen for step in range(max(limit) + 1): pos = torch.full((b * k, 1), step, dtype=torch.long, device=dev) x = self.emb(cur) for blk, c in zip(self.dec, caches): x = blk(x, pos, mem=mem, mem_mask=mem_mask, cache=c) logp = torch.log_softmax(self._logits(x)[:, 0].float(), -1) if no_repeat: _block_ngrams(logp, seqs, no_repeat) _force_eos(logp, torch.tensor([step >= limit[i // k] for i in range(b * k)], device=dev), eos) cand = (scores.view(-1, 1) + logp).view(b, -1) top_s, top_i = cand.topk(2 * k, dim=1) top_s, top_i = top_s.tolist(), top_i.tolist() vocab = logp.size(1) new_scores, origin, tokens = [], [], [] for i in range(b): keep = [] if open_[i]: for s, j in zip(top_s[i], top_i[i]): if s == float("-inf"): break beam_i, tok = divmod(j, vocab) if tok == eos: if len(finished[i]) < k: finished[i].append((norm(s, step + 1), seqs[i * k + beam_i])) else: keep.append((s, i * k + beam_i, tok)) if len(keep) == k: break # Stop once k hypotheses have finished or no open beam can beat the best. if finished[i] and (len(finished[i]) >= k or not keep or ( len_pen >= 0 and norm(keep[0][0], limit[i] + 1) <= max(f[0] for f in finished[i]))): open_[i] = False keep = [] while len(keep) < k: # pad closed or short groups with dead beams keep.append((float("-inf"), i * k, pad)) for s, row, tok in keep: new_scores.append(s) origin.append(row) tokens.append(tok) if not any(open_): break idx = torch.tensor(origin, device=dev) for c in caches: c["self"]["k"] = c["self"]["k"].index_select(0, idx) c["self"]["v"] = c["self"]["v"].index_select(0, idx) seqs = [seqs[o] + [t] for o, t in zip(origin, tokens)] scores = torch.tensor(new_scores, device=dev).view(b, k) cur = torch.tensor(tokens, device=dev).view(-1, 1) out = [] for i in range(b): if not finished[i]: # ran out of steps: take the best open beam row = i * k + int(scores[i].argmax()) finished[i].append((norm(float(scores[i].max()), len(seqs[row])), seqs[row])) out.append(max(finished[i], key=lambda f: f[0])[1]) return out def _block_ngrams(logits, gen, n): """Set to -inf every token that would repeat an n-gram already in that row of gen.""" rows, cols = [], [] for r, g in enumerate(gen): if len(g) < n: continue prefix = g[len(g) - n + 1:] if n > 1 else [] for i in range(len(g) - n + 1): if g[i:i + n - 1] == prefix: rows.append(r) cols.append(g[i + n - 1]) if rows: logits[rows, cols] = float("-inf") def _force_eos(logits, rows, eos): """Make eos the only choice in the rows where `rows` (a bool tensor) is True.""" if rows.any(): e = logits[rows, eos].clone() logits[rows] = float("-inf") logits[rows, eos] = torch.where(torch.isinf(e), torch.zeros_like(e), e)