"""Bitsy Parallel: custom PyTorch decoder, not a wrapper around Llama. Parallel attention/MLP branches, learned query/key RMS normalization, adjacent-pair rotary embeddings, grouped-query causal attention, tied logits. The blocks use established techniques; no claim of a new research invention. """ import math import torch from torch import nn from torch.nn import functional as F from torch.utils.checkpoint import checkpoint from transformers import PreTrainedModel, GenerationMixin from transformers.modeling_outputs import CausalLMOutputWithPast from .configuration_bitsy import BitsyConfig class BitsyRMSNorm(nn.Module): def __init__(self, width, eps): super().__init__() self.weight = nn.Parameter(torch.ones(width)) self.eps = eps def forward(self, x): z = x.float() z = z * torch.rsqrt(z.square().mean(-1, keepdim=True) + self.eps) return (z * self.weight.float()).to(x.dtype) def rotate_pairs(x, positions, theta): d = x.shape[-1] freq = theta ** (-torch.arange(0, d, 2, device=x.device, dtype=torch.float32) / d) angles = positions.float().unsqueeze(-1) * freq cos, sin = angles.cos().unsqueeze(1), angles.sin().unsqueeze(1) a, b = x[..., 0::2].float(), x[..., 1::2].float() return torch.stack((a*cos-b*sin, a*sin+b*cos), -1).flatten(-2).to(x.dtype) class BitsyAttention(nn.Module): def __init__(self, c): super().__init__() self.c = c h, kv = c.hidden_size, c.num_key_value_heads*c.head_dim self.q_proj = nn.Linear(h, h, bias=False) self.k_proj = nn.Linear(h, kv, bias=False) self.v_proj = nn.Linear(h, kv, bias=False) self.out_proj = nn.Linear(h, h, bias=False) self.q_norm = BitsyRMSNorm(c.head_dim, c.rms_norm_eps) self.k_norm = BitsyRMSNorm(c.head_dim, c.rms_norm_eps) def forward(self, x, positions, attention_mask=None, past=None, use_cache=False): b, t, h = x.shape c = self.c q = self.q_norm(self.q_proj(x).view(b,t,c.num_attention_heads,c.head_dim).transpose(1,2)) k = self.k_norm(self.k_proj(x).view(b,t,c.num_key_value_heads,c.head_dim).transpose(1,2)) v = self.v_proj(x).view(b,t,c.num_key_value_heads,c.head_dim).transpose(1,2) q, k = rotate_pairs(q, positions, c.rope_theta), rotate_pairs(k, positions, c.rope_theta) past_len = 0 if past is None else past[0].shape[2] if past is not None: k, v = torch.cat((past[0], k), 2), torch.cat((past[1], v), 2) present = (k, v) if use_cache else None total = k.shape[2] mask = None causal = past_len == 0 and attention_mask is None if not causal: allowed = torch.arange(total, device=x.device)[None,:] <= (past_len+torch.arange(t,device=x.device))[:,None] mask = allowed[None,None,:,:] if attention_mask is not None: if attention_mask.ndim != 2 or attention_mask.shape != (b,total): raise ValueError("attention_mask must be [batch, past_length + current_length]") mask = mask & attention_mask[:,None,None,:].bool() # Native CUDA GQA avoids materializing repeated K/V heads where supported. if x.is_cuda: y = F.scaled_dot_product_attention(q,k,v,attn_mask=mask,is_causal=causal,enable_gqa=True) else: r = c.num_attention_heads // c.num_key_value_heads y = F.scaled_dot_product_attention(q,k.repeat_interleave(r,1),v.repeat_interleave(r,1), attn_mask=mask,is_causal=causal) return self.out_proj(y.transpose(1,2).contiguous().view(b,t,h)), present class BitsyBlock(nn.Module): def __init__(self, c): super().__init__() self.attn_norm = BitsyRMSNorm(c.hidden_size,c.rms_norm_eps) self.mlp_norm = BitsyRMSNorm(c.hidden_size,c.rms_norm_eps) self.attn = BitsyAttention(c) self.gate = nn.Linear(c.hidden_size,c.intermediate_size,bias=False) self.up = nn.Linear(c.hidden_size,c.intermediate_size,bias=False) self.down = nn.Linear(c.intermediate_size,c.hidden_size,bias=False) def forward(self, x, positions, attention_mask=None, past=None, use_cache=False): a, present = self.attn(self.attn_norm(x),positions,attention_mask,past,use_cache) z = self.mlp_norm(x) # Both branches read the same residual stream (unlike serial Llama blocks). return x + a + self.down(F.silu(self.gate(z))*self.up(z)), present class BitsyForCausalLM(PreTrainedModel, GenerationMixin): config_class = BitsyConfig base_model_prefix = "bitsy" main_input_name = "input_ids" _tied_weights_keys = ["lm_head.weight"] supports_gradient_checkpointing = True _supports_cache_class = False _no_split_modules = ["BitsyBlock"] def __init__(self, config): super().__init__(config) self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size) self.blocks = nn.ModuleList([BitsyBlock(config) for _ in range(config.num_hidden_layers)]) self.final_norm = BitsyRMSNorm(config.hidden_size,config.rms_norm_eps) self.lm_head = nn.Linear(config.hidden_size,config.vocab_size,bias=False) self.gradient_checkpointing = False self.post_init() @classmethod def _supports_default_dynamic_cache(cls): return False def _init_weights(self, module): if isinstance(module, (nn.Linear,nn.Embedding)): std = self.config.initializer_range if isinstance(module,nn.Linear) and module.out_features == self.config.hidden_size and module is not self.lm_head: std /= math.sqrt(2*self.config.num_hidden_layers) nn.init.normal_(module.weight,mean=0,std=std) elif isinstance(module,BitsyRMSNorm): nn.init.ones_(module.weight) def get_input_embeddings(self): return self.embed_tokens def set_input_embeddings(self,value): self.embed_tokens=value def get_output_embeddings(self): return self.lm_head def set_output_embeddings(self,value): self.lm_head=value def gradient_checkpointing_enable(self, gradient_checkpointing_kwargs=None): self.gradient_checkpointing = True def gradient_checkpointing_disable(self): self.gradient_checkpointing = False def forward(self, input_ids=None, attention_mask=None, position_ids=None, past_key_values=None, inputs_embeds=None, labels=None, use_cache=None, return_dict=None, output_attentions=False, output_hidden_states=False, **kwargs): if output_attentions or output_hidden_states: raise NotImplementedError("Attention/hidden-state collection is not implemented") if (input_ids is None) == (inputs_embeds is None): raise ValueError("Provide exactly one of input_ids or inputs_embeds") x = self.embed_tokens(input_ids) if inputs_embeds is None else inputs_embeds b,t,_ = x.shape past_len = 0 if past_key_values is None else past_key_values[0][0].shape[2] if past_len+t > self.config.max_position_embeddings: raise ValueError("Context exceeded. Context extension requires deliberate training and validation.") use_cache = self.config.use_cache if use_cache is None else use_cache if self.training and self.gradient_checkpointing: if past_key_values is not None: raise ValueError("Cannot train checkpointed with a KV cache") use_cache = False if position_ids is None: if attention_mask is not None: position_ids=(attention_mask.long().cumsum(-1)-1).clamp_min(0)[:,-t:] else: position_ids=torch.arange(past_len,past_len+t,device=x.device)[None,:].expand(b,-1) presents=[] for i,block in enumerate(self.blocks): past = None if past_key_values is None else past_key_values[i] if self.training and self.gradient_checkpointing: def run(z, block=block): return block(z,position_ids,attention_mask,None,False)[0] x=checkpoint(run,x,use_reentrant=False) else: x,present=block(x,position_ids,attention_mask,past,use_cache) if use_cache: presents.append(present) logits=self.lm_head(self.final_norm(x)) loss=None if labels is not None: # The caller supplies unshifted labels; shift exactly once here. if labels.shape != (b,t) or t < 2: raise ValueError("Labels must match input shape with at least two positions") target=labels[:,1:].to(logits.device) if not (target != -100).any(): raise ValueError("No supervised target tokens") loss=F.cross_entropy(logits[:,:-1,:].float().reshape(-1,self.config.vocab_size), target.reshape(-1),ignore_index=-100) result=CausalLMOutputWithPast(loss=loss,logits=logits, past_key_values=tuple(presents) if use_cache else None) return result if (self.config.use_return_dict if return_dict is None else return_dict) else result.to_tuple() def prepare_inputs_for_generation(self,input_ids,past_key_values=None,attention_mask=None,**kwargs): if past_key_values is not None: past_len=past_key_values[0][0].shape[2] input_ids=input_ids[:,past_len:] if input_ids.shape[1]>past_len else input_ids[:,-1:] return {"input_ids":input_ids,"past_key_values":past_key_values, "attention_mask":attention_mask,"use_cache":kwargs.get("use_cache",True)} @staticmethod def _reorder_cache(past_key_values,beam_idx): return tuple(tuple(x.index_select(0,beam_idx.to(x.device)) for x in layer) for layer in past_key_values)