Bitsy500M / modeling_bitsy.py
zarko1321's picture
Upload 23 files
cdf6b36 verified
Raw History Blame Contribute Delete
9.93 kB
"""Bitsy Parallel: custom PyTorch decoder, not a wrapper around Llama.
Parallel attention/MLP branches, learned query/key RMS normalization,
adjacent-pair rotary embeddings, grouped-query causal attention, tied logits.
The blocks use established techniques; no claim of a new research invention.
"""
import math
import torch
from torch import nn
from torch.nn import functional as F
from torch.utils.checkpoint import checkpoint
from transformers import PreTrainedModel, GenerationMixin
from transformers.modeling_outputs import CausalLMOutputWithPast
from .configuration_bitsy import BitsyConfig
class BitsyRMSNorm(nn.Module):
def __init__(self, width, eps):
super().__init__()
self.weight = nn.Parameter(torch.ones(width))
self.eps = eps
def forward(self, x):
z = x.float()
z = z * torch.rsqrt(z.square().mean(-1, keepdim=True) + self.eps)
return (z * self.weight.float()).to(x.dtype)
def rotate_pairs(x, positions, theta):
d = x.shape[-1]
freq = theta ** (-torch.arange(0, d, 2, device=x.device, dtype=torch.float32) / d)
angles = positions.float().unsqueeze(-1) * freq
cos, sin = angles.cos().unsqueeze(1), angles.sin().unsqueeze(1)
a, b = x[..., 0::2].float(), x[..., 1::2].float()
return torch.stack((a*cos-b*sin, a*sin+b*cos), -1).flatten(-2).to(x.dtype)
class BitsyAttention(nn.Module):
def __init__(self, c):
super().__init__()
self.c = c
h, kv = c.hidden_size, c.num_key_value_heads*c.head_dim
self.q_proj = nn.Linear(h, h, bias=False)
self.k_proj = nn.Linear(h, kv, bias=False)
self.v_proj = nn.Linear(h, kv, bias=False)
self.out_proj = nn.Linear(h, h, bias=False)
self.q_norm = BitsyRMSNorm(c.head_dim, c.rms_norm_eps)
self.k_norm = BitsyRMSNorm(c.head_dim, c.rms_norm_eps)
def forward(self, x, positions, attention_mask=None, past=None, use_cache=False):
b, t, h = x.shape
c = self.c
q = self.q_norm(self.q_proj(x).view(b,t,c.num_attention_heads,c.head_dim).transpose(1,2))
k = self.k_norm(self.k_proj(x).view(b,t,c.num_key_value_heads,c.head_dim).transpose(1,2))
v = self.v_proj(x).view(b,t,c.num_key_value_heads,c.head_dim).transpose(1,2)
q, k = rotate_pairs(q, positions, c.rope_theta), rotate_pairs(k, positions, c.rope_theta)
past_len = 0 if past is None else past[0].shape[2]
if past is not None:
k, v = torch.cat((past[0], k), 2), torch.cat((past[1], v), 2)
present = (k, v) if use_cache else None
total = k.shape[2]
mask = None
causal = past_len == 0 and attention_mask is None
if not causal:
allowed = torch.arange(total, device=x.device)[None,:] <= (past_len+torch.arange(t,device=x.device))[:,None]
mask = allowed[None,None,:,:]
if attention_mask is not None:
if attention_mask.ndim != 2 or attention_mask.shape != (b,total):
raise ValueError("attention_mask must be [batch, past_length + current_length]")
mask = mask & attention_mask[:,None,None,:].bool()
# Native CUDA GQA avoids materializing repeated K/V heads where supported.
if x.is_cuda:
y = F.scaled_dot_product_attention(q,k,v,attn_mask=mask,is_causal=causal,enable_gqa=True)
else:
r = c.num_attention_heads // c.num_key_value_heads
y = F.scaled_dot_product_attention(q,k.repeat_interleave(r,1),v.repeat_interleave(r,1),
attn_mask=mask,is_causal=causal)
return self.out_proj(y.transpose(1,2).contiguous().view(b,t,h)), present
class BitsyBlock(nn.Module):
def __init__(self, c):
super().__init__()
self.attn_norm = BitsyRMSNorm(c.hidden_size,c.rms_norm_eps)
self.mlp_norm = BitsyRMSNorm(c.hidden_size,c.rms_norm_eps)
self.attn = BitsyAttention(c)
self.gate = nn.Linear(c.hidden_size,c.intermediate_size,bias=False)
self.up = nn.Linear(c.hidden_size,c.intermediate_size,bias=False)
self.down = nn.Linear(c.intermediate_size,c.hidden_size,bias=False)
def forward(self, x, positions, attention_mask=None, past=None, use_cache=False):
a, present = self.attn(self.attn_norm(x),positions,attention_mask,past,use_cache)
z = self.mlp_norm(x)
# Both branches read the same residual stream (unlike serial Llama blocks).
return x + a + self.down(F.silu(self.gate(z))*self.up(z)), present
class BitsyForCausalLM(PreTrainedModel, GenerationMixin):
config_class = BitsyConfig
base_model_prefix = "bitsy"
main_input_name = "input_ids"
_tied_weights_keys = ["lm_head.weight"]
supports_gradient_checkpointing = True
_supports_cache_class = False
_no_split_modules = ["BitsyBlock"]
def __init__(self, config):
super().__init__(config)
self.embed_tokens = nn.Embedding(config.vocab_size, config.hidden_size)
self.blocks = nn.ModuleList([BitsyBlock(config) for _ in range(config.num_hidden_layers)])
self.final_norm = BitsyRMSNorm(config.hidden_size,config.rms_norm_eps)
self.lm_head = nn.Linear(config.hidden_size,config.vocab_size,bias=False)
self.gradient_checkpointing = False
self.post_init()
@classmethod
def _supports_default_dynamic_cache(cls):
return False
def _init_weights(self, module):
if isinstance(module, (nn.Linear,nn.Embedding)):
std = self.config.initializer_range
if isinstance(module,nn.Linear) and module.out_features == self.config.hidden_size and module is not self.lm_head:
std /= math.sqrt(2*self.config.num_hidden_layers)
nn.init.normal_(module.weight,mean=0,std=std)
elif isinstance(module,BitsyRMSNorm):
nn.init.ones_(module.weight)
def get_input_embeddings(self): return self.embed_tokens
def set_input_embeddings(self,value): self.embed_tokens=value
def get_output_embeddings(self): return self.lm_head
def set_output_embeddings(self,value): self.lm_head=value
def gradient_checkpointing_enable(self, gradient_checkpointing_kwargs=None):
self.gradient_checkpointing = True
def gradient_checkpointing_disable(self):
self.gradient_checkpointing = False
def forward(self, input_ids=None, attention_mask=None, position_ids=None,
past_key_values=None, inputs_embeds=None, labels=None,
use_cache=None, return_dict=None, output_attentions=False,
output_hidden_states=False, **kwargs):
if output_attentions or output_hidden_states:
raise NotImplementedError("Attention/hidden-state collection is not implemented")
if (input_ids is None) == (inputs_embeds is None):
raise ValueError("Provide exactly one of input_ids or inputs_embeds")
x = self.embed_tokens(input_ids) if inputs_embeds is None else inputs_embeds
b,t,_ = x.shape
past_len = 0 if past_key_values is None else past_key_values[0][0].shape[2]
if past_len+t > self.config.max_position_embeddings:
raise ValueError("Context exceeded. Context extension requires deliberate training and validation.")
use_cache = self.config.use_cache if use_cache is None else use_cache
if self.training and self.gradient_checkpointing:
if past_key_values is not None: raise ValueError("Cannot train checkpointed with a KV cache")
use_cache = False
if position_ids is None:
if attention_mask is not None:
position_ids=(attention_mask.long().cumsum(-1)-1).clamp_min(0)[:,-t:]
else:
position_ids=torch.arange(past_len,past_len+t,device=x.device)[None,:].expand(b,-1)
presents=[]
for i,block in enumerate(self.blocks):
past = None if past_key_values is None else past_key_values[i]
if self.training and self.gradient_checkpointing:
def run(z, block=block):
return block(z,position_ids,attention_mask,None,False)[0]
x=checkpoint(run,x,use_reentrant=False)
else:
x,present=block(x,position_ids,attention_mask,past,use_cache)
if use_cache: presents.append(present)
logits=self.lm_head(self.final_norm(x))
loss=None
if labels is not None:
# The caller supplies unshifted labels; shift exactly once here.
if labels.shape != (b,t) or t < 2:
raise ValueError("Labels must match input shape with at least two positions")
target=labels[:,1:].to(logits.device)
if not (target != -100).any(): raise ValueError("No supervised target tokens")
loss=F.cross_entropy(logits[:,:-1,:].float().reshape(-1,self.config.vocab_size),
target.reshape(-1),ignore_index=-100)
result=CausalLMOutputWithPast(loss=loss,logits=logits,
past_key_values=tuple(presents) if use_cache else None)
return result if (self.config.use_return_dict if return_dict is None else return_dict) else result.to_tuple()
def prepare_inputs_for_generation(self,input_ids,past_key_values=None,attention_mask=None,**kwargs):
if past_key_values is not None:
past_len=past_key_values[0][0].shape[2]
input_ids=input_ids[:,past_len:] if input_ids.shape[1]>past_len else input_ids[:,-1:]
return {"input_ids":input_ids,"past_key_values":past_key_values,
"attention_mask":attention_mask,"use_cache":kwargs.get("use_cache",True)}
@staticmethod
def _reorder_cache(past_key_values,beam_idx):
return tuple(tuple(x.index_select(0,beam_idx.to(x.device)) for x in layer) for layer in past_key_values)