import os import torch import torch.nn as nn import sentencepiece as sePi import math import torch.nn.functional as F from torch.nn.utils.rnn import pad_sequence from mamba_ssm import Mamba2 from mamba_ssm.utils.generation import InferenceParams class PositionalEncoding(nn.Module): def __init__(self, d_model, dropout=0.2, max_len=16000): super().__init__() self.dropout = nn.Dropout(p=dropout) pe = torch.zeros(max_len, d_model) position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)) pe[:, 0::2] = torch.sin(position * div_term) pe[:, 1::2] = torch.cos(position * div_term) pe = pe.unsqueeze(0) self.register_buffer('pe', pe) def forward(self, x): x = x + self.pe[:, :x.size(1)] return self.dropout(x) def createMaskCasual(size): mask = torch.triu(torch.ones(size, size) * float('-inf'), diagonal=1) return mask def apply_repetition_penalty(logits, generated_ids, penalty): if not generated_ids: return logits ids = torch.tensor(list(set(generated_ids)), device=logits.device, dtype=torch.long) score = torch.gather(logits, 0, ids) score = torch.where(score < 0, score * penalty, score / penalty) logits.scatter_(0, ids, score) return logits def sample_with_penalty(logits, generated, temperature=0.7, penalty=1.12, top_k=40, top_p=0.9, min_p=0.02): logits = logits.squeeze().clone() if generated: logits = apply_repetition_penalty(logits, generated, penalty) logits = logits / max(temperature, 1e-6) probs = F.softmax(logits, dim=-1) top_prob, _ = torch.max(probs, dim=-1) min_p_threshold = min_p * top_prob logits[probs < min_p_threshold] = -float('Inf') if top_k > 0: indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None] logits[indices_to_remove] = -float('Inf') if top_p < 1.0: sorted_logits, sorted_indices = torch.sort(logits, descending=True) cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1) sorted_indices_to_remove = cumulative_probs > top_p sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone() sorted_indices_to_remove[..., 0] = 0 indices_to_remove = sorted_indices[sorted_indices_to_remove] logits[indices_to_remove] = -float('Inf') final_probs = F.softmax(logits, dim=-1) return torch.multinomial(final_probs, 1).item() class loopModel(nn.Module): def __init__(self, vocabSize, dModel, numberHeads, dropout=0.2, numDecoderLayers=24): super().__init__() self.dModel = dModel self.embedding = nn.Embedding(vocabSize, dModel) self.norm_dmodel = nn.RMSNorm(dModel) self.norm_dmodel4 = nn.RMSNorm(dModel * 4) self.iP2 = DenseLiquid(in_features=dModel, out_features=dModel * 4) self.iP3 = DenseLiquid(in_features=dModel * 4, out_features=dModel) self.mambaL = nn.ModuleList([ Mamba2( d_model=dModel, d_state=512, d_conv=4, expand=4, layer_idx=i ) for i in range(numDecoderLayers) ]) self.fcO = DenseLiquid(in_features=dModel * 4, out_features=dModel) self.fcD = DenseLiquid(in_features=dModel, out_features=dModel) self.final_projection = nn.Linear(dModel, vocabSize) self.posEmb = nn.Embedding(context_window, dModel) def forward(self, x, inference_params=None): x = self.embedding(x) * math.sqrt(self.dModel) for layer in self.mambaL: x = x + layer( x, inference_params=inference_params ) x = self.norm_dmodel(x) transformInputB = self.iP2(x) transformInputC = self.norm_dmodel4(transformInputB) densing = self.fcO(transformInputC) densingNorm = self.norm_dmodel(densing) densing2 = self.fcD(densingNorm) densingN = self.norm_dmodel(densing2) output = self.final_projection(densingN) return output class DenseLiquid(nn.Module): def __init__(self, in_features, out_features): super(DenseLiquid, self).__init__() self.inF = in_features self.outF = out_features totalOFe = 256 + (2 * out_features) self.fusedProj = nn.Linear(in_features, totalOFe) self.tau = nn.Parameter(torch.ones(out_features)) self.layerN = nn.RMSNorm(out_features) self.dtO = nn.Linear(256, out_features) self._initW() def _initW(self): with torch.no_grad(): wMain = self.fusedProj.weight[256 + self.outF :, :] bMain = self.fusedProj.bias[256 + self.outF :] nn.init.orthogonal_(wMain, gain=0.1) nn.init.zeros_(bMain) def forward(self, x): fusedO = self.fusedProj(x) dtIn, gateIn, linEq = torch.split(fusedO, [256, self.outF, self.outF], dim=-1) leakyN = 1e-5 dt = torch.sigmoid(self.dtO(F.gelu(dtIn))) gate = torch.sigmoid(gateIn) tauV = F.softplus(self.tau) + 1e-5 fOut = F.gelu(linEq) differentiator = (fOut - linEq) / (tauV + 1e-6) differentiator = torch.clamp(differentiator, min=-5.0, max=9.0) newState = linEq + (dt * gate * differentiator) return self.layerN(newState) numberHeads = 16 numEncoderLayers = 24 dModel = 1024 context_window = 12288 vocabSize = 24000 device = "cuda" if device != "cuda": torch.set_num_threads(8) else: pass data = [] sePi = sePi.SentencePieceProcessor() sePi.load("AllOneLMModel.model") model = loopModel(vocabSize, dModel, numberHeads, dropout=0.01, numDecoderLayers=numEncoderLayers) model.to(device) state_dict = torch.load("model_lm.pth") model.load_state_dict(state_dict) showReasoning = True mandatoryThink = False sampling = "Bare" #systemInstruct = "" listing = "" thinking = mandatoryThink chat = True model.eval() fullGenerationContext = "" ee = 0 print("Starting model...") with torch.no_grad(), torch.amp.autocast(device_type='cuda', dtype=torch.bfloat16): fullGenerationContext = "" while True: inference_params = InferenceParams( max_batch_size=1, max_seqlen=context_window ) systemPrompt = "You are an expert legal advisor, you write legislation for a living." user_text = input("Question: ") if mandatoryThink == True: thinking = True if ee == 0: systemInstruct = f"<|system|>{systemPrompt}<|endSystem|>" else: systemInstruct = "" fullGenerationContext += systemInstruct fullGenerationContext += f"<|user|>{user_text}<|assistant|><|think|>" else: if chat == True: if ee == 0: systemInstruct = f"<|system|>{systemPrompt}<|endSystem|>" else: systemInstruct = "" fullGenerationContext += systemInstruct fullGenerationContext += f"<|user|>{user_text}<|assistant|>" else: systemInstruct = "" fullGenerationContext += f"{user_text}" tokens_list = sePi.encode(fullGenerationContext, out_type=int) currentToken = 0 afterList = [] sendThrough = tokens_list[-context_window:] sendThroughTensor = torch.tensor([sendThrough], dtype=torch.long, device=device) sTT = sendThroughTensor.clone().detach() predictions = model(sendThroughTensor, inference_params=inference_params) inference_params.seqlen_offset = len(sendThrough) nextTokenLog = predictions[0, -1, :] layer = model.mambaL[0] conv_state, ssm_state = layer._get_states_from_cache( inference_params, batch_size=1 ) if sampling == True: temp = 0.5 pen = 1.15 top_k = 30 top_p = 0.955 nextTokenID = sample_with_penalty( nextTokenLog, afterList, temperature=temp, penalty=pen, top_k=top_k, top_p=top_p ) elif sampling == "Bare": temp = 0.1 pen = 1.2 top_k = 12 top_p = 0.98 nextTokenID = sample_with_penalty( nextTokenLog, afterList, temperature=temp, penalty=pen, top_k=top_k, top_p=top_p ) else: nextTokenID = torch.argmax(nextTokenLog, dim=-1).item() tokens_list.append(nextTokenID) afterList.append(nextTokenID) for i in range(3096): sendThrough = tokens_list[-context_window:] sendThroughTensor = torch.tensor([[sendThrough[-1]]], dtype=torch.long, device=device) predictions = model(sendThroughTensor, inference_params=inference_params) nextTokenLog = predictions[0, -1, :] if sampling == True: temp = 0.2 pen = 1.15 top_k = 24 top_p = 0.955 nextTokenID = sample_with_penalty( nextTokenLog, afterList, temperature=temp, penalty=pen, top_k=top_k, top_p=top_p ) elif sampling == "Bare": temp = 0.1 pen = 1.15 top_k = 24 top_p = 0.965 nextTokenID = sample_with_penalty( nextTokenLog, afterList, temperature=temp, penalty=pen, top_k=top_k, top_p=top_p ) else: nextTokenID = torch.argmax(nextTokenLog, dim=-1).item() inference_params.seqlen_offset += 1 if nextTokenID == sePi.PieceToId("<|user|>"): nextTokenID = sePi.PieceToId("<|eosText|>") tokens_list.append(nextTokenID) afterList.append(nextTokenID) if currentToken > 2048 and sePi.PieceToId("<|endThink|>") not in tokens_list and thinking == True: tokens_list.append(sePi.PieceToId("<|endThink|>")) thinking = False currentToken += 1 if sePi.PieceToId("<|endThink|>") in tokens_list: thinking = False if thinking == True and showReasoning == False: current_output_stringA = f"Context: {len(tokens_list)}/{context_window}\n Reasoning..." else: current_output_string = sePi.decode_ids(tokens_list) current_output_stringA = f"Context: {len(tokens_list)}/{context_window}\n" + current_output_string print(current_output_stringA) if nextTokenID == sePi.PieceToId("<|think|>"): thinking = True current_output_string = f"{current_output_string}\n\n" if nextTokenID == sePi.PieceToId("<|endThink|>"): thinking = False print(current_output_stringA) if nextTokenID == sePi.PieceToId("<|eosText|>"): break fullGenerationContext = current_output_string #print(fullGenerationContext)