AllOneAI's picture
Upload 3 files
b0e6e39 verified
Raw History Blame Contribute Delete
12.2 kB
import os
import torch
import torch.nn as nn
import sentencepiece as sePi
import math
import torch.nn.functional as F
from torch.nn.utils.rnn import pad_sequence
from mamba_ssm import Mamba2
from mamba_ssm.utils.generation import InferenceParams
class PositionalEncoding(nn.Module):
def __init__(self, d_model, dropout=0.2, max_len=16000):
super().__init__()
self.dropout = nn.Dropout(p=dropout)
pe = torch.zeros(max_len, d_model)
position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1)
div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model))
pe[:, 0::2] = torch.sin(position * div_term)
pe[:, 1::2] = torch.cos(position * div_term)
pe = pe.unsqueeze(0)
self.register_buffer('pe', pe)
def forward(self, x):
x = x + self.pe[:, :x.size(1)]
return self.dropout(x)
def createMaskCasual(size):
mask = torch.triu(torch.ones(size, size) * float('-inf'), diagonal=1)
return mask
def apply_repetition_penalty(logits, generated_ids, penalty):
if not generated_ids:
return logits
ids = torch.tensor(list(set(generated_ids)), device=logits.device, dtype=torch.long)
score = torch.gather(logits, 0, ids)
score = torch.where(score < 0, score * penalty, score / penalty)
logits.scatter_(0, ids, score)
return logits
def sample_with_penalty(logits, generated, temperature=0.7, penalty=1.12, top_k=40, top_p=0.9, min_p=0.02):
logits = logits.squeeze().clone()
if generated:
logits = apply_repetition_penalty(logits, generated, penalty)
logits = logits / max(temperature, 1e-6)
probs = F.softmax(logits, dim=-1)
top_prob, _ = torch.max(probs, dim=-1)
min_p_threshold = min_p * top_prob
logits[probs < min_p_threshold] = -float('Inf')
if top_k > 0:
indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None]
logits[indices_to_remove] = -float('Inf')
if top_p < 1.0:
sorted_logits, sorted_indices = torch.sort(logits, descending=True)
cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
sorted_indices_to_remove = cumulative_probs > top_p
sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
sorted_indices_to_remove[..., 0] = 0
indices_to_remove = sorted_indices[sorted_indices_to_remove]
logits[indices_to_remove] = -float('Inf')
final_probs = F.softmax(logits, dim=-1)
return torch.multinomial(final_probs, 1).item()
class loopModel(nn.Module):
def __init__(self, vocabSize, dModel, numberHeads, dropout=0.2, numDecoderLayers=24):
super().__init__()
self.dModel = dModel
self.embedding = nn.Embedding(vocabSize, dModel)
self.norm_dmodel = nn.RMSNorm(dModel)
self.norm_dmodel4 = nn.RMSNorm(dModel * 4)
self.iP2 = DenseLiquid(in_features=dModel, out_features=dModel * 4)
self.iP3 = DenseLiquid(in_features=dModel * 4, out_features=dModel)
self.mambaL = nn.ModuleList([
Mamba2(
d_model=dModel,
d_state=512,
d_conv=4,
expand=4,
layer_idx=i
) for i in range(numDecoderLayers)
])
self.fcO = DenseLiquid(in_features=dModel * 4, out_features=dModel)
self.fcD = DenseLiquid(in_features=dModel, out_features=dModel)
self.final_projection = nn.Linear(dModel, vocabSize)
self.posEmb = nn.Embedding(context_window, dModel)
def forward(self, x, inference_params=None):
x = self.embedding(x) * math.sqrt(self.dModel)
for layer in self.mambaL:
x = x + layer(
x,
inference_params=inference_params
)
x = self.norm_dmodel(x)
transformInputB = self.iP2(x)
transformInputC = self.norm_dmodel4(transformInputB)
densing = self.fcO(transformInputC)
densingNorm = self.norm_dmodel(densing)
densing2 = self.fcD(densingNorm)
densingN = self.norm_dmodel(densing2)
output = self.final_projection(densingN)
return output
class DenseLiquid(nn.Module):
def __init__(self, in_features, out_features):
super(DenseLiquid, self).__init__()
self.inF = in_features
self.outF = out_features
totalOFe = 256 + (2 * out_features)
self.fusedProj = nn.Linear(in_features, totalOFe)
self.tau = nn.Parameter(torch.ones(out_features))
self.layerN = nn.RMSNorm(out_features)
self.dtO = nn.Linear(256, out_features)
self._initW()
def _initW(self):
with torch.no_grad():
wMain = self.fusedProj.weight[256 + self.outF :, :]
bMain = self.fusedProj.bias[256 + self.outF :]
nn.init.orthogonal_(wMain, gain=0.1)
nn.init.zeros_(bMain)
def forward(self, x):
fusedO = self.fusedProj(x)
dtIn, gateIn, linEq = torch.split(fusedO, [256, self.outF, self.outF], dim=-1)
leakyN = 1e-5
dt = torch.sigmoid(self.dtO(F.gelu(dtIn)))
gate = torch.sigmoid(gateIn)
tauV = F.softplus(self.tau) + 1e-5
fOut = F.gelu(linEq)
differentiator = (fOut - linEq) / (tauV + 1e-6)
differentiator = torch.clamp(differentiator, min=-5.0, max=9.0)
newState = linEq + (dt * gate * differentiator)
return self.layerN(newState)
numberHeads = 16
numEncoderLayers = 24
dModel = 1024
context_window = 12288
vocabSize = 24000
device = "cuda"
if device != "cuda":
torch.set_num_threads(8)
else:
pass
data = []
sePi = sePi.SentencePieceProcessor()
sePi.load("AllOneLMModel.model")
model = loopModel(vocabSize, dModel, numberHeads, dropout=0.01, numDecoderLayers=numEncoderLayers)
model.to(device)
state_dict = torch.load("model_lm.pth")
model.load_state_dict(state_dict)
showReasoning = True
mandatoryThink = False
sampling = "Bare"
#systemInstruct = ""
listing = ""
thinking = mandatoryThink
chat = True
model.eval()
fullGenerationContext = ""
ee = 0
print("Starting model...")
with torch.no_grad(), torch.amp.autocast(device_type='cuda', dtype=torch.bfloat16):
fullGenerationContext = ""
while True:
inference_params = InferenceParams(
max_batch_size=1,
max_seqlen=context_window
)
systemPrompt = "You are an expert legal advisor, you write legislation for a living."
user_text = input("Question: ")
if mandatoryThink == True:
thinking = True
if ee == 0:
systemInstruct = f"<|system|>{systemPrompt}<|endSystem|>"
else:
systemInstruct = ""
fullGenerationContext += systemInstruct
fullGenerationContext += f"<|user|>{user_text}<|assistant|><|think|>"
else:
if chat == True:
if ee == 0:
systemInstruct = f"<|system|>{systemPrompt}<|endSystem|>"
else:
systemInstruct = ""
fullGenerationContext += systemInstruct
fullGenerationContext += f"<|user|>{user_text}<|assistant|>"
else:
systemInstruct = ""
fullGenerationContext += f"{user_text}"
tokens_list = sePi.encode(fullGenerationContext, out_type=int)
currentToken = 0
afterList = []
sendThrough = tokens_list[-context_window:]
sendThroughTensor = torch.tensor([sendThrough], dtype=torch.long, device=device)
sTT = sendThroughTensor.clone().detach()
predictions = model(sendThroughTensor, inference_params=inference_params)
inference_params.seqlen_offset = len(sendThrough)
nextTokenLog = predictions[0, -1, :]
layer = model.mambaL[0]
conv_state, ssm_state = layer._get_states_from_cache(
inference_params,
batch_size=1
)
if sampling == True:
temp = 0.5
pen = 1.15
top_k = 30
top_p = 0.955
nextTokenID = sample_with_penalty(
nextTokenLog,
afterList,
temperature=temp,
penalty=pen,
top_k=top_k,
top_p=top_p
)
elif sampling == "Bare":
temp = 0.1
pen = 1.2
top_k = 12
top_p = 0.98
nextTokenID = sample_with_penalty(
nextTokenLog,
afterList,
temperature=temp,
penalty=pen,
top_k=top_k,
top_p=top_p
)
else:
nextTokenID = torch.argmax(nextTokenLog, dim=-1).item()
tokens_list.append(nextTokenID)
afterList.append(nextTokenID)
for i in range(3096):
sendThrough = tokens_list[-context_window:]
sendThroughTensor = torch.tensor([[sendThrough[-1]]], dtype=torch.long, device=device)
predictions = model(sendThroughTensor, inference_params=inference_params)
nextTokenLog = predictions[0, -1, :]
if sampling == True:
temp = 0.2
pen = 1.15
top_k = 24
top_p = 0.955
nextTokenID = sample_with_penalty(
nextTokenLog,
afterList,
temperature=temp,
penalty=pen,
top_k=top_k,
top_p=top_p
)
elif sampling == "Bare":
temp = 0.1
pen = 1.15
top_k = 24
top_p = 0.965
nextTokenID = sample_with_penalty(
nextTokenLog,
afterList,
temperature=temp,
penalty=pen,
top_k=top_k,
top_p=top_p
)
else:
nextTokenID = torch.argmax(nextTokenLog, dim=-1).item()
inference_params.seqlen_offset += 1
if nextTokenID == sePi.PieceToId("<|user|>"):
nextTokenID = sePi.PieceToId("<|eosText|>")
tokens_list.append(nextTokenID)
afterList.append(nextTokenID)
if currentToken > 2048 and sePi.PieceToId("<|endThink|>") not in tokens_list and thinking == True:
tokens_list.append(sePi.PieceToId("<|endThink|>"))
thinking = False
currentToken += 1
if sePi.PieceToId("<|endThink|>") in tokens_list:
thinking = False
if thinking == True and showReasoning == False:
current_output_stringA = f"Context: {len(tokens_list)}/{context_window}\n Reasoning..."
else:
current_output_string = sePi.decode_ids(tokens_list)
current_output_stringA = f"Context: {len(tokens_list)}/{context_window}\n" + current_output_string
print(current_output_stringA)
if nextTokenID == sePi.PieceToId("<|think|>"):
thinking = True
current_output_string = f"{current_output_string}\n\n"
if nextTokenID == sePi.PieceToId("<|endThink|>"):
thinking = False
print(current_output_stringA)
if nextTokenID == sePi.PieceToId("<|eosText|>"):
break
fullGenerationContext = current_output_string
#print(fullGenerationContext)