Download test.py from AllOneAI/AllOneLM-Nano-1708: direct link, hf CLI and curl.
- Browser
- Download file 12.2 kB
-
https://huggingface.co/AllOneAI/AllOneLM-Nano-1708/resolve/main/test.py
- Command line
-
hf download hf://AllOneAI/AllOneLM-Nano-1708/test.py
-
curl -L -o test.py https://huggingface.co/AllOneAI/AllOneLM-Nano-1708/resolve/main/test.py
12.2 kB
| import os | |
| import torch | |
| import torch.nn as nn | |
| import sentencepiece as sePi | |
| import math | |
| import torch.nn.functional as F | |
| from torch.nn.utils.rnn import pad_sequence | |
| from mamba_ssm import Mamba2 | |
| from mamba_ssm.utils.generation import InferenceParams | |
| class PositionalEncoding(nn.Module): | |
| def __init__(self, d_model, dropout=0.2, max_len=16000): | |
| super().__init__() | |
| self.dropout = nn.Dropout(p=dropout) | |
| pe = torch.zeros(max_len, d_model) | |
| position = torch.arange(0, max_len, dtype=torch.float).unsqueeze(1) | |
| div_term = torch.exp(torch.arange(0, d_model, 2).float() * (-math.log(10000.0) / d_model)) | |
| pe[:, 0::2] = torch.sin(position * div_term) | |
| pe[:, 1::2] = torch.cos(position * div_term) | |
| pe = pe.unsqueeze(0) | |
| self.register_buffer('pe', pe) | |
| def forward(self, x): | |
| x = x + self.pe[:, :x.size(1)] | |
| return self.dropout(x) | |
| def createMaskCasual(size): | |
| mask = torch.triu(torch.ones(size, size) * float('-inf'), diagonal=1) | |
| return mask | |
| def apply_repetition_penalty(logits, generated_ids, penalty): | |
| if not generated_ids: | |
| return logits | |
| ids = torch.tensor(list(set(generated_ids)), device=logits.device, dtype=torch.long) | |
| score = torch.gather(logits, 0, ids) | |
| score = torch.where(score < 0, score * penalty, score / penalty) | |
| logits.scatter_(0, ids, score) | |
| return logits | |
| def sample_with_penalty(logits, generated, temperature=0.7, penalty=1.12, top_k=40, top_p=0.9, min_p=0.02): | |
| logits = logits.squeeze().clone() | |
| if generated: | |
| logits = apply_repetition_penalty(logits, generated, penalty) | |
| logits = logits / max(temperature, 1e-6) | |
| probs = F.softmax(logits, dim=-1) | |
| top_prob, _ = torch.max(probs, dim=-1) | |
| min_p_threshold = min_p * top_prob | |
| logits[probs < min_p_threshold] = -float('Inf') | |
| if top_k > 0: | |
| indices_to_remove = logits < torch.topk(logits, top_k)[0][..., -1, None] | |
| logits[indices_to_remove] = -float('Inf') | |
| if top_p < 1.0: | |
| sorted_logits, sorted_indices = torch.sort(logits, descending=True) | |
| cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1) | |
| sorted_indices_to_remove = cumulative_probs > top_p | |
| sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone() | |
| sorted_indices_to_remove[..., 0] = 0 | |
| indices_to_remove = sorted_indices[sorted_indices_to_remove] | |
| logits[indices_to_remove] = -float('Inf') | |
| final_probs = F.softmax(logits, dim=-1) | |
| return torch.multinomial(final_probs, 1).item() | |
| class loopModel(nn.Module): | |
| def __init__(self, vocabSize, dModel, numberHeads, dropout=0.2, numDecoderLayers=24): | |
| super().__init__() | |
| self.dModel = dModel | |
| self.embedding = nn.Embedding(vocabSize, dModel) | |
| self.norm_dmodel = nn.RMSNorm(dModel) | |
| self.norm_dmodel4 = nn.RMSNorm(dModel * 4) | |
| self.iP2 = DenseLiquid(in_features=dModel, out_features=dModel * 4) | |
| self.iP3 = DenseLiquid(in_features=dModel * 4, out_features=dModel) | |
| self.mambaL = nn.ModuleList([ | |
| Mamba2( | |
| d_model=dModel, | |
| d_state=512, | |
| d_conv=4, | |
| expand=4, | |
| layer_idx=i | |
| ) for i in range(numDecoderLayers) | |
| ]) | |
| self.fcO = DenseLiquid(in_features=dModel * 4, out_features=dModel) | |
| self.fcD = DenseLiquid(in_features=dModel, out_features=dModel) | |
| self.final_projection = nn.Linear(dModel, vocabSize) | |
| self.posEmb = nn.Embedding(context_window, dModel) | |
| def forward(self, x, inference_params=None): | |
| x = self.embedding(x) * math.sqrt(self.dModel) | |
| for layer in self.mambaL: | |
| x = x + layer( | |
| x, | |
| inference_params=inference_params | |
| ) | |
| x = self.norm_dmodel(x) | |
| transformInputB = self.iP2(x) | |
| transformInputC = self.norm_dmodel4(transformInputB) | |
| densing = self.fcO(transformInputC) | |
| densingNorm = self.norm_dmodel(densing) | |
| densing2 = self.fcD(densingNorm) | |
| densingN = self.norm_dmodel(densing2) | |
| output = self.final_projection(densingN) | |
| return output | |
| class DenseLiquid(nn.Module): | |
| def __init__(self, in_features, out_features): | |
| super(DenseLiquid, self).__init__() | |
| self.inF = in_features | |
| self.outF = out_features | |
| totalOFe = 256 + (2 * out_features) | |
| self.fusedProj = nn.Linear(in_features, totalOFe) | |
| self.tau = nn.Parameter(torch.ones(out_features)) | |
| self.layerN = nn.RMSNorm(out_features) | |
| self.dtO = nn.Linear(256, out_features) | |
| self._initW() | |
| def _initW(self): | |
| with torch.no_grad(): | |
| wMain = self.fusedProj.weight[256 + self.outF :, :] | |
| bMain = self.fusedProj.bias[256 + self.outF :] | |
| nn.init.orthogonal_(wMain, gain=0.1) | |
| nn.init.zeros_(bMain) | |
| def forward(self, x): | |
| fusedO = self.fusedProj(x) | |
| dtIn, gateIn, linEq = torch.split(fusedO, [256, self.outF, self.outF], dim=-1) | |
| leakyN = 1e-5 | |
| dt = torch.sigmoid(self.dtO(F.gelu(dtIn))) | |
| gate = torch.sigmoid(gateIn) | |
| tauV = F.softplus(self.tau) + 1e-5 | |
| fOut = F.gelu(linEq) | |
| differentiator = (fOut - linEq) / (tauV + 1e-6) | |
| differentiator = torch.clamp(differentiator, min=-5.0, max=9.0) | |
| newState = linEq + (dt * gate * differentiator) | |
| return self.layerN(newState) | |
| numberHeads = 16 | |
| numEncoderLayers = 24 | |
| dModel = 1024 | |
| context_window = 12288 | |
| vocabSize = 24000 | |
| device = "cuda" | |
| if device != "cuda": | |
| torch.set_num_threads(8) | |
| else: | |
| pass | |
| data = [] | |
| sePi = sePi.SentencePieceProcessor() | |
| sePi.load("AllOneLMModel.model") | |
| model = loopModel(vocabSize, dModel, numberHeads, dropout=0.01, numDecoderLayers=numEncoderLayers) | |
| model.to(device) | |
| state_dict = torch.load("model_lm.pth") | |
| model.load_state_dict(state_dict) | |
| showReasoning = True | |
| mandatoryThink = False | |
| sampling = "Bare" | |
| #systemInstruct = "" | |
| listing = "" | |
| thinking = mandatoryThink | |
| chat = True | |
| model.eval() | |
| fullGenerationContext = "" | |
| ee = 0 | |
| print("Starting model...") | |
| with torch.no_grad(), torch.amp.autocast(device_type='cuda', dtype=torch.bfloat16): | |
| fullGenerationContext = "" | |
| while True: | |
| inference_params = InferenceParams( | |
| max_batch_size=1, | |
| max_seqlen=context_window | |
| ) | |
| systemPrompt = "You are an expert legal advisor, you write legislation for a living." | |
| user_text = input("Question: ") | |
| if mandatoryThink == True: | |
| thinking = True | |
| if ee == 0: | |
| systemInstruct = f"<|system|>{systemPrompt}<|endSystem|>" | |
| else: | |
| systemInstruct = "" | |
| fullGenerationContext += systemInstruct | |
| fullGenerationContext += f"<|user|>{user_text}<|assistant|><|think|>" | |
| else: | |
| if chat == True: | |
| if ee == 0: | |
| systemInstruct = f"<|system|>{systemPrompt}<|endSystem|>" | |
| else: | |
| systemInstruct = "" | |
| fullGenerationContext += systemInstruct | |
| fullGenerationContext += f"<|user|>{user_text}<|assistant|>" | |
| else: | |
| systemInstruct = "" | |
| fullGenerationContext += f"{user_text}" | |
| tokens_list = sePi.encode(fullGenerationContext, out_type=int) | |
| currentToken = 0 | |
| afterList = [] | |
| sendThrough = tokens_list[-context_window:] | |
| sendThroughTensor = torch.tensor([sendThrough], dtype=torch.long, device=device) | |
| sTT = sendThroughTensor.clone().detach() | |
| predictions = model(sendThroughTensor, inference_params=inference_params) | |
| inference_params.seqlen_offset = len(sendThrough) | |
| nextTokenLog = predictions[0, -1, :] | |
| layer = model.mambaL[0] | |
| conv_state, ssm_state = layer._get_states_from_cache( | |
| inference_params, | |
| batch_size=1 | |
| ) | |
| if sampling == True: | |
| temp = 0.5 | |
| pen = 1.15 | |
| top_k = 30 | |
| top_p = 0.955 | |
| nextTokenID = sample_with_penalty( | |
| nextTokenLog, | |
| afterList, | |
| temperature=temp, | |
| penalty=pen, | |
| top_k=top_k, | |
| top_p=top_p | |
| ) | |
| elif sampling == "Bare": | |
| temp = 0.1 | |
| pen = 1.2 | |
| top_k = 12 | |
| top_p = 0.98 | |
| nextTokenID = sample_with_penalty( | |
| nextTokenLog, | |
| afterList, | |
| temperature=temp, | |
| penalty=pen, | |
| top_k=top_k, | |
| top_p=top_p | |
| ) | |
| else: | |
| nextTokenID = torch.argmax(nextTokenLog, dim=-1).item() | |
| tokens_list.append(nextTokenID) | |
| afterList.append(nextTokenID) | |
| for i in range(3096): | |
| sendThrough = tokens_list[-context_window:] | |
| sendThroughTensor = torch.tensor([[sendThrough[-1]]], dtype=torch.long, device=device) | |
| predictions = model(sendThroughTensor, inference_params=inference_params) | |
| nextTokenLog = predictions[0, -1, :] | |
| if sampling == True: | |
| temp = 0.2 | |
| pen = 1.15 | |
| top_k = 24 | |
| top_p = 0.955 | |
| nextTokenID = sample_with_penalty( | |
| nextTokenLog, | |
| afterList, | |
| temperature=temp, | |
| penalty=pen, | |
| top_k=top_k, | |
| top_p=top_p | |
| ) | |
| elif sampling == "Bare": | |
| temp = 0.1 | |
| pen = 1.15 | |
| top_k = 24 | |
| top_p = 0.965 | |
| nextTokenID = sample_with_penalty( | |
| nextTokenLog, | |
| afterList, | |
| temperature=temp, | |
| penalty=pen, | |
| top_k=top_k, | |
| top_p=top_p | |
| ) | |
| else: | |
| nextTokenID = torch.argmax(nextTokenLog, dim=-1).item() | |
| inference_params.seqlen_offset += 1 | |
| if nextTokenID == sePi.PieceToId("<|user|>"): | |
| nextTokenID = sePi.PieceToId("<|eosText|>") | |
| tokens_list.append(nextTokenID) | |
| afterList.append(nextTokenID) | |
| if currentToken > 2048 and sePi.PieceToId("<|endThink|>") not in tokens_list and thinking == True: | |
| tokens_list.append(sePi.PieceToId("<|endThink|>")) | |
| thinking = False | |
| currentToken += 1 | |
| if sePi.PieceToId("<|endThink|>") in tokens_list: | |
| thinking = False | |
| if thinking == True and showReasoning == False: | |
| current_output_stringA = f"Context: {len(tokens_list)}/{context_window}\n Reasoning..." | |
| else: | |
| current_output_string = sePi.decode_ids(tokens_list) | |
| current_output_stringA = f"Context: {len(tokens_list)}/{context_window}\n" + current_output_string | |
| print(current_output_stringA) | |
| if nextTokenID == sePi.PieceToId("<|think|>"): | |
| thinking = True | |
| current_output_string = f"{current_output_string}\n\n" | |
| if nextTokenID == sePi.PieceToId("<|endThink|>"): | |
| thinking = False | |
| print(current_output_stringA) | |
| if nextTokenID == sePi.PieceToId("<|eosText|>"): | |
| break | |
| fullGenerationContext = current_output_string | |
| #print(fullGenerationContext) | |