QKVE-61M / inference.py
Sticzu's picture
Initial release: QKVE-61M architecture, weights, ONNX and documentation
29d2351 verified
Raw History Blame Contribute Delete
3.53 kB
#!/usr/bin/env python3
"""
Przykładowy skrypt inferencyjny dla modelu QKVE-61M.
Wymaga zainstalowanego: torch, transformers, safetensors
"""
import os
import torch
import torch.nn.functional as F
from transformers import AutoTokenizer
from safetensors.torch import load_file
from model import QKVE30Transformer
def load_qkve_model(model_dir=".", device="cuda" if torch.cuda.is_available() else "cpu"):
tokenizer = AutoTokenizer.from_pretrained(model_dir, clean_up_tokenization_spaces=False)
model = QKVE30Transformer(
vocab_size=len(tokenizer),
d_model=384,
num_layers=30,
num_heads=12,
num_kv_heads=4,
base_ffn=512,
max_seq_len=512
).to(device)
safetensors_path = os.path.join(model_dir, "model.safetensors")
bin_path = os.path.join(model_dir, "pytorch_model.bin")
if os.path.exists(safetensors_path):
weights = load_file(safetensors_path)
elif os.path.exists(bin_path):
weights = torch.load(bin_path, map_location=device, weights_only=True)
else:
raise FileNotFoundError("Nie znaleziono pliku wag (model.safetensors ani pytorch_model.bin).")
model.load_state_dict(weights, strict=False)
model.eval()
return model, tokenizer
def generate(model, tokenizer, prompt, max_new_tokens=64, temperature=0.7, top_p=0.9, device="cuda" if torch.cuda.is_available() else "cpu"):
# Formatowanie ChatML
formatted = f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n"
input_ids = tokenizer.encode(formatted, return_tensors="pt").to(device)
eos_token_ids = {2, 32001}
generated = input_ids.clone()
with torch.no_grad():
for _ in range(max_new_tokens):
context = generated[:, -512:]
out = model(context)
logits = out[0] if isinstance(out, tuple) else out
logits = logits[:, -1, :]
if temperature > 0:
logits = logits / temperature
if top_p < 1.0:
sorted_logits, sorted_indices = torch.sort(logits, descending=True)
cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1)
sorted_indices_to_remove = cumulative_probs > top_p
sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone()
sorted_indices_to_remove[..., 0] = 0
indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove)
logits = logits.masked_fill(indices_to_remove, -float("inf"))
probs = F.softmax(logits, dim=-1)
next_token = torch.multinomial(probs, num_samples=1)
else:
next_token = torch.argmax(logits, dim=-1, keepdim=True)
if next_token.item() in eos_token_ids:
break
generated = torch.cat([generated, next_token], dim=1)
output_text = tokenizer.decode(generated[0, input_ids.shape[1]:], skip_special_tokens=True)
return output_text.strip()
if __name__ == "__main__":
import sys
device = "cuda" if torch.cuda.is_available() else "cpu"
print(f"Ładowanie modelu QKVE-61M na urządzeniu: {device}...")
model, tokenizer = load_qkve_model(".", device=device)
prompt = sys.argv[1] if len(sys.argv) > 1 else "Cześć! Kim jesteś i jak działasz?"
print(f"\nPrompt: {prompt}")
print("Odpowiedź:", generate(model, tokenizer, prompt, device=device))