#!/usr/bin/env python3 """ Przykładowy skrypt inferencyjny dla modelu QKVE-61M. Wymaga zainstalowanego: torch, transformers, safetensors """ import os import torch import torch.nn.functional as F from transformers import AutoTokenizer from safetensors.torch import load_file from model import QKVE30Transformer def load_qkve_model(model_dir=".", device="cuda" if torch.cuda.is_available() else "cpu"): tokenizer = AutoTokenizer.from_pretrained(model_dir, clean_up_tokenization_spaces=False) model = QKVE30Transformer( vocab_size=len(tokenizer), d_model=384, num_layers=30, num_heads=12, num_kv_heads=4, base_ffn=512, max_seq_len=512 ).to(device) safetensors_path = os.path.join(model_dir, "model.safetensors") bin_path = os.path.join(model_dir, "pytorch_model.bin") if os.path.exists(safetensors_path): weights = load_file(safetensors_path) elif os.path.exists(bin_path): weights = torch.load(bin_path, map_location=device, weights_only=True) else: raise FileNotFoundError("Nie znaleziono pliku wag (model.safetensors ani pytorch_model.bin).") model.load_state_dict(weights, strict=False) model.eval() return model, tokenizer def generate(model, tokenizer, prompt, max_new_tokens=64, temperature=0.7, top_p=0.9, device="cuda" if torch.cuda.is_available() else "cpu"): # Formatowanie ChatML formatted = f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n" input_ids = tokenizer.encode(formatted, return_tensors="pt").to(device) eos_token_ids = {2, 32001} generated = input_ids.clone() with torch.no_grad(): for _ in range(max_new_tokens): context = generated[:, -512:] out = model(context) logits = out[0] if isinstance(out, tuple) else out logits = logits[:, -1, :] if temperature > 0: logits = logits / temperature if top_p < 1.0: sorted_logits, sorted_indices = torch.sort(logits, descending=True) cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1) sorted_indices_to_remove = cumulative_probs > top_p sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone() sorted_indices_to_remove[..., 0] = 0 indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove) logits = logits.masked_fill(indices_to_remove, -float("inf")) probs = F.softmax(logits, dim=-1) next_token = torch.multinomial(probs, num_samples=1) else: next_token = torch.argmax(logits, dim=-1, keepdim=True) if next_token.item() in eos_token_ids: break generated = torch.cat([generated, next_token], dim=1) output_text = tokenizer.decode(generated[0, input_ids.shape[1]:], skip_special_tokens=True) return output_text.strip() if __name__ == "__main__": import sys device = "cuda" if torch.cuda.is_available() else "cpu" print(f"Ładowanie modelu QKVE-61M na urządzeniu: {device}...") model, tokenizer = load_qkve_model(".", device=device) prompt = sys.argv[1] if len(sys.argv) > 1 else "Cześć! Kim jesteś i jak działasz?" print(f"\nPrompt: {prompt}") print("Odpowiedź:", generate(model, tokenizer, prompt, device=device))