Download inference.py from Sticzu/QKVE-61M: direct link, hf CLI and curl.
- Browser
- Download file 3.53 kB
-
https://huggingface.co/Sticzu/QKVE-61M/resolve/main/inference.py
- Command line
-
hf download hf://Sticzu/QKVE-61M/inference.py
-
curl -L -o inference.py https://huggingface.co/Sticzu/QKVE-61M/resolve/main/inference.py
3.53 kB
| #!/usr/bin/env python3 | |
| """ | |
| Przykładowy skrypt inferencyjny dla modelu QKVE-61M. | |
| Wymaga zainstalowanego: torch, transformers, safetensors | |
| """ | |
| import os | |
| import torch | |
| import torch.nn.functional as F | |
| from transformers import AutoTokenizer | |
| from safetensors.torch import load_file | |
| from model import QKVE30Transformer | |
| def load_qkve_model(model_dir=".", device="cuda" if torch.cuda.is_available() else "cpu"): | |
| tokenizer = AutoTokenizer.from_pretrained(model_dir, clean_up_tokenization_spaces=False) | |
| model = QKVE30Transformer( | |
| vocab_size=len(tokenizer), | |
| d_model=384, | |
| num_layers=30, | |
| num_heads=12, | |
| num_kv_heads=4, | |
| base_ffn=512, | |
| max_seq_len=512 | |
| ).to(device) | |
| safetensors_path = os.path.join(model_dir, "model.safetensors") | |
| bin_path = os.path.join(model_dir, "pytorch_model.bin") | |
| if os.path.exists(safetensors_path): | |
| weights = load_file(safetensors_path) | |
| elif os.path.exists(bin_path): | |
| weights = torch.load(bin_path, map_location=device, weights_only=True) | |
| else: | |
| raise FileNotFoundError("Nie znaleziono pliku wag (model.safetensors ani pytorch_model.bin).") | |
| model.load_state_dict(weights, strict=False) | |
| model.eval() | |
| return model, tokenizer | |
| def generate(model, tokenizer, prompt, max_new_tokens=64, temperature=0.7, top_p=0.9, device="cuda" if torch.cuda.is_available() else "cpu"): | |
| # Formatowanie ChatML | |
| formatted = f"<|im_start|>user\n{prompt}<|im_end|>\n<|im_start|>assistant\n" | |
| input_ids = tokenizer.encode(formatted, return_tensors="pt").to(device) | |
| eos_token_ids = {2, 32001} | |
| generated = input_ids.clone() | |
| with torch.no_grad(): | |
| for _ in range(max_new_tokens): | |
| context = generated[:, -512:] | |
| out = model(context) | |
| logits = out[0] if isinstance(out, tuple) else out | |
| logits = logits[:, -1, :] | |
| if temperature > 0: | |
| logits = logits / temperature | |
| if top_p < 1.0: | |
| sorted_logits, sorted_indices = torch.sort(logits, descending=True) | |
| cumulative_probs = torch.cumsum(F.softmax(sorted_logits, dim=-1), dim=-1) | |
| sorted_indices_to_remove = cumulative_probs > top_p | |
| sorted_indices_to_remove[..., 1:] = sorted_indices_to_remove[..., :-1].clone() | |
| sorted_indices_to_remove[..., 0] = 0 | |
| indices_to_remove = sorted_indices_to_remove.scatter(1, sorted_indices, sorted_indices_to_remove) | |
| logits = logits.masked_fill(indices_to_remove, -float("inf")) | |
| probs = F.softmax(logits, dim=-1) | |
| next_token = torch.multinomial(probs, num_samples=1) | |
| else: | |
| next_token = torch.argmax(logits, dim=-1, keepdim=True) | |
| if next_token.item() in eos_token_ids: | |
| break | |
| generated = torch.cat([generated, next_token], dim=1) | |
| output_text = tokenizer.decode(generated[0, input_ids.shape[1]:], skip_special_tokens=True) | |
| return output_text.strip() | |
| if __name__ == "__main__": | |
| import sys | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| print(f"Ładowanie modelu QKVE-61M na urządzeniu: {device}...") | |
| model, tokenizer = load_qkve_model(".", device=device) | |
| prompt = sys.argv[1] if len(sys.argv) > 1 else "Cześć! Kim jesteś i jak działasz?" | |
| print(f"\nPrompt: {prompt}") | |
| print("Odpowiedź:", generate(model, tokenizer, prompt, device=device)) | |