import argparse from pathlib import Path import torch import torch.nn.functional as F from safetensors.torch import load_file from transformers import AutoTokenizer from src.config import load_config from src.model.moe import probe_grouped_mm from src.model.transformer import Transformer REPO_DIR = Path(__file__).resolve().parent def main(): parser = argparse.ArgumentParser() parser.add_argument("--config", default=str(REPO_DIR / "training_config.yaml")) parser.add_argument("--weights", default=str(REPO_DIR / "model.safetensors")) parser.add_argument("--tokenizer", default=str(REPO_DIR / "tokenizer")) parser.add_argument("--prompt", required=True) parser.add_argument("--max_new_tokens", type=int, default=100) parser.add_argument("--temperature", type=float, default=0.8) parser.add_argument("--top_k", type=int, default=50) args = parser.parse_args() cfg = load_config(args.config) if cfg.model.moe.backend == "auto": cfg.model.moe.backend = "grouped_mm" if probe_grouped_mm() else "fallback" print(f"[moe] backend resolved to: {cfg.model.moe.backend}") device = "cuda" if torch.cuda.is_available() else "cpu" tok = AutoTokenizer.from_pretrained(args.tokenizer) model = Transformer(cfg.model).to(device) state_dict = load_file(args.weights, device=device) model.load_state_dict(state_dict) model.eval() print(f"[weights] loaded {args.weights}") real_vocab_size = len(tok) ids = tok.encode(args.prompt, add_special_tokens=False) input_ids = torch.tensor([ids], dtype=torch.long, device=device) with torch.no_grad(): for _ in range(args.max_new_tokens): seq_len = input_ids.shape[1] if seq_len > cfg.model.max_seq_len: input_ids = input_ids[:, -cfg.model.max_seq_len :] autocast_dtype = torch.bfloat16 if device == "cuda" else torch.float32 with torch.autocast(device_type=device, dtype=autocast_dtype): logits, _, _, _ = model(input_ids) next_logits = logits[0, -1, :real_vocab_size] / args.temperature top_vals, top_idx = next_logits.topk(args.top_k) probs = F.softmax(top_vals.float(), dim=-1) next_id = top_idx[torch.multinomial(probs, 1)] input_ids = torch.cat([input_ids, next_id.view(1, 1)], dim=1) if next_id.item() == tok.eos_token_id: break text = tok.decode(input_ids[0].tolist()) print("=" * 40) print(text) print("=" * 40) if __name__ == "__main__": main()