TheFireHacker's picture
Upload generate.py with huggingface_hub
6ec578b verified
Raw History Blame Contribute Delete
2.59 kB
import argparse
from pathlib import Path
import torch
import torch.nn.functional as F
from safetensors.torch import load_file
from transformers import AutoTokenizer
from src.config import load_config
from src.model.moe import probe_grouped_mm
from src.model.transformer import Transformer
REPO_DIR = Path(__file__).resolve().parent
def main():
parser = argparse.ArgumentParser()
parser.add_argument("--config", default=str(REPO_DIR / "training_config.yaml"))
parser.add_argument("--weights", default=str(REPO_DIR / "model.safetensors"))
parser.add_argument("--tokenizer", default=str(REPO_DIR / "tokenizer"))
parser.add_argument("--prompt", required=True)
parser.add_argument("--max_new_tokens", type=int, default=100)
parser.add_argument("--temperature", type=float, default=0.8)
parser.add_argument("--top_k", type=int, default=50)
args = parser.parse_args()
cfg = load_config(args.config)
if cfg.model.moe.backend == "auto":
cfg.model.moe.backend = "grouped_mm" if probe_grouped_mm() else "fallback"
print(f"[moe] backend resolved to: {cfg.model.moe.backend}")
device = "cuda" if torch.cuda.is_available() else "cpu"
tok = AutoTokenizer.from_pretrained(args.tokenizer)
model = Transformer(cfg.model).to(device)
state_dict = load_file(args.weights, device=device)
model.load_state_dict(state_dict)
model.eval()
print(f"[weights] loaded {args.weights}")
real_vocab_size = len(tok)
ids = tok.encode(args.prompt, add_special_tokens=False)
input_ids = torch.tensor([ids], dtype=torch.long, device=device)
with torch.no_grad():
for _ in range(args.max_new_tokens):
seq_len = input_ids.shape[1]
if seq_len > cfg.model.max_seq_len:
input_ids = input_ids[:, -cfg.model.max_seq_len :]
autocast_dtype = torch.bfloat16 if device == "cuda" else torch.float32
with torch.autocast(device_type=device, dtype=autocast_dtype):
logits, _, _, _ = model(input_ids)
next_logits = logits[0, -1, :real_vocab_size] / args.temperature
top_vals, top_idx = next_logits.topk(args.top_k)
probs = F.softmax(top_vals.float(), dim=-1)
next_id = top_idx[torch.multinomial(probs, 1)]
input_ids = torch.cat([input_ids, next_id.view(1, 1)], dim=1)
if next_id.item() == tok.eos_token_id:
break
text = tok.decode(input_ids[0].tolist())
print("=" * 40)
print(text)
print("=" * 40)
if __name__ == "__main__":
main()