Download generate.py from TheFireHacker/bubblspace-moe0.9b-a0.2b: direct link, hf CLI and curl.
- Browser
- Download file 2.59 kB
-
https://huggingface.co/TheFireHacker/bubblspace-moe0.9b-a0.2b/resolve/main/generate.py
- Command line
-
hf download hf://TheFireHacker/bubblspace-moe0.9b-a0.2b/generate.py
-
curl -L -o generate.py https://huggingface.co/TheFireHacker/bubblspace-moe0.9b-a0.2b/resolve/main/generate.py
2.59 kB
| import argparse | |
| from pathlib import Path | |
| import torch | |
| import torch.nn.functional as F | |
| from safetensors.torch import load_file | |
| from transformers import AutoTokenizer | |
| from src.config import load_config | |
| from src.model.moe import probe_grouped_mm | |
| from src.model.transformer import Transformer | |
| REPO_DIR = Path(__file__).resolve().parent | |
| def main(): | |
| parser = argparse.ArgumentParser() | |
| parser.add_argument("--config", default=str(REPO_DIR / "training_config.yaml")) | |
| parser.add_argument("--weights", default=str(REPO_DIR / "model.safetensors")) | |
| parser.add_argument("--tokenizer", default=str(REPO_DIR / "tokenizer")) | |
| parser.add_argument("--prompt", required=True) | |
| parser.add_argument("--max_new_tokens", type=int, default=100) | |
| parser.add_argument("--temperature", type=float, default=0.8) | |
| parser.add_argument("--top_k", type=int, default=50) | |
| args = parser.parse_args() | |
| cfg = load_config(args.config) | |
| if cfg.model.moe.backend == "auto": | |
| cfg.model.moe.backend = "grouped_mm" if probe_grouped_mm() else "fallback" | |
| print(f"[moe] backend resolved to: {cfg.model.moe.backend}") | |
| device = "cuda" if torch.cuda.is_available() else "cpu" | |
| tok = AutoTokenizer.from_pretrained(args.tokenizer) | |
| model = Transformer(cfg.model).to(device) | |
| state_dict = load_file(args.weights, device=device) | |
| model.load_state_dict(state_dict) | |
| model.eval() | |
| print(f"[weights] loaded {args.weights}") | |
| real_vocab_size = len(tok) | |
| ids = tok.encode(args.prompt, add_special_tokens=False) | |
| input_ids = torch.tensor([ids], dtype=torch.long, device=device) | |
| with torch.no_grad(): | |
| for _ in range(args.max_new_tokens): | |
| seq_len = input_ids.shape[1] | |
| if seq_len > cfg.model.max_seq_len: | |
| input_ids = input_ids[:, -cfg.model.max_seq_len :] | |
| autocast_dtype = torch.bfloat16 if device == "cuda" else torch.float32 | |
| with torch.autocast(device_type=device, dtype=autocast_dtype): | |
| logits, _, _, _ = model(input_ids) | |
| next_logits = logits[0, -1, :real_vocab_size] / args.temperature | |
| top_vals, top_idx = next_logits.topk(args.top_k) | |
| probs = F.softmax(top_vals.float(), dim=-1) | |
| next_id = top_idx[torch.multinomial(probs, 1)] | |
| input_ids = torch.cat([input_ids, next_id.view(1, 1)], dim=1) | |
| if next_id.item() == tok.eos_token_id: | |
| break | |
| text = tok.decode(input_ids[0].tolist()) | |
| print("=" * 40) | |
| print(text) | |
| print("=" * 40) | |
| if __name__ == "__main__": | |
| main() | |