Phase-Technologies commited on
Commit
b2b500d
·
verified ·
1 Parent(s): fa16f53

Add plug-and-play run_inference.py script

Browse files
Files changed (1) hide show
  1. run_inference.py +113 -0
run_inference.py ADDED
@@ -0,0 +1,113 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ #!/usr/bin/env python3
2
+ import os
3
+ import argparse
4
+ import logging
5
+ import torch
6
+ from pathlib import Path
7
+
8
+ # Set up logging
9
+ logging.basicConfig(level=logging.INFO, format="[%(asctime)s][%(levelname)s] %(message)s")
10
+ logger = logging.getLogger(__name__)
11
+
12
+ def get_args():
13
+ parser = argparse.ArgumentParser(description="Run inference on FORGE-500M using a trained model checkpoint")
14
+ parser.add_argument("--model_path", type=str, default=".",
15
+ help="Path to the trained model directory containing config, tokenizer, and weights (defaults to current directory)")
16
+ parser.add_argument("--prompt", type=str, default="The future of artificial intelligence is",
17
+ help="Prompt to generate text from")
18
+ parser.add_argument("--max_new_tokens", type=int, default=128,
19
+ help="Max number of tokens to generate")
20
+ parser.add_argument("--temperature", type=float, default=0.7,
21
+ help="Sampling temperature for text generation")
22
+ parser.add_argument("--top_k", type=int, default=50,
23
+ help="Top-k sampling parameter")
24
+ parser.add_argument("--top_p", type=float, default=0.9,
25
+ help="Top-p sampling parameter")
26
+ parser.add_argument("--repetition_penalty", type=float, default=1.1,
27
+ help="Repetition penalty parameter")
28
+ return parser.parse_args()
29
+
30
+ def main():
31
+ args = get_args()
32
+ device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
33
+ logger.info(f"Using device: {device}")
34
+
35
+ model_dir = Path(args.model_path)
36
+ if not model_dir.exists():
37
+ raise FileNotFoundError(f"Model path does not exist: {model_dir}")
38
+
39
+ # 1. Load Tokenizer
40
+ from tokenizer.crayon_wrapper import ForgeTokenizer
41
+ tok_dir = model_dir / "tokenizer"
42
+ if tok_dir.exists():
43
+ logger.info(f"Loading tokenizer from {tok_dir}...")
44
+ tokenizer = ForgeTokenizer.from_pretrained(str(tok_dir))
45
+ else:
46
+ logger.info(f"Loading tokenizer from model directory root {model_dir}...")
47
+ tokenizer = ForgeTokenizer.from_pretrained(str(model_dir))
48
+
49
+ # 2. Load Model Config
50
+ from config import ForgeModelConfig
51
+ config_path = model_dir / "model_config.json"
52
+ if not config_path.exists():
53
+ config_path = model_dir / "config.json"
54
+ if not config_path.exists():
55
+ raise FileNotFoundError(f"Could not find model_config.json or config.json in {model_dir}")
56
+
57
+ logger.info(f"Loading config from {config_path}")
58
+ model_config = ForgeModelConfig.from_json(str(config_path))
59
+ model_config.vocab_size = tokenizer.vocab_size
60
+
61
+ # 3. Build Model
62
+ from model.forge_model import build_forge_500m
63
+ logger.info("Building FORGE model...")
64
+ model = build_forge_500m(model_config)
65
+
66
+ # 4. Load Model Weights
67
+ loaded_weights = False
68
+ for weight_filename in ("model_bf16.pt", "model.pt", "pytorch_model.bin"):
69
+ weight_path = model_dir / weight_filename
70
+ if weight_path.exists():
71
+ logger.info(f"Loading model weights from {weight_path}...")
72
+ state_dict = torch.load(str(weight_path), map_location="cpu")
73
+ # Convert values to bfloat16
74
+ state_dict = {k: v.to(torch.bfloat16) for k, v in state_dict.items()}
75
+ model.load_state_dict(state_dict, strict=False)
76
+ del state_dict
77
+ torch.cuda.empty_cache()
78
+ loaded_weights = True
79
+ break
80
+
81
+ if not loaded_weights:
82
+ raise FileNotFoundError(
83
+ f"No model weights found in {model_dir}. "
84
+ "Expected one of: model_bf16.pt, model.pt, pytorch_model.bin"
85
+ )
86
+
87
+ model = model.to(device).to(torch.bfloat16)
88
+ model.eval()
89
+
90
+ # 5. Encode prompt
91
+ logger.info(f"Prompt: {args.prompt!r}")
92
+ input_ids = torch.tensor([tokenizer.encode(args.prompt, add_bos=True, add_eos=False)], dtype=torch.long, device=device)
93
+
94
+ # 6. Generate
95
+ logger.info("Generating tokens...")
96
+ with torch.inference_mode():
97
+ output_ids = model.generate(
98
+ input_ids=input_ids,
99
+ max_new_tokens=args.max_new_tokens,
100
+ temperature=args.temperature,
101
+ top_k=args.top_k,
102
+ top_p=args.top_p,
103
+ repetition_penalty=args.repetition_penalty,
104
+ eos_token_id=tokenizer.special_tokens["<|eos|>"],
105
+ pad_token_id=tokenizer.special_tokens["<|pad|>"]
106
+ )
107
+
108
+ # 7. Decode output
109
+ generated_text = tokenizer.decode(output_ids[0].tolist(), skip_special_tokens=False)
110
+ logger.info(f"Generated text:\n{generated_text}")
111
+
112
+ if __name__ == "__main__":
113
+ main()