Add plug-and-play run_inference.py script
Browse files- run_inference.py +113 -0
run_inference.py
ADDED
|
@@ -0,0 +1,113 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
| 1 |
+
#!/usr/bin/env python3
|
| 2 |
+
import os
|
| 3 |
+
import argparse
|
| 4 |
+
import logging
|
| 5 |
+
import torch
|
| 6 |
+
from pathlib import Path
|
| 7 |
+
|
| 8 |
+
# Set up logging
|
| 9 |
+
logging.basicConfig(level=logging.INFO, format="[%(asctime)s][%(levelname)s] %(message)s")
|
| 10 |
+
logger = logging.getLogger(__name__)
|
| 11 |
+
|
| 12 |
+
def get_args():
|
| 13 |
+
parser = argparse.ArgumentParser(description="Run inference on FORGE-500M using a trained model checkpoint")
|
| 14 |
+
parser.add_argument("--model_path", type=str, default=".",
|
| 15 |
+
help="Path to the trained model directory containing config, tokenizer, and weights (defaults to current directory)")
|
| 16 |
+
parser.add_argument("--prompt", type=str, default="The future of artificial intelligence is",
|
| 17 |
+
help="Prompt to generate text from")
|
| 18 |
+
parser.add_argument("--max_new_tokens", type=int, default=128,
|
| 19 |
+
help="Max number of tokens to generate")
|
| 20 |
+
parser.add_argument("--temperature", type=float, default=0.7,
|
| 21 |
+
help="Sampling temperature for text generation")
|
| 22 |
+
parser.add_argument("--top_k", type=int, default=50,
|
| 23 |
+
help="Top-k sampling parameter")
|
| 24 |
+
parser.add_argument("--top_p", type=float, default=0.9,
|
| 25 |
+
help="Top-p sampling parameter")
|
| 26 |
+
parser.add_argument("--repetition_penalty", type=float, default=1.1,
|
| 27 |
+
help="Repetition penalty parameter")
|
| 28 |
+
return parser.parse_args()
|
| 29 |
+
|
| 30 |
+
def main():
|
| 31 |
+
args = get_args()
|
| 32 |
+
device = torch.device("cuda" if torch.cuda.is_available() else "cpu")
|
| 33 |
+
logger.info(f"Using device: {device}")
|
| 34 |
+
|
| 35 |
+
model_dir = Path(args.model_path)
|
| 36 |
+
if not model_dir.exists():
|
| 37 |
+
raise FileNotFoundError(f"Model path does not exist: {model_dir}")
|
| 38 |
+
|
| 39 |
+
# 1. Load Tokenizer
|
| 40 |
+
from tokenizer.crayon_wrapper import ForgeTokenizer
|
| 41 |
+
tok_dir = model_dir / "tokenizer"
|
| 42 |
+
if tok_dir.exists():
|
| 43 |
+
logger.info(f"Loading tokenizer from {tok_dir}...")
|
| 44 |
+
tokenizer = ForgeTokenizer.from_pretrained(str(tok_dir))
|
| 45 |
+
else:
|
| 46 |
+
logger.info(f"Loading tokenizer from model directory root {model_dir}...")
|
| 47 |
+
tokenizer = ForgeTokenizer.from_pretrained(str(model_dir))
|
| 48 |
+
|
| 49 |
+
# 2. Load Model Config
|
| 50 |
+
from config import ForgeModelConfig
|
| 51 |
+
config_path = model_dir / "model_config.json"
|
| 52 |
+
if not config_path.exists():
|
| 53 |
+
config_path = model_dir / "config.json"
|
| 54 |
+
if not config_path.exists():
|
| 55 |
+
raise FileNotFoundError(f"Could not find model_config.json or config.json in {model_dir}")
|
| 56 |
+
|
| 57 |
+
logger.info(f"Loading config from {config_path}")
|
| 58 |
+
model_config = ForgeModelConfig.from_json(str(config_path))
|
| 59 |
+
model_config.vocab_size = tokenizer.vocab_size
|
| 60 |
+
|
| 61 |
+
# 3. Build Model
|
| 62 |
+
from model.forge_model import build_forge_500m
|
| 63 |
+
logger.info("Building FORGE model...")
|
| 64 |
+
model = build_forge_500m(model_config)
|
| 65 |
+
|
| 66 |
+
# 4. Load Model Weights
|
| 67 |
+
loaded_weights = False
|
| 68 |
+
for weight_filename in ("model_bf16.pt", "model.pt", "pytorch_model.bin"):
|
| 69 |
+
weight_path = model_dir / weight_filename
|
| 70 |
+
if weight_path.exists():
|
| 71 |
+
logger.info(f"Loading model weights from {weight_path}...")
|
| 72 |
+
state_dict = torch.load(str(weight_path), map_location="cpu")
|
| 73 |
+
# Convert values to bfloat16
|
| 74 |
+
state_dict = {k: v.to(torch.bfloat16) for k, v in state_dict.items()}
|
| 75 |
+
model.load_state_dict(state_dict, strict=False)
|
| 76 |
+
del state_dict
|
| 77 |
+
torch.cuda.empty_cache()
|
| 78 |
+
loaded_weights = True
|
| 79 |
+
break
|
| 80 |
+
|
| 81 |
+
if not loaded_weights:
|
| 82 |
+
raise FileNotFoundError(
|
| 83 |
+
f"No model weights found in {model_dir}. "
|
| 84 |
+
"Expected one of: model_bf16.pt, model.pt, pytorch_model.bin"
|
| 85 |
+
)
|
| 86 |
+
|
| 87 |
+
model = model.to(device).to(torch.bfloat16)
|
| 88 |
+
model.eval()
|
| 89 |
+
|
| 90 |
+
# 5. Encode prompt
|
| 91 |
+
logger.info(f"Prompt: {args.prompt!r}")
|
| 92 |
+
input_ids = torch.tensor([tokenizer.encode(args.prompt, add_bos=True, add_eos=False)], dtype=torch.long, device=device)
|
| 93 |
+
|
| 94 |
+
# 6. Generate
|
| 95 |
+
logger.info("Generating tokens...")
|
| 96 |
+
with torch.inference_mode():
|
| 97 |
+
output_ids = model.generate(
|
| 98 |
+
input_ids=input_ids,
|
| 99 |
+
max_new_tokens=args.max_new_tokens,
|
| 100 |
+
temperature=args.temperature,
|
| 101 |
+
top_k=args.top_k,
|
| 102 |
+
top_p=args.top_p,
|
| 103 |
+
repetition_penalty=args.repetition_penalty,
|
| 104 |
+
eos_token_id=tokenizer.special_tokens["<|eos|>"],
|
| 105 |
+
pad_token_id=tokenizer.special_tokens["<|pad|>"]
|
| 106 |
+
)
|
| 107 |
+
|
| 108 |
+
# 7. Decode output
|
| 109 |
+
generated_text = tokenizer.decode(output_ids[0].tolist(), skip_special_tokens=False)
|
| 110 |
+
logger.info(f"Generated text:\n{generated_text}")
|
| 111 |
+
|
| 112 |
+
if __name__ == "__main__":
|
| 113 |
+
main()
|