Download run_inference.py from DeepMachineLab/Lulu750M-Instruct-LOCALNOQANT: direct link, hf CLI and curl.
- Browser
- Download file 1.69 kB
-
https://huggingface.co/DeepMachineLab/Lulu750M-Instruct-LOCALNOQANT/resolve/main/run_inference.py
- Command line
-
hf download hf://DeepMachineLab/Lulu750M-Instruct-LOCALNOQANT/run_inference.py
-
curl -L -o run_inference.py https://huggingface.co/DeepMachineLab/Lulu750M-Instruct-LOCALNOQANT/resolve/main/run_inference.py
1.69 kB
| #!/usr/bin/env python3 | |
| """Small CLI launcher for LULUV2 native-bf16 local inference.""" | |
| from __future__ import annotations | |
| import argparse | |
| import torch | |
| from luluv2_live_inference import LULUV2LiveEngine, GenerationConfig | |
| def main(): | |
| p = argparse.ArgumentParser("LULUV2 local inference") | |
| p.add_argument("--ckpt", default="LULUV2-bf16.pt", help="Path to the native-bf16 checkpoint file") | |
| p.add_argument("--tokenizer-dir", default="tokenizer", help="Local tokenizer directory") | |
| p.add_argument("--prompt", required=True, help="User prompt") | |
| p.add_argument("--system", default="You are LuluV2, a helpful local AI assistant.") | |
| p.add_argument("--max-new-tokens", type=int, default=512) | |
| p.add_argument("--temperature", type=float, default=0.65) | |
| p.add_argument("--top-p", type=float, default=0.90) | |
| p.add_argument("--top-k", type=int, default=40) | |
| p.add_argument("--device", default=None) | |
| p.add_argument("--dtype", default="bf16", choices=["bf16", "fp16", "fp32"]) | |
| args = p.parse_args() | |
| engine = LULUV2LiveEngine( | |
| ckpt_path=args.ckpt, | |
| model_py="luluv2_inference_runtime.py", | |
| tokenizer_dir=args.tokenizer_dir, | |
| device=args.device, | |
| dtype=args.dtype, | |
| local_files_only=True, | |
| no_config_download=True, | |
| ) | |
| cfg = GenerationConfig( | |
| max_new_tokens=args.max_new_tokens, | |
| temperature=args.temperature, | |
| top_p=args.top_p, | |
| top_k=args.top_k, | |
| ) | |
| history = [] | |
| for text in engine.generate_stream(args.prompt, history, args.system, cfg): | |
| print(text, end="", flush=True) | |
| print() | |
| if __name__ == "__main__": | |
| torch.set_grad_enabled(False) | |
| main() | |