NOEON / scripts /plan_inference.py
openhands
openhands
Add selective multimodal-inference manifest, downloader and VRAM planner
bb83d2a
Raw History Blame Contribute Delete
1.3 kB
#!/usr/bin/env python3
"""Plan which NOEON download profile fits this machine's GPU memory.
Reads ``weights/inference_manifest.json`` and reports, per profile, the exact
weight bytes, the estimated bf16 VRAM need and an honest verdict. It never
assumes a GPU is present: on a CPU-only box every profile reports
``cpu-only (no CUDA GPU detected)`` instead of a false "fits".
Usage:
python scripts/plan_inference.py
"""
from __future__ import annotations
import json
import sys
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
from noeon.core.resources import detect, plan_profiles # noqa: E402
def main() -> int:
env = detect()
print("environment:")
print(json.dumps(env, indent=2))
plan = plan_profiles(env)
if "error" in plan:
print("error:", plan["error"])
return 1
print("\ninference plan (bf16 estimates):")
for name, v in plan.items():
print(f" {name:20s} {v['n_files']:3d} files "
f"{v['weights_gb']:6.2f} GB weights "
f"~{v['est_vram_gb_bf16']:5.1f} GB VRAM -> {v['verdict']}")
print("\nnote: VRAM figures are estimates (weights x 1.25 for activations), "
"not guarantees.")
return 0
if __name__ == "__main__":
raise SystemExit(main())