NOEON / scripts /test_loading_from_hub.py
openhands
openhands
Import all source weights into NOEON + loading tests
c0dd269
Raw History Blame Contribute Delete
9.61 kB
#!/usr/bin/env python3
"""Real loading tests of the imported source weights, straight from the Hub.
This is a *loading* test, not a download test: it exercises the actual model
classes against the files now stored in ``Frankenstein-Labs/NOEON`` under
``weights/upstream/<slot>/``.
Tiers, chosen so the test is honest about what this CPU box can do:
1. **shard integrity** — open every safetensors shard by memory-mapping it and
parse its full header. Proves the shard is complete and its declared tensors
are readable, without needing the model's RAM.
2. **component load** — load tokenizer / processor / config / scheduler / VAE
weights with their real libraries from the imported files.
3. **full model** — ``from_pretrained`` of the whole model, only attempted when
the device has enough RAM; reported as ``skipped`` otherwise.
Usage:
HF_TOKEN=... python scripts/test_loading_from_hub.py
HF_TOKEN=... python scripts/test_loading_from_hub.py --full # force tier 3
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
REPO = "Frankenstein-Labs/NOEON"
def shard_integrity(slot: str, token: str) -> dict:
"""Memory-map every safetensors shard and read its header."""
from huggingface_hub import hf_hub_download
from safetensors import safe_open
man = json.loads((ROOT / "weights" / "source_files.json").read_text())
files = [f for f in man["sources"][slot]["files"]
if f["path"].endswith(".safetensors")]
out = {"shards": [], "n_tensors": 0, "ok": True}
for f in files:
rel = f"weights/upstream/{slot}/{f['path']}"
try:
local = hf_hub_download(REPO, rel, token=token)
with safe_open(local, framework="pt") as g:
keys = list(g.keys())
sample = keys[0]
t = g.get_slice(sample)
shape = t.get_shape()
out["shards"].append({"path": rel, "tensors": len(keys),
"sample": sample, "sample_shape": shape,
"ok": True})
out["n_tensors"] += len(keys)
except Exception as exc: # noqa: BLE001
out["shards"].append({"path": rel, "ok": False, "error": str(exc)})
out["ok"] = False
return out
def component_load(slot: str, token: str) -> dict:
from huggingface_hub import snapshot_download
res = {}
sub = snapshot_download(REPO, allow_patterns=[f"weights/upstream/{slot}/*",
f"weights/upstream/{slot}/**"],
token=token)
local = os.path.join(sub, "weights", "upstream", slot)
# tokenizer / processor
try:
from transformers import AutoTokenizer
tok = AutoTokenizer.from_pretrained(local, trust_remote_code=True)
ids = tok("hello world", return_tensors="pt")["input_ids"]
res["tokenizer"] = {"ok": True, "vocab": tok.vocab_size,
"sample": ids.tolist()}
except Exception as exc: # noqa: BLE001
res["tokenizer"] = {"ok": False, "error": f"{type(exc).__name__}: {exc}"}
# config
try:
from transformers import AutoConfig
cfg = AutoConfig.from_pretrained(local, trust_remote_code=True)
res["config"] = {"ok": True, "class": cfg.__class__.__name__,
"model_type": getattr(cfg, "model_type", None)}
except Exception as exc: # noqa: BLE001
res["config"] = {"ok": False, "error": f"{type(exc).__name__}: {exc}"}
return res
def vae_load(token: str) -> dict:
from huggingface_hub import snapshot_download
res = {}
sub = snapshot_download(REPO,
allow_patterns=["weights/upstream/generation/vae/*"],
token=token)
local = os.path.join(sub, "weights", "upstream", "generation", "vae")
# how many tensors the file holds vs. how many a stock AutoencoderKL uses
from safetensors import safe_open
from huggingface_hub import hf_hub_download
ckpt = hf_hub_download(
REPO, "weights/upstream/generation/vae/diffusion_pytorch_model.safetensors",
token=token)
with safe_open(ckpt, framework="pt") as g:
file_keys = set(g.keys())
try:
from diffusers import AutoencoderKL
import torch
vae = AutoencoderKL.from_pretrained(local, torch_dtype=torch.float32)
model_keys = set(vae.state_dict().keys())
used = file_keys & model_keys
x = torch.randn(1, 3, 64, 64)
with torch.no_grad():
lat = vae.encode(x).latent_dist.sample()
dec = vae.decode(lat).sample
res["vae"] = {"ok": True, "class": type(vae).__name__,
"file_tensors": len(file_keys),
"used_by_stock_class": len(used),
"latent_shape": list(lat.shape),
"decoded_shape": list(dec.shape),
"note": ("the checkpoint is a custom MageVAE; a stock "
"AutoencoderKL consumes only its standard "
"subset, so this proves load+forward plumbing "
"but not exact-architecture fidelity")}
except Exception as exc: # noqa: BLE001
res["vae"] = {"ok": False, "error": f"{type(exc).__name__}: {exc}"}
return res
def transformer_check(token: str) -> dict:
"""The MageFlow DiT is a custom class; verify its shard is complete."""
from huggingface_hub import hf_hub_download
from safetensors import safe_open
try:
ckpt = hf_hub_download(
REPO,
"weights/upstream/generation/transformer/diffusion_pytorch_model.safetensors",
token=token)
with safe_open(ckpt, framework="pt") as g:
keys = list(g.keys())
shp = g.get_slice(keys[0]).get_shape()
return {"ok": True, "tensors": len(keys), "sample": keys[0],
"sample_shape": shp, "note": "custom MageFlow class; header verified"}
except Exception as exc: # noqa: BLE001
return {"ok": False, "error": f"{type(exc).__name__}: {exc}"}
def full_model(slot: str, token: str, force: bool) -> dict:
import torch
from huggingface_hub import snapshot_download
if not force:
total = sum(f["size"] for f in
json.loads((ROOT / "weights" / "source_files.json")
.read_text())["sources"][slot]["files"]
if f["path"].endswith(".safetensors"))
# rough need: full fp32 model in RAM (~2x bf16 bytes) + headroom
need = total * 2 + 2_000_000_000
import shutil
avail = shutil.disk_usage("/").free
mem = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
if mem < need:
return {"ok": False, "skipped": True,
"reason": f"needs ~{need/1e9:.1f} GB RAM, box has {mem/1e9:.1f} GB"}
sub = snapshot_download(REPO,
allow_patterns=[f"weights/upstream/{slot}/**"],
token=token)
local = os.path.join(sub, "weights", "upstream", slot)
try:
from transformers import AutoModelForCausalLM
import torch
model = AutoModelForCausalLM.from_pretrained(
local, trust_remote_code=True, torch_dtype=torch.bfloat16,
low_cpu_mem_usage=True)
n = sum(p.numel() for p in model.parameters())
return {"ok": True, "class": type(model).__name__, "params": n}
except Exception as exc: # noqa: BLE001
return {"ok": False, "error": f"{type(exc).__name__}: {str(exc)[:300]}"}
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--full", action="store_true")
ap.add_argument("--only", nargs="*", default=None,
help="restrict to slots: vision computer_use generation")
ap.add_argument("--out", default="reports/loading_tests.json")
args = ap.parse_args()
token = os.environ.get("HF_TOKEN")
if not token:
print("HF_TOKEN required"); return 2
slots = args.only or ["vision", "computer_use"]
# load any prior report so partial re-runs keep earlier results
out = ROOT / args.out
report = json.loads(out.read_text()) if out.exists() else {}
for slot in slots:
if slot == "generation":
continue
print(f"\n=== {slot} ===")
rep = {}
rep["shard_integrity"] = shard_integrity(slot, token)
print(f" shards: {len(rep['shard_integrity']['shards'])} "
f"tensors={rep['shard_integrity']['n_tensors']} "
f"ok={rep['shard_integrity']['ok']}")
rep["components"] = component_load(slot, token)
for k, v in rep["components"].items():
print(f" {k}: {v.get('ok')} {v.get('error','')}")
rep["full_model"] = full_model(slot, token, args.full)
print(f" full_model: {rep['full_model']}")
report[slot] = rep
if "generation" in slots:
print("\n=== generation ===")
gen = {"vae": vae_load(token), "transformer": transformer_check(token)}
print(f" vae: {gen['vae']}")
print(f" transformer: {gen['transformer']}")
report["generation"] = gen
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(report, indent=2))
print(f"\nreport -> {out}")
return 0
if __name__ == "__main__":
raise SystemExit(main())