File size: 9,605 Bytes
c0dd269 | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 149 150 151 152 153 154 155 156 157 158 159 160 161 162 163 164 165 166 167 168 169 170 171 172 173 174 175 176 177 178 179 180 181 182 183 184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219 220 221 222 223 224 225 226 227 228 | #!/usr/bin/env python3
"""Real loading tests of the imported source weights, straight from the Hub.
This is a *loading* test, not a download test: it exercises the actual model
classes against the files now stored in ``Frankenstein-Labs/NOEON`` under
``weights/upstream/<slot>/``.
Tiers, chosen so the test is honest about what this CPU box can do:
1. **shard integrity** — open every safetensors shard by memory-mapping it and
parse its full header. Proves the shard is complete and its declared tensors
are readable, without needing the model's RAM.
2. **component load** — load tokenizer / processor / config / scheduler / VAE
weights with their real libraries from the imported files.
3. **full model** — ``from_pretrained`` of the whole model, only attempted when
the device has enough RAM; reported as ``skipped`` otherwise.
Usage:
HF_TOKEN=... python scripts/test_loading_from_hub.py
HF_TOKEN=... python scripts/test_loading_from_hub.py --full # force tier 3
"""
from __future__ import annotations
import argparse
import json
import os
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
REPO = "Frankenstein-Labs/NOEON"
def shard_integrity(slot: str, token: str) -> dict:
"""Memory-map every safetensors shard and read its header."""
from huggingface_hub import hf_hub_download
from safetensors import safe_open
man = json.loads((ROOT / "weights" / "source_files.json").read_text())
files = [f for f in man["sources"][slot]["files"]
if f["path"].endswith(".safetensors")]
out = {"shards": [], "n_tensors": 0, "ok": True}
for f in files:
rel = f"weights/upstream/{slot}/{f['path']}"
try:
local = hf_hub_download(REPO, rel, token=token)
with safe_open(local, framework="pt") as g:
keys = list(g.keys())
sample = keys[0]
t = g.get_slice(sample)
shape = t.get_shape()
out["shards"].append({"path": rel, "tensors": len(keys),
"sample": sample, "sample_shape": shape,
"ok": True})
out["n_tensors"] += len(keys)
except Exception as exc: # noqa: BLE001
out["shards"].append({"path": rel, "ok": False, "error": str(exc)})
out["ok"] = False
return out
def component_load(slot: str, token: str) -> dict:
from huggingface_hub import snapshot_download
res = {}
sub = snapshot_download(REPO, allow_patterns=[f"weights/upstream/{slot}/*",
f"weights/upstream/{slot}/**"],
token=token)
local = os.path.join(sub, "weights", "upstream", slot)
# tokenizer / processor
try:
from transformers import AutoTokenizer
tok = AutoTokenizer.from_pretrained(local, trust_remote_code=True)
ids = tok("hello world", return_tensors="pt")["input_ids"]
res["tokenizer"] = {"ok": True, "vocab": tok.vocab_size,
"sample": ids.tolist()}
except Exception as exc: # noqa: BLE001
res["tokenizer"] = {"ok": False, "error": f"{type(exc).__name__}: {exc}"}
# config
try:
from transformers import AutoConfig
cfg = AutoConfig.from_pretrained(local, trust_remote_code=True)
res["config"] = {"ok": True, "class": cfg.__class__.__name__,
"model_type": getattr(cfg, "model_type", None)}
except Exception as exc: # noqa: BLE001
res["config"] = {"ok": False, "error": f"{type(exc).__name__}: {exc}"}
return res
def vae_load(token: str) -> dict:
from huggingface_hub import snapshot_download
res = {}
sub = snapshot_download(REPO,
allow_patterns=["weights/upstream/generation/vae/*"],
token=token)
local = os.path.join(sub, "weights", "upstream", "generation", "vae")
# how many tensors the file holds vs. how many a stock AutoencoderKL uses
from safetensors import safe_open
from huggingface_hub import hf_hub_download
ckpt = hf_hub_download(
REPO, "weights/upstream/generation/vae/diffusion_pytorch_model.safetensors",
token=token)
with safe_open(ckpt, framework="pt") as g:
file_keys = set(g.keys())
try:
from diffusers import AutoencoderKL
import torch
vae = AutoencoderKL.from_pretrained(local, torch_dtype=torch.float32)
model_keys = set(vae.state_dict().keys())
used = file_keys & model_keys
x = torch.randn(1, 3, 64, 64)
with torch.no_grad():
lat = vae.encode(x).latent_dist.sample()
dec = vae.decode(lat).sample
res["vae"] = {"ok": True, "class": type(vae).__name__,
"file_tensors": len(file_keys),
"used_by_stock_class": len(used),
"latent_shape": list(lat.shape),
"decoded_shape": list(dec.shape),
"note": ("the checkpoint is a custom MageVAE; a stock "
"AutoencoderKL consumes only its standard "
"subset, so this proves load+forward plumbing "
"but not exact-architecture fidelity")}
except Exception as exc: # noqa: BLE001
res["vae"] = {"ok": False, "error": f"{type(exc).__name__}: {exc}"}
return res
def transformer_check(token: str) -> dict:
"""The MageFlow DiT is a custom class; verify its shard is complete."""
from huggingface_hub import hf_hub_download
from safetensors import safe_open
try:
ckpt = hf_hub_download(
REPO,
"weights/upstream/generation/transformer/diffusion_pytorch_model.safetensors",
token=token)
with safe_open(ckpt, framework="pt") as g:
keys = list(g.keys())
shp = g.get_slice(keys[0]).get_shape()
return {"ok": True, "tensors": len(keys), "sample": keys[0],
"sample_shape": shp, "note": "custom MageFlow class; header verified"}
except Exception as exc: # noqa: BLE001
return {"ok": False, "error": f"{type(exc).__name__}: {exc}"}
def full_model(slot: str, token: str, force: bool) -> dict:
import torch
from huggingface_hub import snapshot_download
if not force:
total = sum(f["size"] for f in
json.loads((ROOT / "weights" / "source_files.json")
.read_text())["sources"][slot]["files"]
if f["path"].endswith(".safetensors"))
# rough need: full fp32 model in RAM (~2x bf16 bytes) + headroom
need = total * 2 + 2_000_000_000
import shutil
avail = shutil.disk_usage("/").free
mem = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES")
if mem < need:
return {"ok": False, "skipped": True,
"reason": f"needs ~{need/1e9:.1f} GB RAM, box has {mem/1e9:.1f} GB"}
sub = snapshot_download(REPO,
allow_patterns=[f"weights/upstream/{slot}/**"],
token=token)
local = os.path.join(sub, "weights", "upstream", slot)
try:
from transformers import AutoModelForCausalLM
import torch
model = AutoModelForCausalLM.from_pretrained(
local, trust_remote_code=True, torch_dtype=torch.bfloat16,
low_cpu_mem_usage=True)
n = sum(p.numel() for p in model.parameters())
return {"ok": True, "class": type(model).__name__, "params": n}
except Exception as exc: # noqa: BLE001
return {"ok": False, "error": f"{type(exc).__name__}: {str(exc)[:300]}"}
def main() -> int:
ap = argparse.ArgumentParser()
ap.add_argument("--full", action="store_true")
ap.add_argument("--only", nargs="*", default=None,
help="restrict to slots: vision computer_use generation")
ap.add_argument("--out", default="reports/loading_tests.json")
args = ap.parse_args()
token = os.environ.get("HF_TOKEN")
if not token:
print("HF_TOKEN required"); return 2
slots = args.only or ["vision", "computer_use"]
# load any prior report so partial re-runs keep earlier results
out = ROOT / args.out
report = json.loads(out.read_text()) if out.exists() else {}
for slot in slots:
if slot == "generation":
continue
print(f"\n=== {slot} ===")
rep = {}
rep["shard_integrity"] = shard_integrity(slot, token)
print(f" shards: {len(rep['shard_integrity']['shards'])} "
f"tensors={rep['shard_integrity']['n_tensors']} "
f"ok={rep['shard_integrity']['ok']}")
rep["components"] = component_load(slot, token)
for k, v in rep["components"].items():
print(f" {k}: {v.get('ok')} {v.get('error','')}")
rep["full_model"] = full_model(slot, token, args.full)
print(f" full_model: {rep['full_model']}")
report[slot] = rep
if "generation" in slots:
print("\n=== generation ===")
gen = {"vae": vae_load(token), "transformer": transformer_check(token)}
print(f" vae: {gen['vae']}")
print(f" transformer: {gen['transformer']}")
report["generation"] = gen
out.parent.mkdir(parents=True, exist_ok=True)
out.write_text(json.dumps(report, indent=2))
print(f"\nreport -> {out}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
|