#!/usr/bin/env python3 """Real loading tests of the imported source weights, straight from the Hub. This is a *loading* test, not a download test: it exercises the actual model classes against the files now stored in ``Frankenstein-Labs/NOEON`` under ``weights/upstream//``. Tiers, chosen so the test is honest about what this CPU box can do: 1. **shard integrity** — open every safetensors shard by memory-mapping it and parse its full header. Proves the shard is complete and its declared tensors are readable, without needing the model's RAM. 2. **component load** — load tokenizer / processor / config / scheduler / VAE weights with their real libraries from the imported files. 3. **full model** — ``from_pretrained`` of the whole model, only attempted when the device has enough RAM; reported as ``skipped`` otherwise. Usage: HF_TOKEN=... python scripts/test_loading_from_hub.py HF_TOKEN=... python scripts/test_loading_from_hub.py --full # force tier 3 """ from __future__ import annotations import argparse import json import os import sys from pathlib import Path ROOT = Path(__file__).resolve().parent.parent REPO = "Frankenstein-Labs/NOEON" def shard_integrity(slot: str, token: str) -> dict: """Memory-map every safetensors shard and read its header.""" from huggingface_hub import hf_hub_download from safetensors import safe_open man = json.loads((ROOT / "weights" / "source_files.json").read_text()) files = [f for f in man["sources"][slot]["files"] if f["path"].endswith(".safetensors")] out = {"shards": [], "n_tensors": 0, "ok": True} for f in files: rel = f"weights/upstream/{slot}/{f['path']}" try: local = hf_hub_download(REPO, rel, token=token) with safe_open(local, framework="pt") as g: keys = list(g.keys()) sample = keys[0] t = g.get_slice(sample) shape = t.get_shape() out["shards"].append({"path": rel, "tensors": len(keys), "sample": sample, "sample_shape": shape, "ok": True}) out["n_tensors"] += len(keys) except Exception as exc: # noqa: BLE001 out["shards"].append({"path": rel, "ok": False, "error": str(exc)}) out["ok"] = False return out def component_load(slot: str, token: str) -> dict: from huggingface_hub import snapshot_download res = {} sub = snapshot_download(REPO, allow_patterns=[f"weights/upstream/{slot}/*", f"weights/upstream/{slot}/**"], token=token) local = os.path.join(sub, "weights", "upstream", slot) # tokenizer / processor try: from transformers import AutoTokenizer tok = AutoTokenizer.from_pretrained(local, trust_remote_code=True) ids = tok("hello world", return_tensors="pt")["input_ids"] res["tokenizer"] = {"ok": True, "vocab": tok.vocab_size, "sample": ids.tolist()} except Exception as exc: # noqa: BLE001 res["tokenizer"] = {"ok": False, "error": f"{type(exc).__name__}: {exc}"} # config try: from transformers import AutoConfig cfg = AutoConfig.from_pretrained(local, trust_remote_code=True) res["config"] = {"ok": True, "class": cfg.__class__.__name__, "model_type": getattr(cfg, "model_type", None)} except Exception as exc: # noqa: BLE001 res["config"] = {"ok": False, "error": f"{type(exc).__name__}: {exc}"} return res def vae_load(token: str) -> dict: from huggingface_hub import snapshot_download res = {} sub = snapshot_download(REPO, allow_patterns=["weights/upstream/generation/vae/*"], token=token) local = os.path.join(sub, "weights", "upstream", "generation", "vae") # how many tensors the file holds vs. how many a stock AutoencoderKL uses from safetensors import safe_open from huggingface_hub import hf_hub_download ckpt = hf_hub_download( REPO, "weights/upstream/generation/vae/diffusion_pytorch_model.safetensors", token=token) with safe_open(ckpt, framework="pt") as g: file_keys = set(g.keys()) try: from diffusers import AutoencoderKL import torch vae = AutoencoderKL.from_pretrained(local, torch_dtype=torch.float32) model_keys = set(vae.state_dict().keys()) used = file_keys & model_keys x = torch.randn(1, 3, 64, 64) with torch.no_grad(): lat = vae.encode(x).latent_dist.sample() dec = vae.decode(lat).sample res["vae"] = {"ok": True, "class": type(vae).__name__, "file_tensors": len(file_keys), "used_by_stock_class": len(used), "latent_shape": list(lat.shape), "decoded_shape": list(dec.shape), "note": ("the checkpoint is a custom MageVAE; a stock " "AutoencoderKL consumes only its standard " "subset, so this proves load+forward plumbing " "but not exact-architecture fidelity")} except Exception as exc: # noqa: BLE001 res["vae"] = {"ok": False, "error": f"{type(exc).__name__}: {exc}"} return res def transformer_check(token: str) -> dict: """The MageFlow DiT is a custom class; verify its shard is complete.""" from huggingface_hub import hf_hub_download from safetensors import safe_open try: ckpt = hf_hub_download( REPO, "weights/upstream/generation/transformer/diffusion_pytorch_model.safetensors", token=token) with safe_open(ckpt, framework="pt") as g: keys = list(g.keys()) shp = g.get_slice(keys[0]).get_shape() return {"ok": True, "tensors": len(keys), "sample": keys[0], "sample_shape": shp, "note": "custom MageFlow class; header verified"} except Exception as exc: # noqa: BLE001 return {"ok": False, "error": f"{type(exc).__name__}: {exc}"} def full_model(slot: str, token: str, force: bool) -> dict: import torch from huggingface_hub import snapshot_download if not force: total = sum(f["size"] for f in json.loads((ROOT / "weights" / "source_files.json") .read_text())["sources"][slot]["files"] if f["path"].endswith(".safetensors")) # rough need: full fp32 model in RAM (~2x bf16 bytes) + headroom need = total * 2 + 2_000_000_000 import shutil avail = shutil.disk_usage("/").free mem = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") if mem < need: return {"ok": False, "skipped": True, "reason": f"needs ~{need/1e9:.1f} GB RAM, box has {mem/1e9:.1f} GB"} sub = snapshot_download(REPO, allow_patterns=[f"weights/upstream/{slot}/**"], token=token) local = os.path.join(sub, "weights", "upstream", slot) try: from transformers import AutoModelForCausalLM import torch model = AutoModelForCausalLM.from_pretrained( local, trust_remote_code=True, torch_dtype=torch.bfloat16, low_cpu_mem_usage=True) n = sum(p.numel() for p in model.parameters()) return {"ok": True, "class": type(model).__name__, "params": n} except Exception as exc: # noqa: BLE001 return {"ok": False, "error": f"{type(exc).__name__}: {str(exc)[:300]}"} def main() -> int: ap = argparse.ArgumentParser() ap.add_argument("--full", action="store_true") ap.add_argument("--only", nargs="*", default=None, help="restrict to slots: vision computer_use generation") ap.add_argument("--out", default="reports/loading_tests.json") args = ap.parse_args() token = os.environ.get("HF_TOKEN") if not token: print("HF_TOKEN required"); return 2 slots = args.only or ["vision", "computer_use"] # load any prior report so partial re-runs keep earlier results out = ROOT / args.out report = json.loads(out.read_text()) if out.exists() else {} for slot in slots: if slot == "generation": continue print(f"\n=== {slot} ===") rep = {} rep["shard_integrity"] = shard_integrity(slot, token) print(f" shards: {len(rep['shard_integrity']['shards'])} " f"tensors={rep['shard_integrity']['n_tensors']} " f"ok={rep['shard_integrity']['ok']}") rep["components"] = component_load(slot, token) for k, v in rep["components"].items(): print(f" {k}: {v.get('ok')} {v.get('error','')}") rep["full_model"] = full_model(slot, token, args.full) print(f" full_model: {rep['full_model']}") report[slot] = rep if "generation" in slots: print("\n=== generation ===") gen = {"vae": vae_load(token), "transformer": transformer_check(token)} print(f" vae: {gen['vae']}") print(f" transformer: {gen['transformer']}") report["generation"] = gen out.parent.mkdir(parents=True, exist_ok=True) out.write_text(json.dumps(report, indent=2)) print(f"\nreport -> {out}") return 0 if __name__ == "__main__": raise SystemExit(main())