Download scripts/test_loading_from_hub.py from Frankenstein-Labs/NOEON: direct link, hf CLI and curl.
- Browser
- Download file 9.61 kB
-
https://huggingface.co/Frankenstein-Labs/NOEON/resolve/main/scripts/test_loading_from_hub.py
- Command line
-
hf download hf://Frankenstein-Labs/NOEON/scripts/test_loading_from_hub.py
-
curl -L -o test_loading_from_hub.py https://huggingface.co/Frankenstein-Labs/NOEON/resolve/main/scripts/test_loading_from_hub.py
9.61 kB
| #!/usr/bin/env python3 | |
| """Real loading tests of the imported source weights, straight from the Hub. | |
| This is a *loading* test, not a download test: it exercises the actual model | |
| classes against the files now stored in ``Frankenstein-Labs/NOEON`` under | |
| ``weights/upstream/<slot>/``. | |
| Tiers, chosen so the test is honest about what this CPU box can do: | |
| 1. **shard integrity** — open every safetensors shard by memory-mapping it and | |
| parse its full header. Proves the shard is complete and its declared tensors | |
| are readable, without needing the model's RAM. | |
| 2. **component load** — load tokenizer / processor / config / scheduler / VAE | |
| weights with their real libraries from the imported files. | |
| 3. **full model** — ``from_pretrained`` of the whole model, only attempted when | |
| the device has enough RAM; reported as ``skipped`` otherwise. | |
| Usage: | |
| HF_TOKEN=... python scripts/test_loading_from_hub.py | |
| HF_TOKEN=... python scripts/test_loading_from_hub.py --full # force tier 3 | |
| """ | |
| from __future__ import annotations | |
| import argparse | |
| import json | |
| import os | |
| import sys | |
| from pathlib import Path | |
| ROOT = Path(__file__).resolve().parent.parent | |
| REPO = "Frankenstein-Labs/NOEON" | |
| def shard_integrity(slot: str, token: str) -> dict: | |
| """Memory-map every safetensors shard and read its header.""" | |
| from huggingface_hub import hf_hub_download | |
| from safetensors import safe_open | |
| man = json.loads((ROOT / "weights" / "source_files.json").read_text()) | |
| files = [f for f in man["sources"][slot]["files"] | |
| if f["path"].endswith(".safetensors")] | |
| out = {"shards": [], "n_tensors": 0, "ok": True} | |
| for f in files: | |
| rel = f"weights/upstream/{slot}/{f['path']}" | |
| try: | |
| local = hf_hub_download(REPO, rel, token=token) | |
| with safe_open(local, framework="pt") as g: | |
| keys = list(g.keys()) | |
| sample = keys[0] | |
| t = g.get_slice(sample) | |
| shape = t.get_shape() | |
| out["shards"].append({"path": rel, "tensors": len(keys), | |
| "sample": sample, "sample_shape": shape, | |
| "ok": True}) | |
| out["n_tensors"] += len(keys) | |
| except Exception as exc: # noqa: BLE001 | |
| out["shards"].append({"path": rel, "ok": False, "error": str(exc)}) | |
| out["ok"] = False | |
| return out | |
| def component_load(slot: str, token: str) -> dict: | |
| from huggingface_hub import snapshot_download | |
| res = {} | |
| sub = snapshot_download(REPO, allow_patterns=[f"weights/upstream/{slot}/*", | |
| f"weights/upstream/{slot}/**"], | |
| token=token) | |
| local = os.path.join(sub, "weights", "upstream", slot) | |
| # tokenizer / processor | |
| try: | |
| from transformers import AutoTokenizer | |
| tok = AutoTokenizer.from_pretrained(local, trust_remote_code=True) | |
| ids = tok("hello world", return_tensors="pt")["input_ids"] | |
| res["tokenizer"] = {"ok": True, "vocab": tok.vocab_size, | |
| "sample": ids.tolist()} | |
| except Exception as exc: # noqa: BLE001 | |
| res["tokenizer"] = {"ok": False, "error": f"{type(exc).__name__}: {exc}"} | |
| # config | |
| try: | |
| from transformers import AutoConfig | |
| cfg = AutoConfig.from_pretrained(local, trust_remote_code=True) | |
| res["config"] = {"ok": True, "class": cfg.__class__.__name__, | |
| "model_type": getattr(cfg, "model_type", None)} | |
| except Exception as exc: # noqa: BLE001 | |
| res["config"] = {"ok": False, "error": f"{type(exc).__name__}: {exc}"} | |
| return res | |
| def vae_load(token: str) -> dict: | |
| from huggingface_hub import snapshot_download | |
| res = {} | |
| sub = snapshot_download(REPO, | |
| allow_patterns=["weights/upstream/generation/vae/*"], | |
| token=token) | |
| local = os.path.join(sub, "weights", "upstream", "generation", "vae") | |
| # how many tensors the file holds vs. how many a stock AutoencoderKL uses | |
| from safetensors import safe_open | |
| from huggingface_hub import hf_hub_download | |
| ckpt = hf_hub_download( | |
| REPO, "weights/upstream/generation/vae/diffusion_pytorch_model.safetensors", | |
| token=token) | |
| with safe_open(ckpt, framework="pt") as g: | |
| file_keys = set(g.keys()) | |
| try: | |
| from diffusers import AutoencoderKL | |
| import torch | |
| vae = AutoencoderKL.from_pretrained(local, torch_dtype=torch.float32) | |
| model_keys = set(vae.state_dict().keys()) | |
| used = file_keys & model_keys | |
| x = torch.randn(1, 3, 64, 64) | |
| with torch.no_grad(): | |
| lat = vae.encode(x).latent_dist.sample() | |
| dec = vae.decode(lat).sample | |
| res["vae"] = {"ok": True, "class": type(vae).__name__, | |
| "file_tensors": len(file_keys), | |
| "used_by_stock_class": len(used), | |
| "latent_shape": list(lat.shape), | |
| "decoded_shape": list(dec.shape), | |
| "note": ("the checkpoint is a custom MageVAE; a stock " | |
| "AutoencoderKL consumes only its standard " | |
| "subset, so this proves load+forward plumbing " | |
| "but not exact-architecture fidelity")} | |
| except Exception as exc: # noqa: BLE001 | |
| res["vae"] = {"ok": False, "error": f"{type(exc).__name__}: {exc}"} | |
| return res | |
| def transformer_check(token: str) -> dict: | |
| """The MageFlow DiT is a custom class; verify its shard is complete.""" | |
| from huggingface_hub import hf_hub_download | |
| from safetensors import safe_open | |
| try: | |
| ckpt = hf_hub_download( | |
| REPO, | |
| "weights/upstream/generation/transformer/diffusion_pytorch_model.safetensors", | |
| token=token) | |
| with safe_open(ckpt, framework="pt") as g: | |
| keys = list(g.keys()) | |
| shp = g.get_slice(keys[0]).get_shape() | |
| return {"ok": True, "tensors": len(keys), "sample": keys[0], | |
| "sample_shape": shp, "note": "custom MageFlow class; header verified"} | |
| except Exception as exc: # noqa: BLE001 | |
| return {"ok": False, "error": f"{type(exc).__name__}: {exc}"} | |
| def full_model(slot: str, token: str, force: bool) -> dict: | |
| import torch | |
| from huggingface_hub import snapshot_download | |
| if not force: | |
| total = sum(f["size"] for f in | |
| json.loads((ROOT / "weights" / "source_files.json") | |
| .read_text())["sources"][slot]["files"] | |
| if f["path"].endswith(".safetensors")) | |
| # rough need: full fp32 model in RAM (~2x bf16 bytes) + headroom | |
| need = total * 2 + 2_000_000_000 | |
| import shutil | |
| avail = shutil.disk_usage("/").free | |
| mem = os.sysconf("SC_PAGE_SIZE") * os.sysconf("SC_PHYS_PAGES") | |
| if mem < need: | |
| return {"ok": False, "skipped": True, | |
| "reason": f"needs ~{need/1e9:.1f} GB RAM, box has {mem/1e9:.1f} GB"} | |
| sub = snapshot_download(REPO, | |
| allow_patterns=[f"weights/upstream/{slot}/**"], | |
| token=token) | |
| local = os.path.join(sub, "weights", "upstream", slot) | |
| try: | |
| from transformers import AutoModelForCausalLM | |
| import torch | |
| model = AutoModelForCausalLM.from_pretrained( | |
| local, trust_remote_code=True, torch_dtype=torch.bfloat16, | |
| low_cpu_mem_usage=True) | |
| n = sum(p.numel() for p in model.parameters()) | |
| return {"ok": True, "class": type(model).__name__, "params": n} | |
| except Exception as exc: # noqa: BLE001 | |
| return {"ok": False, "error": f"{type(exc).__name__}: {str(exc)[:300]}"} | |
| def main() -> int: | |
| ap = argparse.ArgumentParser() | |
| ap.add_argument("--full", action="store_true") | |
| ap.add_argument("--only", nargs="*", default=None, | |
| help="restrict to slots: vision computer_use generation") | |
| ap.add_argument("--out", default="reports/loading_tests.json") | |
| args = ap.parse_args() | |
| token = os.environ.get("HF_TOKEN") | |
| if not token: | |
| print("HF_TOKEN required"); return 2 | |
| slots = args.only or ["vision", "computer_use"] | |
| # load any prior report so partial re-runs keep earlier results | |
| out = ROOT / args.out | |
| report = json.loads(out.read_text()) if out.exists() else {} | |
| for slot in slots: | |
| if slot == "generation": | |
| continue | |
| print(f"\n=== {slot} ===") | |
| rep = {} | |
| rep["shard_integrity"] = shard_integrity(slot, token) | |
| print(f" shards: {len(rep['shard_integrity']['shards'])} " | |
| f"tensors={rep['shard_integrity']['n_tensors']} " | |
| f"ok={rep['shard_integrity']['ok']}") | |
| rep["components"] = component_load(slot, token) | |
| for k, v in rep["components"].items(): | |
| print(f" {k}: {v.get('ok')} {v.get('error','')}") | |
| rep["full_model"] = full_model(slot, token, args.full) | |
| print(f" full_model: {rep['full_model']}") | |
| report[slot] = rep | |
| if "generation" in slots: | |
| print("\n=== generation ===") | |
| gen = {"vae": vae_load(token), "transformer": transformer_check(token)} | |
| print(f" vae: {gen['vae']}") | |
| print(f" transformer: {gen['transformer']}") | |
| report["generation"] = gen | |
| out.parent.mkdir(parents=True, exist_ok=True) | |
| out.write_text(json.dumps(report, indent=2)) | |
| print(f"\nreport -> {out}") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |