Download scripts/verify_real_bytes.py from Frankenstein-Labs/NOEON: direct link, hf CLI and curl.
- Browser
- Download file 5.32 kB
-
https://huggingface.co/Frankenstein-Labs/NOEON/resolve/main/scripts/verify_real_bytes.py
- Command line
-
hf download hf://Frankenstein-Labs/NOEON/scripts/verify_real_bytes.py
-
curl -L -o verify_real_bytes.py https://huggingface.co/Frankenstein-Labs/NOEON/resolve/main/scripts/verify_real_bytes.py
5.32 kB
| #!/usr/bin/env python3 | |
| """Run NOEON's bridges on REAL bytes fetched directly from the sources. | |
| For each capability this pulls, with HTTP range requests (a few KB): | |
| * the safetensors header of a real shard, to locate a genuine 2560-d tensor; | |
| * a small slice of that tensor's *actual data*, straight from the Hub. | |
| The slice is fed through the corresponding NOEON bridge, proving the data path | |
| consumes real source bytes at the shared 2560-d width — not synthetic noise. | |
| It records the file, the tensor name, its real shape, the byte range used and the | |
| bridge output in ``weights/real_bytes_forward.json``. | |
| Usage: | |
| python scripts/verify_real_bytes.py | |
| """ | |
| from __future__ import annotations | |
| import json | |
| import struct | |
| import sys | |
| import urllib.request | |
| from pathlib import Path | |
| sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) | |
| import torch | |
| from noeon.core.bridge import BridgeBank | |
| from noeon.core.config import NoeonConfig | |
| from noeon.core.registry import SOURCE_MODELS | |
| ROOT = Path(__file__).resolve().parent.parent | |
| SHARED = 2560 | |
| _DT = {"BF16": 2, "F16": 2, "F32": 4, "I64": 8, "I32": 4, "U8": 1, "I8": 1, | |
| "BOOL": 1, "F8_E4M3": 1, "F8_E5M2": 1} | |
| def _get(url: str, rng: str | None = None, token: str | None = None) -> bytes: | |
| headers = {"User-Agent": "noeon/0.2"} | |
| if rng: | |
| headers["Range"] = rng | |
| if token: | |
| headers["Authorization"] = f"Bearer {token}" | |
| req = urllib.request.Request(url, headers=headers) | |
| with urllib.request.urlopen(req, timeout=120) as r: | |
| return r.read() | |
| def read_header(url: str, token: str | None) -> dict: | |
| buf = _get(url, "bytes=0-1048575", token) | |
| n = struct.unpack("<Q", buf[:8])[0] | |
| if len(buf) < 8 + n: | |
| buf = _get(url, f"bytes=0-{8+n-1}", token) | |
| hdr = json.loads(buf[8:8 + n].decode("utf-8")) | |
| hdr.pop("__metadata__", None) | |
| return hdr | |
| def pick_tensor(hdr: dict) -> str | None: | |
| """Prefer a 2560-d embed row; fall back to a 2560-d context projection.""" | |
| for name, t in hdr.items(): | |
| if name.endswith("embed_tokens.weight") and t["shape"][-1] == SHARED: | |
| return name | |
| for name, t in hdr.items(): | |
| if name.endswith("txt_in.weight") and SHARED in t["shape"]: | |
| return name | |
| for name, t in hdr.items(): | |
| if len(t["shape"]) == 2 and t["shape"][-1] == SHARED: | |
| return name | |
| return None | |
| def fetch_rows(url: str, hdr: dict, name: str, rows: int, token: str | None): | |
| """Fetch ``rows`` rows of tensor ``name`` via a byte range request.""" | |
| info = hdr[name] | |
| dtype, shape = info["dtype"], info["shape"] | |
| itemsize = _DT.get(dtype, 2) | |
| start, end = info["data_offsets"] | |
| row_bytes = shape[-1] * itemsize | |
| n_bytes = row_bytes * rows | |
| abs_start = 8 + struct.unpack("<Q", _get(url, "bytes=0-7", token))[0] + start | |
| raw = _get(url, f"bytes={abs_start}-{abs_start + n_bytes - 1}", token) | |
| torch_dtype = {"BF16": torch.bfloat16, "F16": torch.float16, | |
| "F32": torch.float32}.get(dtype, torch.bfloat16) | |
| t = torch.frombuffer(bytearray(raw), dtype=torch_dtype).reshape(rows, shape[-1]) | |
| return t.float(), (abs_start, abs_start + n_bytes - 1) | |
| def main() -> int: | |
| import os | |
| token = os.environ.get("HF_TOKEN") | |
| banks = BridgeBank(NoeonConfig().bridges) | |
| banks.eval() | |
| manifest = json.loads((ROOT / "weights" / "source_files.json").read_text()) | |
| report = {"shared_dim": SHARED, "capabilities": {}} | |
| for key, src in SOURCE_MODELS.items(): | |
| files = [f for f in manifest["sources"][key]["files"] if f["is_weight"]] | |
| done = False | |
| for f in files: | |
| url = (f"https://huggingface.co/{src.repo}/resolve/{src.revision}/" | |
| f"{f['path']}") | |
| try: | |
| hdr = read_header(url, token) | |
| except Exception as e: # noqa: BLE001 | |
| continue | |
| name = pick_tensor(hdr) | |
| if not name: | |
| continue | |
| try: | |
| sample, byte_range = fetch_rows(url, hdr, name, rows=4, token=token) | |
| except Exception as e: # noqa: BLE001 | |
| print(f"[{key}] fetch failed on {name}: {e}") | |
| continue | |
| with torch.no_grad(): | |
| out = banks(key, sample.unsqueeze(0)) | |
| report["capabilities"][key] = { | |
| "repo": src.repo, | |
| "file": f["path"], | |
| "tensor": name, | |
| "tensor_shape": hdr[name]["shape"], | |
| "dtype": hdr[name]["dtype"], | |
| "byte_range": list(byte_range), | |
| "input_shape": list(sample.shape), | |
| "input_mean": round(float(sample.mean()), 6), | |
| "bridge_output_shape": list(out.shape), | |
| "bridge_output_mean": round(float(out.mean()), 6), | |
| } | |
| print(f"[{key}] {f['path']}::{name} {tuple(sample.shape)} " | |
| f"-> bridge {tuple(out.shape)} (real bytes)") | |
| done = True | |
| break | |
| if not done: | |
| report["capabilities"][key] = {"error": "no 2560-d tensor found"} | |
| out_path = ROOT / "weights" / "real_bytes_forward.json" | |
| out_path.write_text(json.dumps(report, indent=2)) | |
| print(f"\nwrote {out_path}") | |
| return 0 | |
| if __name__ == "__main__": | |
| raise SystemExit(main()) | |