NOEON / scripts /verify_real_bytes.py
openhands
openhands
NOEON: real context system, inter-model communication, integrated brain
3f71e8c
Raw History Blame Contribute Delete
5.32 kB
#!/usr/bin/env python3
"""Run NOEON's bridges on REAL bytes fetched directly from the sources.
For each capability this pulls, with HTTP range requests (a few KB):
* the safetensors header of a real shard, to locate a genuine 2560-d tensor;
* a small slice of that tensor's *actual data*, straight from the Hub.
The slice is fed through the corresponding NOEON bridge, proving the data path
consumes real source bytes at the shared 2560-d width — not synthetic noise.
It records the file, the tensor name, its real shape, the byte range used and the
bridge output in ``weights/real_bytes_forward.json``.
Usage:
python scripts/verify_real_bytes.py
"""
from __future__ import annotations
import json
import struct
import sys
import urllib.request
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
import torch
from noeon.core.bridge import BridgeBank
from noeon.core.config import NoeonConfig
from noeon.core.registry import SOURCE_MODELS
ROOT = Path(__file__).resolve().parent.parent
SHARED = 2560
_DT = {"BF16": 2, "F16": 2, "F32": 4, "I64": 8, "I32": 4, "U8": 1, "I8": 1,
"BOOL": 1, "F8_E4M3": 1, "F8_E5M2": 1}
def _get(url: str, rng: str | None = None, token: str | None = None) -> bytes:
headers = {"User-Agent": "noeon/0.2"}
if rng:
headers["Range"] = rng
if token:
headers["Authorization"] = f"Bearer {token}"
req = urllib.request.Request(url, headers=headers)
with urllib.request.urlopen(req, timeout=120) as r:
return r.read()
def read_header(url: str, token: str | None) -> dict:
buf = _get(url, "bytes=0-1048575", token)
n = struct.unpack("<Q", buf[:8])[0]
if len(buf) < 8 + n:
buf = _get(url, f"bytes=0-{8+n-1}", token)
hdr = json.loads(buf[8:8 + n].decode("utf-8"))
hdr.pop("__metadata__", None)
return hdr
def pick_tensor(hdr: dict) -> str | None:
"""Prefer a 2560-d embed row; fall back to a 2560-d context projection."""
for name, t in hdr.items():
if name.endswith("embed_tokens.weight") and t["shape"][-1] == SHARED:
return name
for name, t in hdr.items():
if name.endswith("txt_in.weight") and SHARED in t["shape"]:
return name
for name, t in hdr.items():
if len(t["shape"]) == 2 and t["shape"][-1] == SHARED:
return name
return None
def fetch_rows(url: str, hdr: dict, name: str, rows: int, token: str | None):
"""Fetch ``rows`` rows of tensor ``name`` via a byte range request."""
info = hdr[name]
dtype, shape = info["dtype"], info["shape"]
itemsize = _DT.get(dtype, 2)
start, end = info["data_offsets"]
row_bytes = shape[-1] * itemsize
n_bytes = row_bytes * rows
abs_start = 8 + struct.unpack("<Q", _get(url, "bytes=0-7", token))[0] + start
raw = _get(url, f"bytes={abs_start}-{abs_start + n_bytes - 1}", token)
torch_dtype = {"BF16": torch.bfloat16, "F16": torch.float16,
"F32": torch.float32}.get(dtype, torch.bfloat16)
t = torch.frombuffer(bytearray(raw), dtype=torch_dtype).reshape(rows, shape[-1])
return t.float(), (abs_start, abs_start + n_bytes - 1)
def main() -> int:
import os
token = os.environ.get("HF_TOKEN")
banks = BridgeBank(NoeonConfig().bridges)
banks.eval()
manifest = json.loads((ROOT / "weights" / "source_files.json").read_text())
report = {"shared_dim": SHARED, "capabilities": {}}
for key, src in SOURCE_MODELS.items():
files = [f for f in manifest["sources"][key]["files"] if f["is_weight"]]
done = False
for f in files:
url = (f"https://huggingface.co/{src.repo}/resolve/{src.revision}/"
f"{f['path']}")
try:
hdr = read_header(url, token)
except Exception as e: # noqa: BLE001
continue
name = pick_tensor(hdr)
if not name:
continue
try:
sample, byte_range = fetch_rows(url, hdr, name, rows=4, token=token)
except Exception as e: # noqa: BLE001
print(f"[{key}] fetch failed on {name}: {e}")
continue
with torch.no_grad():
out = banks(key, sample.unsqueeze(0))
report["capabilities"][key] = {
"repo": src.repo,
"file": f["path"],
"tensor": name,
"tensor_shape": hdr[name]["shape"],
"dtype": hdr[name]["dtype"],
"byte_range": list(byte_range),
"input_shape": list(sample.shape),
"input_mean": round(float(sample.mean()), 6),
"bridge_output_shape": list(out.shape),
"bridge_output_mean": round(float(out.mean()), 6),
}
print(f"[{key}] {f['path']}::{name} {tuple(sample.shape)} "
f"-> bridge {tuple(out.shape)} (real bytes)")
done = True
break
if not done:
report["capabilities"][key] = {"error": "no 2560-d tensor found"}
out_path = ROOT / "weights" / "real_bytes_forward.json"
out_path.write_text(json.dumps(report, indent=2))
print(f"\nwrote {out_path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())