File size: 5,322 Bytes
3f71e8c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93 94 95 96 97 98 99 100 101 102 103 104 105 106 107 108 109 110 111 112 113 114 115 116 117 118 119 120 121 122 123 124 125 126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142 143 144 145 146 147 148 | #!/usr/bin/env python3
"""Run NOEON's bridges on REAL bytes fetched directly from the sources.
For each capability this pulls, with HTTP range requests (a few KB):
* the safetensors header of a real shard, to locate a genuine 2560-d tensor;
* a small slice of that tensor's *actual data*, straight from the Hub.
The slice is fed through the corresponding NOEON bridge, proving the data path
consumes real source bytes at the shared 2560-d width — not synthetic noise.
It records the file, the tensor name, its real shape, the byte range used and the
bridge output in ``weights/real_bytes_forward.json``.
Usage:
python scripts/verify_real_bytes.py
"""
from __future__ import annotations
import json
import struct
import sys
import urllib.request
from pathlib import Path
sys.path.insert(0, str(Path(__file__).resolve().parent.parent))
import torch
from noeon.core.bridge import BridgeBank
from noeon.core.config import NoeonConfig
from noeon.core.registry import SOURCE_MODELS
ROOT = Path(__file__).resolve().parent.parent
SHARED = 2560
_DT = {"BF16": 2, "F16": 2, "F32": 4, "I64": 8, "I32": 4, "U8": 1, "I8": 1,
"BOOL": 1, "F8_E4M3": 1, "F8_E5M2": 1}
def _get(url: str, rng: str | None = None, token: str | None = None) -> bytes:
headers = {"User-Agent": "noeon/0.2"}
if rng:
headers["Range"] = rng
if token:
headers["Authorization"] = f"Bearer {token}"
req = urllib.request.Request(url, headers=headers)
with urllib.request.urlopen(req, timeout=120) as r:
return r.read()
def read_header(url: str, token: str | None) -> dict:
buf = _get(url, "bytes=0-1048575", token)
n = struct.unpack("<Q", buf[:8])[0]
if len(buf) < 8 + n:
buf = _get(url, f"bytes=0-{8+n-1}", token)
hdr = json.loads(buf[8:8 + n].decode("utf-8"))
hdr.pop("__metadata__", None)
return hdr
def pick_tensor(hdr: dict) -> str | None:
"""Prefer a 2560-d embed row; fall back to a 2560-d context projection."""
for name, t in hdr.items():
if name.endswith("embed_tokens.weight") and t["shape"][-1] == SHARED:
return name
for name, t in hdr.items():
if name.endswith("txt_in.weight") and SHARED in t["shape"]:
return name
for name, t in hdr.items():
if len(t["shape"]) == 2 and t["shape"][-1] == SHARED:
return name
return None
def fetch_rows(url: str, hdr: dict, name: str, rows: int, token: str | None):
"""Fetch ``rows`` rows of tensor ``name`` via a byte range request."""
info = hdr[name]
dtype, shape = info["dtype"], info["shape"]
itemsize = _DT.get(dtype, 2)
start, end = info["data_offsets"]
row_bytes = shape[-1] * itemsize
n_bytes = row_bytes * rows
abs_start = 8 + struct.unpack("<Q", _get(url, "bytes=0-7", token))[0] + start
raw = _get(url, f"bytes={abs_start}-{abs_start + n_bytes - 1}", token)
torch_dtype = {"BF16": torch.bfloat16, "F16": torch.float16,
"F32": torch.float32}.get(dtype, torch.bfloat16)
t = torch.frombuffer(bytearray(raw), dtype=torch_dtype).reshape(rows, shape[-1])
return t.float(), (abs_start, abs_start + n_bytes - 1)
def main() -> int:
import os
token = os.environ.get("HF_TOKEN")
banks = BridgeBank(NoeonConfig().bridges)
banks.eval()
manifest = json.loads((ROOT / "weights" / "source_files.json").read_text())
report = {"shared_dim": SHARED, "capabilities": {}}
for key, src in SOURCE_MODELS.items():
files = [f for f in manifest["sources"][key]["files"] if f["is_weight"]]
done = False
for f in files:
url = (f"https://huggingface.co/{src.repo}/resolve/{src.revision}/"
f"{f['path']}")
try:
hdr = read_header(url, token)
except Exception as e: # noqa: BLE001
continue
name = pick_tensor(hdr)
if not name:
continue
try:
sample, byte_range = fetch_rows(url, hdr, name, rows=4, token=token)
except Exception as e: # noqa: BLE001
print(f"[{key}] fetch failed on {name}: {e}")
continue
with torch.no_grad():
out = banks(key, sample.unsqueeze(0))
report["capabilities"][key] = {
"repo": src.repo,
"file": f["path"],
"tensor": name,
"tensor_shape": hdr[name]["shape"],
"dtype": hdr[name]["dtype"],
"byte_range": list(byte_range),
"input_shape": list(sample.shape),
"input_mean": round(float(sample.mean()), 6),
"bridge_output_shape": list(out.shape),
"bridge_output_mean": round(float(out.mean()), 6),
}
print(f"[{key}] {f['path']}::{name} {tuple(sample.shape)} "
f"-> bridge {tuple(out.shape)} (real bytes)")
done = True
break
if not done:
report["capabilities"][key] = {"error": "no 2560-d tensor found"}
out_path = ROOT / "weights" / "real_bytes_forward.json"
out_path.write_text(json.dumps(report, indent=2))
print(f"\nwrote {out_path}")
return 0
if __name__ == "__main__":
raise SystemExit(main())
|