#!/usr/bin/env python3 """Run NOEON's bridges on REAL bytes fetched directly from the sources. For each capability this pulls, with HTTP range requests (a few KB): * the safetensors header of a real shard, to locate a genuine 2560-d tensor; * a small slice of that tensor's *actual data*, straight from the Hub. The slice is fed through the corresponding NOEON bridge, proving the data path consumes real source bytes at the shared 2560-d width — not synthetic noise. It records the file, the tensor name, its real shape, the byte range used and the bridge output in ``weights/real_bytes_forward.json``. Usage: python scripts/verify_real_bytes.py """ from __future__ import annotations import json import struct import sys import urllib.request from pathlib import Path sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) import torch from noeon.core.bridge import BridgeBank from noeon.core.config import NoeonConfig from noeon.core.registry import SOURCE_MODELS ROOT = Path(__file__).resolve().parent.parent SHARED = 2560 _DT = {"BF16": 2, "F16": 2, "F32": 4, "I64": 8, "I32": 4, "U8": 1, "I8": 1, "BOOL": 1, "F8_E4M3": 1, "F8_E5M2": 1} def _get(url: str, rng: str | None = None, token: str | None = None) -> bytes: headers = {"User-Agent": "noeon/0.2"} if rng: headers["Range"] = rng if token: headers["Authorization"] = f"Bearer {token}" req = urllib.request.Request(url, headers=headers) with urllib.request.urlopen(req, timeout=120) as r: return r.read() def read_header(url: str, token: str | None) -> dict: buf = _get(url, "bytes=0-1048575", token) n = struct.unpack(" str | None: """Prefer a 2560-d embed row; fall back to a 2560-d context projection.""" for name, t in hdr.items(): if name.endswith("embed_tokens.weight") and t["shape"][-1] == SHARED: return name for name, t in hdr.items(): if name.endswith("txt_in.weight") and SHARED in t["shape"]: return name for name, t in hdr.items(): if len(t["shape"]) == 2 and t["shape"][-1] == SHARED: return name return None def fetch_rows(url: str, hdr: dict, name: str, rows: int, token: str | None): """Fetch ``rows`` rows of tensor ``name`` via a byte range request.""" info = hdr[name] dtype, shape = info["dtype"], info["shape"] itemsize = _DT.get(dtype, 2) start, end = info["data_offsets"] row_bytes = shape[-1] * itemsize n_bytes = row_bytes * rows abs_start = 8 + struct.unpack(" int: import os token = os.environ.get("HF_TOKEN") banks = BridgeBank(NoeonConfig().bridges) banks.eval() manifest = json.loads((ROOT / "weights" / "source_files.json").read_text()) report = {"shared_dim": SHARED, "capabilities": {}} for key, src in SOURCE_MODELS.items(): files = [f for f in manifest["sources"][key]["files"] if f["is_weight"]] done = False for f in files: url = (f"https://huggingface.co/{src.repo}/resolve/{src.revision}/" f"{f['path']}") try: hdr = read_header(url, token) except Exception as e: # noqa: BLE001 continue name = pick_tensor(hdr) if not name: continue try: sample, byte_range = fetch_rows(url, hdr, name, rows=4, token=token) except Exception as e: # noqa: BLE001 print(f"[{key}] fetch failed on {name}: {e}") continue with torch.no_grad(): out = banks(key, sample.unsqueeze(0)) report["capabilities"][key] = { "repo": src.repo, "file": f["path"], "tensor": name, "tensor_shape": hdr[name]["shape"], "dtype": hdr[name]["dtype"], "byte_range": list(byte_range), "input_shape": list(sample.shape), "input_mean": round(float(sample.mean()), 6), "bridge_output_shape": list(out.shape), "bridge_output_mean": round(float(out.mean()), 6), } print(f"[{key}] {f['path']}::{name} {tuple(sample.shape)} " f"-> bridge {tuple(out.shape)} (real bytes)") done = True break if not done: report["capabilities"][key] = {"error": "no 2560-d tensor found"} out_path = ROOT / "weights" / "real_bytes_forward.json" out_path.write_text(json.dumps(report, indent=2)) print(f"\nwrote {out_path}") return 0 if __name__ == "__main__": raise SystemExit(main())