File size: 5,322 Bytes
3f71e8c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
#!/usr/bin/env python3
"""Run NOEON's bridges on REAL bytes fetched directly from the sources.

For each capability this pulls, with HTTP range requests (a few KB):

* the safetensors header of a real shard, to locate a genuine 2560-d tensor;
* a small slice of that tensor's *actual data*, straight from the Hub.

The slice is fed through the corresponding NOEON bridge, proving the data path
consumes real source bytes at the shared 2560-d width — not synthetic noise.

It records the file, the tensor name, its real shape, the byte range used and the
bridge output in ``weights/real_bytes_forward.json``.

Usage:
    python scripts/verify_real_bytes.py
"""

from __future__ import annotations

import json
import struct
import sys
import urllib.request
from pathlib import Path

sys.path.insert(0, str(Path(__file__).resolve().parent.parent))

import torch

from noeon.core.bridge import BridgeBank
from noeon.core.config import NoeonConfig
from noeon.core.registry import SOURCE_MODELS

ROOT = Path(__file__).resolve().parent.parent
SHARED = 2560
_DT = {"BF16": 2, "F16": 2, "F32": 4, "I64": 8, "I32": 4, "U8": 1, "I8": 1,
       "BOOL": 1, "F8_E4M3": 1, "F8_E5M2": 1}


def _get(url: str, rng: str | None = None, token: str | None = None) -> bytes:
    headers = {"User-Agent": "noeon/0.2"}
    if rng:
        headers["Range"] = rng
    if token:
        headers["Authorization"] = f"Bearer {token}"
    req = urllib.request.Request(url, headers=headers)
    with urllib.request.urlopen(req, timeout=120) as r:
        return r.read()


def read_header(url: str, token: str | None) -> dict:
    buf = _get(url, "bytes=0-1048575", token)
    n = struct.unpack("<Q", buf[:8])[0]
    if len(buf) < 8 + n:
        buf = _get(url, f"bytes=0-{8+n-1}", token)
    hdr = json.loads(buf[8:8 + n].decode("utf-8"))
    hdr.pop("__metadata__", None)
    return hdr


def pick_tensor(hdr: dict) -> str | None:
    """Prefer a 2560-d embed row; fall back to a 2560-d context projection."""
    for name, t in hdr.items():
        if name.endswith("embed_tokens.weight") and t["shape"][-1] == SHARED:
            return name
    for name, t in hdr.items():
        if name.endswith("txt_in.weight") and SHARED in t["shape"]:
            return name
    for name, t in hdr.items():
        if len(t["shape"]) == 2 and t["shape"][-1] == SHARED:
            return name
    return None


def fetch_rows(url: str, hdr: dict, name: str, rows: int, token: str | None):
    """Fetch ``rows`` rows of tensor ``name`` via a byte range request."""
    info = hdr[name]
    dtype, shape = info["dtype"], info["shape"]
    itemsize = _DT.get(dtype, 2)
    start, end = info["data_offsets"]
    row_bytes = shape[-1] * itemsize
    n_bytes = row_bytes * rows
    abs_start = 8 + struct.unpack("<Q", _get(url, "bytes=0-7", token))[0] + start
    raw = _get(url, f"bytes={abs_start}-{abs_start + n_bytes - 1}", token)
    torch_dtype = {"BF16": torch.bfloat16, "F16": torch.float16,
                   "F32": torch.float32}.get(dtype, torch.bfloat16)
    t = torch.frombuffer(bytearray(raw), dtype=torch_dtype).reshape(rows, shape[-1])
    return t.float(), (abs_start, abs_start + n_bytes - 1)


def main() -> int:
    import os
    token = os.environ.get("HF_TOKEN")
    banks = BridgeBank(NoeonConfig().bridges)
    banks.eval()

    manifest = json.loads((ROOT / "weights" / "source_files.json").read_text())
    report = {"shared_dim": SHARED, "capabilities": {}}

    for key, src in SOURCE_MODELS.items():
        files = [f for f in manifest["sources"][key]["files"] if f["is_weight"]]
        done = False
        for f in files:
            url = (f"https://huggingface.co/{src.repo}/resolve/{src.revision}/"
                   f"{f['path']}")
            try:
                hdr = read_header(url, token)
            except Exception as e:  # noqa: BLE001
                continue
            name = pick_tensor(hdr)
            if not name:
                continue
            try:
                sample, byte_range = fetch_rows(url, hdr, name, rows=4, token=token)
            except Exception as e:  # noqa: BLE001
                print(f"[{key}] fetch failed on {name}: {e}")
                continue
            with torch.no_grad():
                out = banks(key, sample.unsqueeze(0))
            report["capabilities"][key] = {
                "repo": src.repo,
                "file": f["path"],
                "tensor": name,
                "tensor_shape": hdr[name]["shape"],
                "dtype": hdr[name]["dtype"],
                "byte_range": list(byte_range),
                "input_shape": list(sample.shape),
                "input_mean": round(float(sample.mean()), 6),
                "bridge_output_shape": list(out.shape),
                "bridge_output_mean": round(float(out.mean()), 6),
            }
            print(f"[{key}] {f['path']}::{name} {tuple(sample.shape)} "
                  f"-> bridge {tuple(out.shape)}  (real bytes)")
            done = True
            break
        if not done:
            report["capabilities"][key] = {"error": "no 2560-d tensor found"}

    out_path = ROOT / "weights" / "real_bytes_forward.json"
    out_path.write_text(json.dumps(report, indent=2))
    print(f"\nwrote {out_path}")
    return 0


if __name__ == "__main__":
    raise SystemExit(main())