File size: 3,372 Bytes
d52c89c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 | """Explicit partial nanochat matrix initialization, never full-model conversion."""
import json
from pathlib import Path
import torch
from .tokens import sha256
INSPECTED_REVISION = "92d63d4e8bb4df75c3b71618f31ddde2378b2bcd"
LEGACY_REVISION = "d6d86cbf4c0bcc1de5bbab28cae1f98038d0362a"
MATRICES = (
"attn.c_q.weight",
"attn.c_k.weight",
"attn.c_v.weight",
"attn.c_proj.weight",
"mlp.c_fc.weight",
"mlp.c_proj.weight",
)
def initialize_temporal(model, weights_path, metadata_path, *, source_revision):
"""Only use after obtaining authorized, pinned weights and their matching metadata.
Linear tensors in both implementations are [out_features,in_features]. No
transposition, reshaping, truncation, layer interpolation or optimizer transfer.
"""
if source_revision not in {INSPECTED_REVISION, LEGACY_REVISION}:
raise ValueError("Unreviewed nanochat source revision")
meta = json.loads(Path(metadata_path).read_text())
source = meta["model_config"]
c = model.config
expected = {
"n_layer": c.layers,
"n_embd": c.width,
"n_head": c.heads,
"n_kv_head": c.heads,
}
if any(source.get(k) != v for k, v in expected.items()):
raise ValueError("Partial transfer requires exact layers/width/MHA heads")
state = torch.load(weights_path, map_location="cpu", weights_only=True)
normalized = {k.removeprefix("_orig_mod."): v for k, v in state.items()}
if len(normalized) != len(state):
raise ValueError("Duplicate compiled/uncompiled state keys")
destination = model.state_dict()
mapping = {
f"transformer.h.{i}.{suffix}": f"temporal.{i}.{suffix}"
for i in range(c.layers)
for suffix in MATRICES
}
# Validate the entire transaction before mutating any parameter.
for src, dst in mapping.items():
tensor = normalized[src]
if (
tensor.shape != destination[dst].shape
or not tensor.is_floating_point()
or not torch.isfinite(tensor).all()
):
raise ValueError(f"Invalid matrix {src}")
with torch.no_grad():
for src, dst in mapping.items():
destination[dst].copy_(normalized[src])
return {
"arm": "text_initialized_partial",
"source_revision": source_revision,
"weights_sha256": sha256(weights_path),
"metadata_sha256": sha256(metadata_path),
"source_config": source,
"source_revision_basis": "Caller-declared; not cryptographic training-code attestation",
"attention_conventions": {
"source_rotary_base": 10000
if source_revision == LEGACY_REVISION
else 100000,
"target_rotary_base": 100000,
"source_qk_scale_each": 1.0 if source_revision == LEGACY_REVISION else 1.2,
"target_qk_scale_each": 1.2,
},
"behavioral_mismatches": (
[
"RoPE base changes from 10000 to 100000",
"Each normalized Q/K gains a 1.2 multiplier",
]
if source_revision == LEGACY_REVISION
else []
),
"transferred": mapping,
"omitted_source_tensors": sorted(set(normalized) - set(mapping)),
"claim": "Partial matrix initialization only; no equivalence or benefit established",
}
|