Safetensors
English
File size: 3,372 Bytes
d52c89c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
"""Explicit partial nanochat matrix initialization, never full-model conversion."""

import json
from pathlib import Path

import torch

from .tokens import sha256

INSPECTED_REVISION = "92d63d4e8bb4df75c3b71618f31ddde2378b2bcd"
LEGACY_REVISION = "d6d86cbf4c0bcc1de5bbab28cae1f98038d0362a"
MATRICES = (
    "attn.c_q.weight",
    "attn.c_k.weight",
    "attn.c_v.weight",
    "attn.c_proj.weight",
    "mlp.c_fc.weight",
    "mlp.c_proj.weight",
)


def initialize_temporal(model, weights_path, metadata_path, *, source_revision):
    """Only use after obtaining authorized, pinned weights and their matching metadata.

    Linear tensors in both implementations are [out_features,in_features]. No
    transposition, reshaping, truncation, layer interpolation or optimizer transfer.
    """
    if source_revision not in {INSPECTED_REVISION, LEGACY_REVISION}:
        raise ValueError("Unreviewed nanochat source revision")
    meta = json.loads(Path(metadata_path).read_text())
    source = meta["model_config"]
    c = model.config
    expected = {
        "n_layer": c.layers,
        "n_embd": c.width,
        "n_head": c.heads,
        "n_kv_head": c.heads,
    }
    if any(source.get(k) != v for k, v in expected.items()):
        raise ValueError("Partial transfer requires exact layers/width/MHA heads")
    state = torch.load(weights_path, map_location="cpu", weights_only=True)
    normalized = {k.removeprefix("_orig_mod."): v for k, v in state.items()}
    if len(normalized) != len(state):
        raise ValueError("Duplicate compiled/uncompiled state keys")
    destination = model.state_dict()
    mapping = {
        f"transformer.h.{i}.{suffix}": f"temporal.{i}.{suffix}"
        for i in range(c.layers)
        for suffix in MATRICES
    }
    # Validate the entire transaction before mutating any parameter.
    for src, dst in mapping.items():
        tensor = normalized[src]
        if (
            tensor.shape != destination[dst].shape
            or not tensor.is_floating_point()
            or not torch.isfinite(tensor).all()
        ):
            raise ValueError(f"Invalid matrix {src}")
    with torch.no_grad():
        for src, dst in mapping.items():
            destination[dst].copy_(normalized[src])
    return {
        "arm": "text_initialized_partial",
        "source_revision": source_revision,
        "weights_sha256": sha256(weights_path),
        "metadata_sha256": sha256(metadata_path),
        "source_config": source,
        "source_revision_basis": "Caller-declared; not cryptographic training-code attestation",
        "attention_conventions": {
            "source_rotary_base": 10000
            if source_revision == LEGACY_REVISION
            else 100000,
            "target_rotary_base": 100000,
            "source_qk_scale_each": 1.0 if source_revision == LEGACY_REVISION else 1.2,
            "target_qk_scale_each": 1.2,
        },
        "behavioral_mismatches": (
            [
                "RoPE base changes from 10000 to 100000",
                "Each normalized Q/K gains a 1.2 multiplier",
            ]
            if source_revision == LEGACY_REVISION
            else []
        ),
        "transferred": mapping,
        "omitted_source_tensors": sorted(set(normalized) - set(mapping)),
        "claim": "Partial matrix initialization only; no equivalence or benefit established",
    }