Download scripts/duplex/model/transfer.py from Trelis/duplex-0: direct link, hf CLI and curl.
- Browser
- Download file 3.37 kB
-
https://huggingface.co/Trelis/duplex-0/resolve/main/scripts/duplex/model/transfer.py
- Command line
-
hf download hf://Trelis/duplex-0/scripts/duplex/model/transfer.py
-
curl -L -o transfer.py https://huggingface.co/Trelis/duplex-0/resolve/main/scripts/duplex/model/transfer.py
3.37 kB
| """Explicit partial nanochat matrix initialization, never full-model conversion.""" | |
| import json | |
| from pathlib import Path | |
| import torch | |
| from .tokens import sha256 | |
| INSPECTED_REVISION = "92d63d4e8bb4df75c3b71618f31ddde2378b2bcd" | |
| LEGACY_REVISION = "d6d86cbf4c0bcc1de5bbab28cae1f98038d0362a" | |
| MATRICES = ( | |
| "attn.c_q.weight", | |
| "attn.c_k.weight", | |
| "attn.c_v.weight", | |
| "attn.c_proj.weight", | |
| "mlp.c_fc.weight", | |
| "mlp.c_proj.weight", | |
| ) | |
| def initialize_temporal(model, weights_path, metadata_path, *, source_revision): | |
| """Only use after obtaining authorized, pinned weights and their matching metadata. | |
| Linear tensors in both implementations are [out_features,in_features]. No | |
| transposition, reshaping, truncation, layer interpolation or optimizer transfer. | |
| """ | |
| if source_revision not in {INSPECTED_REVISION, LEGACY_REVISION}: | |
| raise ValueError("Unreviewed nanochat source revision") | |
| meta = json.loads(Path(metadata_path).read_text()) | |
| source = meta["model_config"] | |
| c = model.config | |
| expected = { | |
| "n_layer": c.layers, | |
| "n_embd": c.width, | |
| "n_head": c.heads, | |
| "n_kv_head": c.heads, | |
| } | |
| if any(source.get(k) != v for k, v in expected.items()): | |
| raise ValueError("Partial transfer requires exact layers/width/MHA heads") | |
| state = torch.load(weights_path, map_location="cpu", weights_only=True) | |
| normalized = {k.removeprefix("_orig_mod."): v for k, v in state.items()} | |
| if len(normalized) != len(state): | |
| raise ValueError("Duplicate compiled/uncompiled state keys") | |
| destination = model.state_dict() | |
| mapping = { | |
| f"transformer.h.{i}.{suffix}": f"temporal.{i}.{suffix}" | |
| for i in range(c.layers) | |
| for suffix in MATRICES | |
| } | |
| # Validate the entire transaction before mutating any parameter. | |
| for src, dst in mapping.items(): | |
| tensor = normalized[src] | |
| if ( | |
| tensor.shape != destination[dst].shape | |
| or not tensor.is_floating_point() | |
| or not torch.isfinite(tensor).all() | |
| ): | |
| raise ValueError(f"Invalid matrix {src}") | |
| with torch.no_grad(): | |
| for src, dst in mapping.items(): | |
| destination[dst].copy_(normalized[src]) | |
| return { | |
| "arm": "text_initialized_partial", | |
| "source_revision": source_revision, | |
| "weights_sha256": sha256(weights_path), | |
| "metadata_sha256": sha256(metadata_path), | |
| "source_config": source, | |
| "source_revision_basis": "Caller-declared; not cryptographic training-code attestation", | |
| "attention_conventions": { | |
| "source_rotary_base": 10000 | |
| if source_revision == LEGACY_REVISION | |
| else 100000, | |
| "target_rotary_base": 100000, | |
| "source_qk_scale_each": 1.0 if source_revision == LEGACY_REVISION else 1.2, | |
| "target_qk_scale_each": 1.2, | |
| }, | |
| "behavioral_mismatches": ( | |
| [ | |
| "RoPE base changes from 10000 to 100000", | |
| "Each normalized Q/K gains a 1.2 multiplier", | |
| ] | |
| if source_revision == LEGACY_REVISION | |
| else [] | |
| ), | |
| "transferred": mapping, | |
| "omitted_source_tensors": sorted(set(normalized) - set(mapping)), | |
| "claim": "Partial matrix initialization only; no equivalence or benefit established", | |
| } | |