File size: 2,945 Bytes
d52c89c d90680b d52c89c d90680b d52c89c | 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 16 17 18 19 20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52 53 54 55 56 57 58 59 60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 | """Versioned research checkpoints, exact single-device RNG/optimizer resume."""
import json
from dataclasses import asdict
from pathlib import Path
import torch
from .network import Config, DuplexModel, factorization, from_legacy
# The one- and two-stream format of 14 November, before text tracks.
LEGACY_FACTORIZATION = "delay-0-1-1-1-1-1-1-1"
def save(path, model, optimizer, step, provenance, sampler):
path = Path(path)
path.parent.mkdir(parents=True, exist_ok=True)
state = {
"schema_version": 3,
"factorization": factorization(model.config),
"config": asdict(model.config),
"model": model.state_dict(),
"optimizer": optimizer.state_dict(),
"step": step,
"provenance": provenance,
"torch_rng": torch.get_rng_state(),
"sampler_rng": sampler.get_state(),
"cuda_rng": torch.cuda.get_rng_state_all() if torch.cuda.is_available() else [],
}
temporary = path.with_suffix(path.suffix + ".tmp")
torch.save(state, temporary)
temporary.replace(path)
def model_from(config, weights, stored):
"""A model from a stored config and weights; 14 November checkpoints convert."""
if stored == LEGACY_FACTORIZATION:
return from_legacy(config, weights)
# Checkpoints before 17 November 2026 have no shift or head fields and load
# with the defaults they were trained under.
config = Config(**config)
if stored != factorization(config):
raise ValueError("Unsupported codebook factorization")
model = DuplexModel(config)
model.load_state_dict(weights, strict=True)
return model
def load(path, device="cpu"):
# weights_only rejects arbitrary pickled classes; only tensors/basic containers are used.
state = torch.load(path, map_location="cpu", weights_only=True)
if state["schema_version"] not in (2, 3):
raise ValueError("Unsupported checkpoint schema")
model = model_from(state["config"], state["model"], state["factorization"])
return model.to(device), state
def load_published(folder, device="cpu"):
"""A checkpoint folder written by `publish.py` (config.json and model.safetensors)."""
from safetensors.torch import load_file
folder = Path(folder)
config = json.loads((folder / "config.json").read_text())
stored = config.pop("factorization")
weights = load_file(folder / "model.safetensors")
return model_from(config, weights, stored).to(device).eval()
def resume(state, optimizer, sampler):
if state["schema_version"] != 3:
raise ValueError("Resume needs a checkpoint of this trainer")
optimizer.load_state_dict(state["optimizer"])
torch.set_rng_state(state["torch_rng"])
sampler.set_state(state["sampler_rng"])
if state["cuda_rng"]:
if not torch.cuda.is_available():
raise ValueError("Exact CUDA resume requires CUDA")
torch.cuda.set_rng_state_all(state["cuda_rng"])
|