Safetensors
English
File size: 2,945 Bytes
d52c89c
 
 
 
 
 
 
 
d90680b
d52c89c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
d90680b
d52c89c
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
"""Versioned research checkpoints, exact single-device RNG/optimizer resume."""

import json
from dataclasses import asdict
from pathlib import Path

import torch

from .network import Config, DuplexModel, factorization, from_legacy

# The one- and two-stream format of 14 November, before text tracks.
LEGACY_FACTORIZATION = "delay-0-1-1-1-1-1-1-1"


def save(path, model, optimizer, step, provenance, sampler):
    path = Path(path)
    path.parent.mkdir(parents=True, exist_ok=True)
    state = {
        "schema_version": 3,
        "factorization": factorization(model.config),
        "config": asdict(model.config),
        "model": model.state_dict(),
        "optimizer": optimizer.state_dict(),
        "step": step,
        "provenance": provenance,
        "torch_rng": torch.get_rng_state(),
        "sampler_rng": sampler.get_state(),
        "cuda_rng": torch.cuda.get_rng_state_all() if torch.cuda.is_available() else [],
    }
    temporary = path.with_suffix(path.suffix + ".tmp")
    torch.save(state, temporary)
    temporary.replace(path)


def model_from(config, weights, stored):
    """A model from a stored config and weights; 14 November checkpoints convert."""
    if stored == LEGACY_FACTORIZATION:
        return from_legacy(config, weights)
    # Checkpoints before 17 November 2026 have no shift or head fields and load
    # with the defaults they were trained under.
    config = Config(**config)
    if stored != factorization(config):
        raise ValueError("Unsupported codebook factorization")
    model = DuplexModel(config)
    model.load_state_dict(weights, strict=True)
    return model


def load(path, device="cpu"):
    # weights_only rejects arbitrary pickled classes; only tensors/basic containers are used.
    state = torch.load(path, map_location="cpu", weights_only=True)
    if state["schema_version"] not in (2, 3):
        raise ValueError("Unsupported checkpoint schema")
    model = model_from(state["config"], state["model"], state["factorization"])
    return model.to(device), state


def load_published(folder, device="cpu"):
    """A checkpoint folder written by `publish.py` (config.json and model.safetensors)."""
    from safetensors.torch import load_file

    folder = Path(folder)
    config = json.loads((folder / "config.json").read_text())
    stored = config.pop("factorization")
    weights = load_file(folder / "model.safetensors")
    return model_from(config, weights, stored).to(device).eval()


def resume(state, optimizer, sampler):
    if state["schema_version"] != 3:
        raise ValueError("Resume needs a checkpoint of this trainer")
    optimizer.load_state_dict(state["optimizer"])
    torch.set_rng_state(state["torch_rng"])
    sampler.set_state(state["sampler_rng"])
    if state["cuda_rng"]:
        if not torch.cuda.is_available():
            raise ValueError("Exact CUDA resume requires CUDA")
        torch.cuda.set_rng_state_all(state["cuda_rng"])