alter_ego / serving /src /checkpoint.rs
geraldaton20's picture
initial upload: source code (no build artifacts)
f4f166e verified
Raw History Blame Contribute Delete
9.41 kB
//! Loads a checkpoint in the exact format written by
//! `training/kernels/train.py` (`save_local`): a `meta.json` plus
//! `state_*.bin` shards, each a concatenation of `[json-header]\n[raw-bytes]`
//! blobs, indexed by `meta["assignments"][name] = [shard_index, offset]`.
//!
//! Only model tensors (not `__optim__` / `__rng_torch__`) are loaded; those are
//! training-time state irrelevant to serving inference.
use std::collections::HashMap;
use std::path::Path;
use ndarray::Array2;
use serde::Deserialize;
use sha2::{Digest, Sha256};
use crate::error::{Error, Result};
use crate::model::ParamMap;
#[derive(Debug, Deserialize)]
#[allow(dead_code)]
struct Meta {
step: usize,
tokens: usize,
shards: Vec<String>,
#[serde(default)]
shard_shas: Vec<String>,
assignments: HashMap<String, (usize, usize)>,
tensor_meta: HashMap<String, TensorMeta>,
#[serde(rename = "final", default)]
final_: Option<bool>,
}
#[derive(Debug, Deserialize)]
#[allow(dead_code)]
struct TensorMeta {
shape: Vec<usize>,
dtype: String,
}
#[derive(Debug, Clone)]
pub struct CheckpointMeta {
pub step: usize,
pub tokens: usize,
pub final_checkpoint: bool,
}
fn read_exact(buf: &[u8], off: usize, n: usize) -> Result<&[u8]> {
if off + n > buf.len() {
return Err(Error::Checkpoint(format!(
"read past end: off={off} n={n} len={}",
buf.len()
)));
}
Ok(&buf[off..off + n])
}
fn parse_blob<'a>(buf: &'a [u8], off: usize) -> Result<(&'a [u8], &'a [u8])> {
// header is a JSON line ending at the first '\n'
let nl = buf[off..]
.iter()
.position(|&b| b == b'\n')
.ok_or_else(|| Error::Checkpoint("missing newline in blob header".into()))?;
let header = read_exact(buf, off, nl)?;
let data_start = off + nl + 1;
Ok((header, &buf[data_start..]))
}
pub fn load_checkpoint(dir: &Path, verify: bool) -> Result<(ParamMap, CheckpointMeta)> {
let meta_path = dir.join("meta.json");
let meta_txt = std::fs::read_to_string(&meta_path)
.map_err(|_| Error::Checkpoint("meta.json not found in checkpoint dir".into()))?;
let meta: Meta = serde_json::from_str(&meta_txt)?;
// Read all shard files into memory.
let mut shards: Vec<Vec<u8>> = Vec::with_capacity(meta.shards.len());
for name in &meta.shards {
let bytes = std::fs::read(dir.join(name))?;
if verify {
let mut h = Sha256::new();
h.update(&bytes);
let got = hex::encode(h.finalize());
if let Some(exp) = meta.shard_shas.get(meta.shards.iter().position(|n| n == name).unwrap()) {
if &got != exp {
return Err(Error::Checkpoint(format!(
"shard {name} sha mismatch (corrupt/partial)"
)));
}
}
}
shards.push(bytes);
}
let mut params: ParamMap = HashMap::new();
for (name, (shard_idx, offset)) in &meta.assignments {
if name.starts_with("__") {
continue; // __optim__ / __rng_torch__ — training state, skip
}
let buf = shards
.get(*shard_idx)
.ok_or_else(|| Error::Checkpoint(format!("shard index {shard_idx} out of range")))?;
let (header, rest) = parse_blob(buf, *offset)?;
let hm: serde_json::Value = serde_json::from_slice(header)?;
let nbytes = hm["nbytes"].as_u64().unwrap() as usize;
let shape: Vec<usize> = serde_json::from_value(hm["shape"].clone())?;
let dtype = hm["dtype"].as_str().unwrap_or("float32");
let raw = read_exact(rest, 0, nbytes)?;
let data = match dtype {
"float32" => f32_slice(raw)?,
"float64" => f64_slice(raw)?.into_iter().map(|v| v as f32).collect(),
other => return Err(Error::Dtype(other.into())),
};
// Normalize to a 2-D array.
let arr_shape: (usize, usize) = match shape.len() {
1 => (1, shape[0]),
2 => (shape[0], shape[1]),
3 => (shape[1], shape[2]), // drop leading 1 (e.g. pos [1,maxlen,d])
_ => {
return Err(Error::Checkpoint(format!(
"unsupported ndim {} for tensor {name}",
shape.len()
)))
}
};
let arr = Array2::from_shape_vec(arr_shape, data)
.map_err(|e| Error::Checkpoint(format!("reshape {name}: {e}")))?;
params.insert(name.clone(), arr);
}
let cmeta = CheckpointMeta {
step: meta.step,
tokens: meta.tokens,
final_checkpoint: meta.final_.unwrap_or(false),
};
Ok((params, cmeta))
}
fn f32_slice(raw: &[u8]) -> Result<Vec<f32>> {
if raw.len() % 4 != 0 {
return Err(Error::Checkpoint("float32 blob length not multiple of 4".into()));
}
Ok(raw
.chunks_exact(4)
.map(|c| f32::from_le_bytes([c[0], c[1], c[2], c[3]]))
.collect())
}
fn f64_slice(raw: &[u8]) -> Result<Vec<f64>> {
if raw.len() % 8 != 0 {
return Err(Error::Checkpoint("float64 blob length not multiple of 8".into()));
}
Ok(raw
.chunks_exact(8)
.map(|c| f64::from_le_bytes([c[0], c[1], c[2], c[3], c[4], c[5], c[6], c[7]]))
.collect())
}
#[cfg(test)]
mod tests {
use super::*;
use crate::model::{ParamMap, San, SanConfig};
use ndarray::Array2;
use sha2::Sha256;
fn arr(r: usize, c: usize) -> Array2<f32> {
Array2::from_shape_vec((r, c), vec![0.01; r * c]).unwrap()
}
fn tiny_cfg() -> SanConfig {
SanConfig {
vocab: 10,
d_model: 4,
n_layers: 1,
ffn_ratio: 2,
n_heads: 2,
max_seq_len: 8,
tied_embeddings: true,
}
}
/// A minimal but complete set of tensors (attention is bias-free, like the
/// real trained model).
fn tiny_params() -> ParamMap {
let d = 4;
let ffn = 8;
let v = 10;
let maxlen = 8;
let mut p: ParamMap = HashMap::new();
p.insert("tok_emb.weight".into(), arr(v, d));
p.insert("pos".into(), arr(maxlen, d));
p.insert("head.weight".into(), arr(v, d));
p.insert("blocks.0.ln1.weight".into(), arr(1, d));
p.insert("blocks.0.ln1.bias".into(), arr(1, d));
p.insert("blocks.0.attn.qkv.weight".into(), arr(3 * d, d));
p.insert("blocks.0.attn.o.weight".into(), arr(d, d));
p.insert("blocks.0.ln2.weight".into(), arr(1, d));
p.insert("blocks.0.ln2.bias".into(), arr(1, d));
p.insert("blocks.0.mlp.0.weight".into(), arr(ffn, d));
p.insert("blocks.0.mlp.0.bias".into(), arr(1, ffn));
p.insert("blocks.0.mlp.2.weight".into(), arr(d, ffn));
p.insert("blocks.0.mlp.2.bias".into(), arr(1, d));
p.insert("ln_f.weight".into(), arr(1, d));
p.insert("ln_f.bias".into(), arr(1, d));
p
}
/// Serialize params in the exact `train.py` save_local format (so this test
/// also guards against drift between the two implementations).
fn write_checkpoint(dir: &Path, params: &ParamMap) {
let mut shard: Vec<u8> = Vec::new();
let mut assignments = serde_json::Map::new();
let mut tensor_meta = serde_json::Map::new();
for (name, a) in params {
let (r, c) = a.dim();
let nbytes = r * c * 4;
let header = serde_json::json!({
"name": name, "shape": [r, c], "dtype": "float32", "nbytes": nbytes
})
.to_string();
let offset = shard.len();
shard.extend_from_slice(header.as_bytes());
shard.push(b'\n');
for &v in a.iter() {
shard.extend_from_slice(&v.to_le_bytes());
}
assignments.insert(name.clone(), serde_json::json!([0, offset]));
tensor_meta.insert(name.clone(), serde_json::json!({ "shape": [r, c], "dtype": "float32" }));
}
std::fs::write(dir.join("state_000.bin"), &shard).unwrap();
let sha = {
let mut h = Sha256::new();
h.update(&shard);
hex::encode(h.finalize())
};
let meta = serde_json::json!({
"step": 1,
"tokens": 100,
"shards": ["state_000.bin"],
"shard_shas": [sha],
"assignments": assignments,
"tensor_meta": tensor_meta,
"qat_enabled": false,
"final": false,
"created": 0,
});
std::fs::write(dir.join("meta.json"), meta.to_string()).unwrap();
}
#[test]
fn roundtrip_load_and_forward() {
let tmp = std::env::temp_dir().join("ae_ckpt_test");
let _ = std::fs::remove_dir_all(&tmp);
std::fs::create_dir_all(&tmp).unwrap();
let params = tiny_params();
write_checkpoint(&tmp, &params);
let (loaded, meta) = load_checkpoint(&tmp, true).unwrap();
assert_eq!(loaded.len(), params.len(), "param count mismatch");
let emb = loaded.get("tok_emb.weight").unwrap();
assert_eq!(emb.dim(), (10, 4));
assert_eq!(meta.step, 1);
let san = San::from_params(loaded, tiny_cfg()).unwrap();
let logits = san.forward(&[1, 2, 3]).unwrap();
assert_eq!(logits.dim(), (3, 10));
}
}