alter_ego / serving /src /lora.rs
geraldaton20's picture
initial upload: source code (no build artifacts)
f4f166e verified
Raw History Blame Contribute Delete
7.83 kB
//! LoRA rank-16 adapters over the frozen student base - the ONLY mutable
//! state (PRD section 2). Double-buffered atomic handoff (PRD section 6):
//! inference always reads the ACTIVE buffer; background training mutates a
//! CLONE of the STANDBY buffer, then publishes it (flip) between turns.
use std::collections::HashMap;
use std::sync::{Arc, RwLock};
use ndarray::Array2;
use crate::error::{Error, Result};
use crate::model::{ParamMap, SanConfig};
pub const RANK: usize = 16;
/// Adapted linears per transformer block (packed QKV + output projection).
pub const ADAPTED_KEYS: &[&str] = &["attn.qkv", "attn.o"];
/// One low-rank pair: delta W = B @ A (B starts zero => identity delta).
#[derive(Debug, Clone)]
pub struct LoraPair {
pub a: Array2<f32>, // [rank, in]
pub b: Array2<f32>, // [out, rank]
}
impl LoraPair {
pub fn fresh(out_: usize, in_: usize, rank: usize) -> Self {
Self {
a: random_gauss((rank, in_), 0.02),
b: Array2::zeros((out_, rank)),
}
}
}
fn random_gauss(shape: (usize, usize), scale: f32) -> Array2<f32> {
// Deterministic xorshift so fresh adapters are reproducible across runs.
let n = shape.0 * shape.1;
let mut s: u64 = 0x9E3779B97F4A7C15 ^ (n as u64);
let mut data = Vec::with_capacity(n);
for _ in 0..n {
s ^= s << 13;
s ^= s >> 7;
s ^= s << 17;
let u = ((s >> 40) as f32) / ((1u32 << 24) as f32); // [0,1)
data.push((u * 2.0 - 1.0) * scale);
}
Array2::from_shape_vec(shape, data).expect("shape")
}
/// All adapter tensors for one student architecture.
#[derive(Debug, Clone)]
pub struct Adapter {
pub cfg: SanConfig,
pub pairs: HashMap<String, LoraPair>, // key: "blocks.{i}.{ADAPTED_KEY}"
}
impl Adapter {
pub fn fresh(cfg: &SanConfig) -> Self {
let mut pairs = HashMap::new();
let d = cfg.d_model;
for i in 0..cfg.n_layers {
for k in ADAPTED_KEYS {
let out_ = if *k == "attn.qkv" { 3 * d } else { d };
pairs.insert(format!("blocks.{i}.{k}"), LoraPair::fresh(out_, d, RANK));
}
}
Self { cfg: cfg.clone(), pairs }
}
pub fn from_params(map: &ParamMap, cfg: &SanConfig) -> Result<Self> {
let mut ad = Self::fresh(cfg);
for i in 0..cfg.n_layers {
for k in ADAPTED_KEYS {
let key = format!("blocks.{i}.{k}");
let ka = format!("lora.{key}.A");
let kb = format!("lora.{key}.B");
let a = map.get(&ka).ok_or_else(|| Error::MissingTensor(ka.clone()))?;
let b = map.get(&kb).ok_or_else(|| Error::MissingTensor(kb.clone()))?;
let pair = ad.pairs.get_mut(&key).ok_or_else(|| {
Error::Model(format!("adapter key {key} missing"))
})?;
if a.dim() != pair.a.dim() || b.dim() != pair.b.dim() {
return Err(Error::Shape {
name: key,
expected: vec![pair.a.dim().0, pair.b.dim().1],
got: vec![a.dim().0, b.dim().1],
});
}
pair.a = a.clone();
pair.b = b.clone();
}
}
Ok(ad)
}
pub fn to_params(&self) -> ParamMap {
let mut m = ParamMap::new();
for (key, p) in &self.pairs {
m.insert(format!("lora.{key}.A"), p.a.clone());
m.insert(format!("lora.{key}.B"), p.b.clone());
}
m
}
/// Materialized delta for one adapted linear: [out, in].
pub fn delta(&self, key: &str) -> Option<Array2<f32>> {
self.pairs.get(key).map(|p| p.b.dot(&p.a))
}
/// SGD step on A/B only. grads keyed like `pairs`.
pub fn sgd_step(
&mut self,
grads: &HashMap<String, (Array2<f32>, Array2<f32>)>,
lr: f32,
) {
for (key, (da, db)) in grads {
if let Some(p) = self.pairs.get_mut(key) {
p.a = &p.a - &(da * lr);
p.b = &p.b - &(db * lr);
}
}
}
/// Cheap fingerprint of trainable state (for frozen-base assertions).
pub fn checksum(&self) -> u64 {
let mut h: u64 = 0xcbf29ce484222325;
for key in self.pairs.keys() {
for v in key.bytes() {
h = (h ^ v as u64).wrapping_mul(0x100000001b3);
}
let p = &self.pairs[key];
for m in [&p.a, &p.b] {
for x in m.iter() {
h = (h ^ (*x).to_bits() as u64).wrapping_mul(0x100000001b3);
}
}
}
h
}
pub fn param_count(&self) -> usize {
self.pairs.values().map(|p| p.a.len() + p.b.len()).sum()
}
}
/// Double buffer: index flips atomically under a short write lock; readers
/// clone the Arc, never touch locked state afterwards.
pub struct AdapterHandle {
inner: RwLock<HandleInner>,
}
struct HandleInner {
active: usize,
bufs: [Arc<Adapter>; 2],
}
impl AdapterHandle {
pub fn new(cfg: &SanConfig) -> Self {
Self {
inner: RwLock::new(HandleInner {
active: 0,
bufs: [Arc::new(Adapter::fresh(cfg)), Arc::new(Adapter::fresh(cfg))],
}),
}
}
pub fn from_adapter(ad: Adapter) -> Self {
Self {
inner: RwLock::new(HandleInner {
active: 0,
bufs: [Arc::new(ad), Arc::new(Adapter::fresh(&default_cfg_hint()))],
}),
}
}
pub fn active(&self) -> Arc<Adapter> {
let g = self.inner.read().expect("adapter lock");
g.bufs[g.active].clone()
}
pub fn standby_clone(&self) -> Adapter {
let g = self.inner.read().expect("adapter lock");
(**&g.bufs[1 - g.active]).clone()
}
/// Publish trained state: fill the standby slot and flip.
pub fn publish(&self, trained: Adapter) {
let mut g = self.inner.write().expect("adapter lock");
let slot = 1 - g.active;
g.bufs[slot] = Arc::new(trained);
g.active = slot;
}
}
fn default_cfg_hint() -> SanConfig {
SanConfig {
vocab: 0,
d_model: 0,
n_layers: 0,
ffn_ratio: 1,
n_heads: 1,
max_seq_len: 0,
tied_embeddings: true,
}
}
#[cfg(test)]
mod tests {
use super::*;
#[test]
fn fresh_is_identity_delta() {
let cfg = crate::model::SanConfig {
vocab: 16, d_model: 8, n_layers: 2, ffn_ratio: 2, n_heads: 2,
max_seq_len: 8, tied_embeddings: true,
};
let ad = Adapter::fresh(&cfg);
assert_eq!(ad.param_count(), 1536);
for i in 0..cfg.n_layers {
let d = ad.delta(&format!("blocks.{i}.attn.qkv")).unwrap();
assert!(d.iter().all(|&x| x == 0.0), "B zero => zero delta");
}
}
#[test]
fn handle_flip_visible_to_readers() {
let cfg = crate::model::SanConfig {
vocab: 16, d_model: 8, n_layers: 1, ffn_ratio: 2, n_heads: 2,
max_seq_len: 8, tied_embeddings: true,
};
let h = AdapterHandle::new(&cfg);
let before = h.active().checksum();
let mut tr = h.standby_clone();
let key = "blocks.0.attn.o".to_string();
let dim = tr.pairs[&key].b.dim();
tr.pairs.get_mut(&key).unwrap().b[[dim.0 - 1, dim.1 - 1]] = 0.5;
h.publish(tr);
assert_ne!(h.active().checksum(), before, "flip must expose trained buf");
}
}