DuoNeural's picture
Upload CRSN-8L-Phase2-Extension 102.4M tokens checkpoint and model card
c46d5c2 verified
Raw History Blame Contribute Delete
24.3 kB
#!/usr/bin/env python3
"""
CRSN (Clifford Resonant Sheaf Network) Production Pretraining Daemon
Hardware Target: AMD Ryzen 5 3550H (4 Cores / Zen+ AVX2 SIMD)
Architecture:
- Clifford Resonant Sheaf Coupling (8 Layers, D=384, D_state=192, D_ff=512)
- Reversible RevNet Trunk with Contractive Phase-Space Adjoint
- Softplus Stiffness Floor: K(q) >= 1e-4 (Eliminates negative bifurcation)
- Sequence-Normalized Adjoint Integration (1/T)
- Dual Optimizer: Muon (2D matrices) + Segregated AdamW (Embeddings & Resonance Bank)
"""
import os
import sys
import time
import math
import psutil
import resource
import argparse
import numpy as np
import torch
import torch.nn as nn
import torch.nn.functional as F
from torch.utils.cpp_extension import load
from tokenizers import Tokenizer
# ==============================================================================
# 0. HARDWARE THREAD PINNING (AMD RYZEN 5 3550H: 4 PHYSICAL CORES)
# ==============================================================================
os.environ["OMP_NUM_THREADS"] = "4"
os.environ["MKL_NUM_THREADS"] = "4"
torch.set_num_threads(4)
torch.set_num_interop_threads(1)
def get_peak_rss_mb():
usage = resource.getrusage(resource.RUSAGE_SELF).ru_maxrss
return usage / 1024.0
def get_current_rss_mb():
return psutil.Process().memory_info().rss / (1024.0 * 1024.0)
# ==============================================================================
# 1. LOAD NATIVE C++ OPENMP / AVX2 SIMD KERNEL
# ==============================================================================
current_dir = os.path.dirname(os.path.abspath(__file__))
cpp_src = os.path.join(current_dir, "crsn_kernel.cpp")
crsn_cpp = load(
name="crsn_cpp",
sources=[cpp_src],
extra_cflags=["-O3", "-march=native", "-mavx2", "-mfma", "-fopenmp", "-ffast-math", "-fno-math-errno"],
extra_ldflags=["-fopenmp"],
verbose=False
)
class CRSNCppFunction(torch.autograd.Function):
@staticmethod
def forward(ctx, F_ext, gamma, omega_sq, alpha, dt):
outputs, q_hist, p_hist = crsn_cpp.forward(F_ext, gamma, omega_sq, alpha, float(dt))
ctx.save_for_backward(gamma, q_hist, p_hist, omega_sq, alpha)
ctx.dt = dt
return outputs
@staticmethod
def backward(ctx, grad_out):
gamma, q_hist, p_hist, omega_sq, alpha = ctx.saved_tensors
dF_ext, dgamma, domega_sq, dalpha = crsn_cpp.backward(
grad_out, gamma, q_hist, p_hist, omega_sq, alpha, float(ctx.dt)
)
return dF_ext, dgamma, domega_sq, dalpha, None
# ==============================================================================
# 2. ZERO-COPY UINT16 MEMMAP DATASTREAMER
# ==============================================================================
class MemmapDataset:
def __init__(self, bin_path: str, seq_len: int = 256):
if not os.path.exists(bin_path):
raise FileNotFoundError(f"Binary file not found: {bin_path}. Run prepare_data.py first!")
self.data = np.memmap(bin_path, dtype=np.uint16, mode='r')
self.seq_len = seq_len
self.num_tokens = len(self.data)
if self.num_tokens <= seq_len + 1:
raise ValueError(f"Dataset in {bin_path} has only {self.num_tokens} tokens, need > {seq_len + 1}")
def sample_batch(self, batch_size: int, device: torch.device):
ix = np.random.randint(0, self.num_tokens - self.seq_len - 1, size=batch_size)
batch = np.stack([self.data[i : i + self.seq_len + 1].astype(np.int64) for i in ix])
t_batch = torch.from_numpy(batch).to(device)
return t_batch[:, :-1].contiguous(), t_batch[:, 1:].contiguous()
# ==============================================================================
# 3. OPTIMIZER: NEWTON-SCHULZ ORTHOGONALIZED MOMENTUM (MUON)
# ==============================================================================
def zeropower_via_newtonschulz5(G: torch.Tensor, steps: int = 5, eps: float = 1e-7) -> torch.Tensor:
assert G.ndim == 2
a, b, c = 3.4445, -4.7750, 2.0315
X = G.float()
norm = X.norm() + eps
X = X / norm
transposed = X.size(0) > X.size(1)
if transposed:
X = X.T
for _ in range(steps):
A = torch.mm(X, X.T)
B = b * A + c * torch.mm(A, A)
X = a * X + torch.mm(B, X)
if transposed:
X = X.T
return X.to(dtype=G.dtype)
class Muon(torch.optim.Optimizer):
def __init__(self, params, lr: float = 0.02, momentum: float = 0.95, nesterov: bool = True, weight_decay: float = 0.0):
defaults = dict(lr=lr, momentum=momentum, nesterov=nesterov, weight_decay=weight_decay)
super().__init__(params, defaults)
@torch.no_grad()
def step(self):
for group in self.param_groups:
lr = group['lr']
momentum = group['momentum']
nesterov = group['nesterov']
weight_decay = group['weight_decay']
for p in group['params']:
if p.grad is None:
continue
g = p.grad
if weight_decay != 0:
g = g.add(p, alpha=weight_decay)
state = self.state[p]
if 'momentum_buffer' not in state:
buf = state['momentum_buffer'] = torch.zeros_like(g)
else:
buf = state['momentum_buffer']
buf.mul_(momentum).add_(g)
if nesterov:
g = g.add(buf, alpha=momentum)
else:
g = buf
u = zeropower_via_newtonschulz5(g)
p.data.add_(u, alpha=-lr)
# ==============================================================================
# 4. ARCHITECTURE: CLIFFORD RESONANT SHEAF NETWORK (CRSN)
# ==============================================================================
class RMSNorm(nn.Module):
def __init__(self, dim: int, eps: float = 1e-6):
super().__init__()
self.eps = eps
self.weight = nn.Parameter(torch.ones(dim))
def forward(self, x: torch.Tensor) -> torch.Tensor:
variance = x.pow(2).mean(-1, keepdim=True)
return x * torch.rsqrt(variance + self.eps) * self.weight
class CliffordResonantCell(nn.Module):
def __init__(self, d_state: int, n_layers: int = 8, dt: float = 0.1):
super().__init__()
self.d_state = d_state
self.dt = dt
self.norm = RMSNorm(d_state)
self.w_force = nn.Linear(d_state, d_state, bias=False)
self.w_damp = nn.Linear(d_state, d_state, bias=True)
self.w_out = nn.Linear(d_state, d_state, bias=False)
nn.init.normal_(self.w_out.weight, mean=0.0, std=0.02 / math.sqrt(2 * n_layers))
taus = torch.exp(torch.linspace(math.log(2.0), math.log(1024.0), d_state))
self.omega_sq = nn.Parameter((2.0 * math.pi / taus) ** 2)
# Softplus parameterization guarantees alpha >= 0 (no saddle bifurcation)
self.alpha_raw = nn.Parameter(torch.ones(d_state) * -2.0)
@property
def alpha(self):
return F.softplus(self.alpha_raw) + 1e-4
def forward(self, x: torch.Tensor) -> torch.Tensor:
x_norm = self.norm(x)
F_ext = self.w_force(x_norm)
# Bounded non-zero dissipation
gamma = torch.clamp(torch.sigmoid(self.w_damp(x_norm)), min=0.01, max=0.99)
# Bounded stiffness spectrum protecting against resonance singularities
omega_sq = torch.clamp(torch.abs(self.omega_sq), min=1e-4, max=100.0)
y = CRSNCppFunction.apply(F_ext, gamma, omega_sq, self.alpha, self.dt)
return self.w_out(y)
class SwiGLU(nn.Module):
def __init__(self, d_state: int, d_ff: int, n_layers: int = 8):
super().__init__()
self.norm = RMSNorm(d_state)
self.w_gate = nn.Linear(d_state, d_ff, bias=False)
self.w_up = nn.Linear(d_state, d_ff, bias=False)
self.w_down = nn.Linear(d_ff, d_state, bias=False)
nn.init.normal_(self.w_down.weight, mean=0.0, std=0.02 / math.sqrt(2 * n_layers))
def forward(self, x: torch.Tensor) -> torch.Tensor:
x_norm = self.norm(x)
return self.w_down(F.silu(self.w_gate(x_norm)) * self.w_up(x_norm))
class ReversibleBlockFunction(torch.autograd.Function):
@staticmethod
def forward(ctx, y1: torch.Tensor, y2: torch.Tensor, f_module: nn.Module, g_module: nn.Module, scale: float = 0.70710678):
ctx.f_module = f_module
ctx.g_module = g_module
ctx.scale = scale
with torch.no_grad():
f_out = f_module(y2) * scale
y1_next = y1 + f_out
g_out = g_module(y1_next) * scale
y2_next = y2 + g_out
ctx.save_for_backward(y1_next.detach(), y2_next.detach())
return y1_next, y2_next
@staticmethod
def backward(ctx, dy1_next: torch.Tensor, dy2_next: torch.Tensor):
y1_next, y2_next = ctx.saved_tensors
f_module = ctx.f_module
g_module = ctx.g_module
scale = ctx.scale
# Reconstruct y2 and backprop through G
with torch.enable_grad():
y1_next_eval = y1_next.detach().requires_grad_(True)
g_eval = g_module(y1_next_eval) * scale
g_eval.backward(dy2_next)
dy1_from_g = y1_next_eval.grad
y2 = y2_next - g_eval.detach()
dy1 = dy1_next + dy1_from_g
# Reconstruct y1 and backprop through F
with torch.enable_grad():
y2_eval = y2.detach().requires_grad_(True)
f_eval = f_module(y2_eval) * scale
f_eval.backward(dy1)
dy2_from_f = y2_eval.grad
y1 = y1_next - f_eval.detach()
dy2 = dy2_next + dy2_from_f
return dy1, dy2, None, None, None
class ReversibleBlock(nn.Module):
def __init__(self, d_state: int, d_ff: int, n_layers: int = 8):
super().__init__()
self.f_module = CliffordResonantCell(d_state=d_state, n_layers=n_layers)
self.g_module = SwiGLU(d_state=d_state, d_ff=d_ff, n_layers=n_layers)
self.scale = 1.0 / math.sqrt(2.0) # 0.7071 residual scale
def forward(self, y1: torch.Tensor, y2: torch.Tensor):
return ReversibleBlockFunction.apply(y1, y2, self.f_module, self.g_module, self.scale)
class CRSNLanguageModel(nn.Module):
def __init__(self, vocab_size: int = 16384, d_model: int = 384, n_layers: int = 8):
super().__init__()
self.vocab_size = vocab_size
self.d_model = d_model
self.d_state = d_model // 2 # 192 dims = 24 x 8 Clifford bivectors
self.d_ff = 512 # Exact balanced SwiGLU ratio
self.n_layers = n_layers
self.embedding = nn.Embedding(vocab_size, d_model)
nn.init.normal_(self.embedding.weight, mean=0.0, std=0.02)
self.layers = nn.ModuleList([
ReversibleBlock(d_state=self.d_state, d_ff=self.d_ff, n_layers=n_layers)
for _ in range(n_layers)
])
self.final_norm = RMSNorm(d_model)
def forward_features(self, input_ids: torch.Tensor) -> torch.Tensor:
x = self.embedding(input_ids)
y1, y2 = torch.chunk(x, chunks=2, dim=-1)
for layer in self.layers:
y1, y2 = layer(y1, y2)
out = torch.cat([y1, y2], dim=-1)
return self.final_norm(out)
def forward(self, input_ids: torch.Tensor, targets: torch.Tensor = None, chunk_tokens: int = 512):
out = self.forward_features(input_ids)
if targets is None:
return F.linear(out, self.embedding.weight)
flat_out = out.view(-1, self.d_model)
flat_targets = targets.view(-1)
total_tokens = flat_out.size(0)
total_loss = 0.0
for start in range(0, total_tokens, chunk_tokens):
end = min(start + chunk_tokens, total_tokens)
logits_chunk = F.linear(flat_out[start:end], self.embedding.weight)
loss_chunk = F.cross_entropy(logits_chunk, flat_targets[start:end], reduction='sum')
total_loss = total_loss + loss_chunk
return total_loss / total_tokens
# ==============================================================================
# 5. AUTOREGRESSIVE GENERATION
# ==============================================================================
@torch.no_grad()
def generate(model: CRSNLanguageModel, tokenizer, prompt: str, max_new_tokens: int = 48, temperature: float = 0.8, top_k: int = 40):
model.eval()
input_ids = tokenizer.encode(prompt).ids
input_tensor = torch.tensor([input_ids], dtype=torch.long)
for _ in range(max_new_tokens):
curr_input = input_tensor if input_tensor.size(1) <= 512 else input_tensor[:, -512:]
logits = model(curr_input)
next_token_logits = logits[:, -1, :] / max(1e-5, temperature)
if top_k > 0:
v, _ = torch.topk(next_token_logits, min(top_k, next_token_logits.size(-1)))
next_token_logits[next_token_logits < v[:, [-1]]] = -float('Inf')
probs = F.softmax(next_token_logits, dim=-1)
next_token = torch.multinomial(probs, num_samples=1)
input_tensor = torch.cat([input_tensor, next_token], dim=1)
if next_token.item() == tokenizer.token_to_id("</s>"):
break
model.train()
generated_ids = input_tensor[0].tolist()
return tokenizer.decode(generated_ids)
# ==============================================================================
# 6. PRODUCTION TRAINING LOOP
# ==============================================================================
def get_lr_factor(step, warmup_steps, total_steps, min_ratio=0.1):
if step < warmup_steps:
return max(1e-4, float(step) / float(max(1, warmup_steps)))
progress = float(step - warmup_steps) / float(max(1, total_steps - warmup_steps))
progress = min(1.0, max(0.0, progress))
cosine = 0.5 * (1.0 + math.cos(math.pi * progress))
return min_ratio + (1.0 - min_ratio) * cosine
def train_production():
p = argparse.ArgumentParser()
p.add_argument("--data-dir", default="/home/duoneural/srsn_ryzen")
p.add_argument("--train-bin", default=None)
p.add_argument("--val-bin", default=None)
p.add_argument("--batch-size", type=int, default=8)
p.add_argument("--seq-len", type=int, default=256)
p.add_argument("--grad-accum-steps", type=int, default=4)
p.add_argument("--total-optimizer-steps", type=int, default=2500)
p.add_argument("--warmup-steps", type=int, default=25)
p.add_argument("--muon-lr", type=float, default=0.02)
p.add_argument("--adam-lr", type=float, default=1e-3)
p.add_argument("--save-interval", type=int, default=25)
p.add_argument("--eval-interval", type=int, default=25)
p.add_argument("--ckpt-dir", default=None, help="Directory to save checkpoints")
p.add_argument("--resume", default=None, help="Path to checkpoint to resume training from")
args = p.parse_args()
device = torch.device("cpu")
print(f"\n==============================================================================", flush=True)
print(f"[Init] CRSN Production Pre-Training (AMD Ryzen 5 3550H / Zen+ AVX2)", flush=True)
print(f" Batch={args.batch_size}, SeqLen={args.seq_len}, Accum={args.grad_accum_steps}", flush=True)
print(f" Tokens/OptStep: {args.batch_size * args.seq_len * args.grad_accum_steps:,}", flush=True)
print(f"==============================================================================", flush=True)
tok_path = os.path.join(args.data_dir, "tokenizer_16k.json")
# Resolve train/val dataset paths
train_bin = args.train_bin or ("/dev/shm/train_golden.bin" if os.path.exists("/dev/shm/train_golden.bin") else os.path.join(args.data_dir, "train_golden.bin"))
val_bin = args.val_bin or ("/dev/shm/val_golden.bin" if os.path.exists("/dev/shm/val_golden.bin") else os.path.join(args.data_dir, "val_golden.bin"))
if not os.path.exists(tok_path):
raise FileNotFoundError(f"Tokenizer not found at {tok_path}!")
tokenizer = Tokenizer.from_file(tok_path)
vocab_size = tokenizer.get_vocab_size()
print(f"[Data] Training Dataset: {train_bin} ({os.path.getsize(train_bin)/(1024*1024):.2f} MB)")
print(f"[Data] Validation Dataset: {val_bin} ({os.path.getsize(val_bin)/1024:.2f} KB)")
train_ds = MemmapDataset(train_bin, seq_len=args.seq_len)
val_ds = MemmapDataset(val_bin, seq_len=args.seq_len)
print(f"[Data] Loaded memmap dataset: Train={train_ds.num_tokens:,} tokens | Val={val_ds.num_tokens:,} tokens", flush=True)
model = CRSNLanguageModel(vocab_size=vocab_size, d_model=384, n_layers=8).to(device)
total_params = sum(p.numel() for p in model.parameters())
embed_params = sum(p.numel() for p in model.embedding.parameters())
layer_params = sum(p.numel() for p in model.layers.parameters())
print(f"[Model] 16k Vocab CRSN-Compact Parameter Breakdown:")
print(f" - Embedding Table ({vocab_size} x 384): {embed_params / 1e6:.2f}M ({embed_params:,} params)")
print(f" - 8 RevNet Trunk Layers: {layer_params / 1e6:.2f}M ({layer_params:,} params)")
print(f" - Total Tied Parameter Footprint: {total_params / 1e6:.2f}M ({total_params:,} params)", flush=True)
# Segregate parameter groups
muon_params = [p for n, p in model.named_parameters() if p.ndim == 2 and "embedding" not in n]
adam_decay_params = [p for n, p in model.named_parameters() if "embedding" in n]
adam_no_decay_params = [p for n, p in model.named_parameters() if p.ndim != 2 and "embedding" not in n]
print(f"[Optim] Parameter Segregation:")
print(f" - Muon 2D Matrices (weight_decay=0.01): {sum(p.numel() for p in muon_params) / 1e6:.2f}M ({len(muon_params)} tensors)")
print(f" - AdamW Decay (Embedding, wd=0.01): {sum(p.numel() for p in adam_decay_params) / 1e6:.2f}M ({len(adam_decay_params)} tensors)")
print(f" - AdamW No-Decay (omega_sq, alpha, norms): {sum(p.numel() for p in adam_no_decay_params):,} params ({len(adam_no_decay_params)} tensors)", flush=True)
optimizer_muon = Muon(muon_params, lr=args.muon_lr, momentum=0.95, weight_decay=0.01)
optimizer_adam = torch.optim.AdamW([
{"params": adam_decay_params, "weight_decay": 0.01},
{"params": adam_no_decay_params, "weight_decay": 0.0}
], lr=args.adam_lr, betas=(0.9, 0.95))
ckpt_dir = args.ckpt_dir or os.path.join(args.data_dir, "checkpoints_crsn")
os.makedirs(ckpt_dir, exist_ok=True)
opt_step = 0
total_micro_steps = args.total_optimizer_steps * args.grad_accum_steps
best_val_loss = float("inf")
if args.resume and os.path.exists(args.resume):
print(f"[Resume] Loading weights from {args.resume}...", flush=True)
ckpt = torch.load(args.resume, map_location="cpu", weights_only=False)
sd = ckpt.get("model_state", ckpt.get("model", ckpt))
model.load_state_dict(sd, strict=False)
if "loss" in ckpt:
best_val_loss = ckpt["loss"]
print(f"[Resume] Initialized best_val_loss to {best_val_loss:.4f}", flush=True)
t0 = time.perf_counter()
print(f"[Run] Launching CRSN Pretraining: Steps 1 -> {args.total_optimizer_steps} ({total_micro_steps} micro-steps)...", flush=True)
model.train()
optimizer_muon.zero_grad()
optimizer_adam.zero_grad()
for m_step in range(1, total_micro_steps + 1):
step_t0 = time.perf_counter()
inputs, targets = train_ds.sample_batch(args.batch_size, device)
loss = model(inputs, targets=targets)
if not torch.isfinite(loss):
print(f" [GUARD] Non-finite loss ({loss.item()}) at uStep {m_step:04d}! Skipping backward pass.", flush=True)
continue
(loss / args.grad_accum_steps).backward()
step_dt = time.perf_counter() - step_t0
tok_s = (args.batch_size * args.seq_len) / max(1e-5, step_dt)
if m_step % args.grad_accum_steps == 0:
opt_step += 1
lr_scale = get_lr_factor(opt_step, args.warmup_steps, args.total_optimizer_steps)
for g in optimizer_muon.param_groups:
g['lr'] = args.muon_lr * lr_scale
for g in optimizer_adam.param_groups:
g['lr'] = args.adam_lr * lr_scale
# Segregated Gradient Clipping
norm_muon = torch.nn.utils.clip_grad_norm_(muon_params, max_norm=1.0)
norm_embed = torch.nn.utils.clip_grad_norm_(adam_decay_params, max_norm=1.0)
norm_reson = torch.nn.utils.clip_grad_norm_(adam_no_decay_params, max_norm=0.5)
comp_norm = math.sqrt(norm_muon.item()**2 + norm_embed.item()**2 + norm_reson.item()**2)
if not math.isfinite(comp_norm):
print(f" [GUARD] Non-finite GradNorm ({comp_norm:.3f}) at Step {opt_step:03d}! Skipping update.", flush=True)
optimizer_muon.zero_grad()
optimizer_adam.zero_grad()
continue
optimizer_muon.step()
optimizer_adam.step()
optimizer_muon.zero_grad()
optimizer_adam.zero_grad()
rss = get_current_rss_mb()
print(f"Step {opt_step:03d}/{args.total_optimizer_steps:03d} (uStep {m_step:04d}) | Loss: {loss.item():.4f} | GradNorm: {comp_norm:.3f} (M:{norm_muon.item():.2f}, E:{norm_embed.item():.2f}, R:{norm_reson.item():.2f}) | LR: {lr_scale:.3f} | Speed: {tok_s:6.1f} tok/s | RSS: {rss:5.1f} MB", flush=True)
# Periodic Validation & Generation
if opt_step % args.eval_interval == 0 or opt_step == args.total_optimizer_steps:
model.eval()
val_losses = []
with torch.no_grad():
for _ in range(4):
v_in, v_tgt = val_ds.sample_batch(args.batch_size, device)
v_loss = model(v_in, targets=v_tgt)
val_losses.append(v_loss.item())
avg_val_loss = float(np.mean(val_losses))
model.train()
sample_story = generate(model, tokenizer, prompt="Once upon a time, Lily", max_new_tokens=48, temperature=0.8)
sample_reason = generate(model, tokenizer, prompt="User: Is a cat an animal?\n\nAssistant:", max_new_tokens=32, temperature=0.8)
print(f" --> [Validation] Val Loss: {avg_val_loss:.4f} (ppl={math.exp(min(20, avg_val_loss)):.1f})", flush=True)
print(f" --> [Sample Story] \"{sample_story.strip()}\"", flush=True)
print(f" --> [Sample Reason] \"{sample_reason.strip()}\"", flush=True)
# Save best checkpoint
if avg_val_loss < best_val_loss:
best_val_loss = avg_val_loss
best_path = os.path.join(ckpt_dir, "crsn_best.pt")
torch.save({"opt_step": opt_step, "loss": avg_val_loss, "model_state": model.state_dict()}, best_path)
print(f" --> [Checkpoint] Saved NEW BEST checkpoint to {best_path}!", flush=True)
# Periodic Checkpoint
if opt_step % args.save_interval == 0 or opt_step == args.total_optimizer_steps:
ckpt_path = os.path.join(ckpt_dir, f"crsn_step{opt_step:05d}.pt")
torch.save({"opt_step": opt_step, "loss": loss.item(), "model_state": model.state_dict()}, ckpt_path)
print(f" --> [Checkpoint] Saved atomic checkpoint to {ckpt_path}", flush=True)
total_time = time.perf_counter() - t0
total_tokens = args.total_optimizer_steps * args.batch_size * args.seq_len * args.grad_accum_steps
print(f"\n==============================================================================")
print(f"[Complete] CRSN Pre-training Finished!")
print(f" Total Tokens: {total_tokens:,}")
print(f" Total Wall Time: {total_time:.2f}s ({total_time/60:.2f} mins)")
print(f" Average Speed: {total_tokens / total_time:.2f} tokens/sec")
print(f" Best Val Loss: {best_val_loss:.4f}")
print(f"==============================================================================", flush=True)
if __name__ == "__main__":
train_production()