unified-v1-blt-none-s42: state and inference synchronized at checkpoint-000000000000

experiments.csv

@@ -14,3 +14,4 @@ unified-v1-evabyte-none-s42,ablation,causal,evabyte,none,42,25,0,running,fresh_m
 unified-v1-evabyte-none-s42,ablation,causal,evabyte,none,42,50,0,running,fresh_matched_seed,3644,3644,1.026456674999963,0.0,10668800,1024,3.0752868056297302,6.275834560394287,2420984,20.154760339000404,120119.71163533424,0.12007386237382889,0,bf16,0,269ba207e744573779747b593ffbcd8a87683fd55c83f6b168c658206d9aeb98,af33c522ea4349d5aae0491f7fe085d4bea1b044b08ff57eb064354bbb96435f,"{""climbmix"": {""documents_emitted"": 530, ""epoch"": 0, ""prepared_position"": 1664, ""rows_read"": 530, ""shard"": 0, ""shards"": 6543}, ""rewrite6"": {""documents_emitted"": 214, ""epoch"": 0, ""prepared_position"": 273, ""rows_read"": 215, ""shard"": 0, ""shards"": 1}, ""ultra_qa"": {""documents_emitted"": 109, ""epoch"": 0, ""prepared_position"": 475, ""rows_read"": 110, ""shard"": 0, ""shards"": 616}, ""ultra_style"": {""documents_emitted"": 136, ""epoch"": 0, ""prepared_position"": 481, ""rows_read"": 137, ""shard"": 0, ""shards"": 552}}"
 unified-v1-evabyte-none-s42,ablation,causal,evabyte,none,42,75,0,running,fresh_matched_seed,5450,5450,1.026456674999963,0.0,10668800,1024,2.74536669254303,6.275834560394287,3637662,29.178130834000626,124670.83723406687,0.1350708305835724,0,bf16,0,269ba207e744573779747b593ffbcd8a87683fd55c83f6b168c658206d9aeb98,af33c522ea4349d5aae0491f7fe085d4bea1b044b08ff57eb064354bbb96435f,"{""climbmix"": {""documents_emitted"": 750, ""epoch"": 0, ""prepared_position"": 2482, ""rows_read"": 751, ""shard"": 0, ""shards"": 6543}, ""rewrite6"": {""documents_emitted"": 307, ""epoch"": 0, ""prepared_position"": 405, ""rows_read"": 308, ""shard"": 0, ""shards"": 1}, ""ultra_qa"": {""documents_emitted"": 163, ""epoch"": 0, ""prepared_position"": 712, ""rows_read"": 164, ""shard"": 0, ""shards"": 616}, ""ultra_style"": {""documents_emitted"": 204, ""epoch"": 0, ""prepared_position"": 720, ""rows_read"": 205, ""shard"": 0, ""shards"": 552}}"
 unified-v1-evabyte-none-s42,ablation,causal,evabyte,none,42,100,0,complete,fresh_matched_seed,7230,7230,1.026456674999963,0.0,10668800,1024,2.70014488697052,5.0743002961663635,4841990,38.114180579000845,127039.06856829327,0.1096256747841835,0,bf16,0,269ba207e744573779747b593ffbcd8a87683fd55c83f6b168c658206d9aeb98,af33c522ea4349d5aae0491f7fe085d4bea1b044b08ff57eb064354bbb96435f,"{""climbmix"": {""documents_emitted"": 989, ""epoch"": 0, ""prepared_position"": 3299, ""rows_read"": 990, ""shard"": 0, ""shards"": 6543}, ""rewrite6"": {""documents_emitted"": 389, ""epoch"": 0, ""prepared_position"": 522, ""rows_read"": 390, ""shard"": 0, ""shards"": 1}, ""ultra_qa"": {""documents_emitted"": 217, ""epoch"": 0, ""prepared_position"": 950, ""rows_read"": 218, ""shard"": 0, ""shards"": 616}, ""ultra_style"": {""documents_emitted"": 273, ""epoch"": 0, ""prepared_position"": 959, ""rows_read"": 274, ""shard"": 0, ""shards"": 552}}"
+unified-v1-blt-none-s42,ablation,causal,blt,none,42,0,0,initialized,fresh_matched_seed,0,0,0.8857087290000436,19.13628118099996,10425600,1024,,,0,0.0,0.0,,0,,100,269ba207e744573779747b593ffbcd8a87683fd55c83f6b168c658206d9aeb98,af33c522ea4349d5aae0491f7fe085d4bea1b044b08ff57eb064354bbb96435f,"{""climbmix"": {""documents_emitted"": 0, ""epoch"": 0, ""prepared_position"": null, ""rows_read"": 0, ""shard"": 0, ""shards"": 6543}, ""rewrite6"": {""documents_emitted"": 0, ""epoch"": 0, ""prepared_position"": null, ""rows_read"": 0, ""shard"": 0, ""shards"": 1}, ""ultra_qa"": {""documents_emitted"": 0, ""epoch"": 0, ""prepared_position"": null, ""rows_read"": 0, ""shard"": 0, ""shards"": 616}, ""ultra_style"": {""documents_emitted"": 0, ""epoch"": 0, ""prepared_position"": null, ""rows_read"": 0, ""shard"": 0, ""shards"": 552}}"

unified/ablations/unified-v1-blt-none-s42/checkpoint-000000000000/COMPLETE

@@ -0,0 +1 @@
+complete

unified/ablations/unified-v1-blt-none-s42/checkpoint-000000000000/metadata.json

@@ -0,0 +1,9 @@
+{
+  "format": "cortex-unified-v1",
+  "step": 0,
+  "lineage": "08a9fd42eadb43a3861b2c914ee6cbab",
+  "signature": "20afdd7a8d6a2f194a6a1772bd00d11fb1c26476d78881d3364c0bdd8421d331",
+  "sha256": "825adbb7a6662705a98219b3cc73321d13d64fe83c7f6c0d09283ec5bceec749",
+  "bytes": 42090107,
+  "saved_at": 1790265642.8075945
+}

unified/ablations/unified-v1-blt-none-s42/checkpoint-000000000000/training.pt

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:825adbb7a6662705a98219b3cc73321d13d64fe83c7f6c0d09283ec5bceec749
+size 42090107

unified/ablations/unified-v1-blt-none-s42/inference/data_progress.json

@@ -0,0 +1,47 @@
+{
+  "step": 0,
+  "prepared_dataset": {
+    "repo": "appvoid/rewrite7",
+    "revision": "5b26a55d9021b2ab1e3e3ea35e4f0aba8bd5a930",
+    "recipe": "794a6de09517a367ca643fda6b9a2a511c975f0a62cb0aa3fbdd0321a6a34e80"
+  },
+  "observations": {
+    "presentations": 0,
+    "unique_samples": 0,
+    "ranges": {}
+  },
+  "coverage": {
+    "climbmix": {
+      "epoch": 0,
+      "shard": 0,
+      "shards": 6543,
+      "rows_read": 0,
+      "prepared_position": null,
+      "documents_emitted": 0
+    },
+    "rewrite6": {
+      "epoch": 0,
+      "shard": 0,
+      "shards": 1,
+      "rows_read": 0,
+      "prepared_position": null,
+      "documents_emitted": 0
+    },
+    "ultra_style": {
+      "epoch": 0,
+      "shard": 0,
+      "shards": 552,
+      "rows_read": 0,
+      "prepared_position": null,
+      "documents_emitted": 0
+    },
+    "ultra_qa": {
+      "epoch": 0,
+      "shard": 0,
+      "shards": 616,
+      "rows_read": 0,
+      "prepared_position": null,
+      "documents_emitted": 0
+    }
+  }
+}

unified/ablations/unified-v1-blt-none-s42/inference/experiment_config.json

@@ -0,0 +1,97 @@
+{
+  "variant": "blt",
+  "config": {
+    "dim": 320,
+    "ff": 864,
+    "heads": 5,
+    "context": 1024,
+    "body_layers": 6,
+    "loops": 1,
+    "local_dim": 128,
+    "local_ff": 384,
+    "local_heads": 2,
+    "encoder_layers": 1,
+    "decoder_layers": 1,
+    "local_window": 128,
+    "query_block": 128,
+    "eva_window": 256,
+    "eva_chunk": 16,
+    "eva_heads": 8,
+    "hash_buckets": 1024,
+    "hash_sizes": [
+      3,
+      4,
+      5,
+      6,
+      7,
+      8
+    ],
+    "cross_slots": 2,
+    "entropy_dim": 64,
+    "entropy_ff": 192,
+    "entropy_heads": 1,
+    "entropy_layers": 2,
+    "patch_size": 4.0,
+    "byteflow_bins": 16,
+    "byteflow_eps2": 1.0,
+    "byteflow_score": "l2",
+    "router_window": 64,
+    "grad_checkpoint": false
+  },
+  "arch": "blt",
+  "total_params": 10425600,
+  "trainable_params": 10285632,
+  "target_params": 10826304,
+  "depth": 7,
+  "frozen_params": 139968,
+  "param_delta_pct": -3.701207725184883,
+  "causal_ranking_eligible": true,
+  "fidelity": "scaled architecture",
+  "data_progress": {
+    "step": 0,
+    "prepared_dataset": {
+      "repo": "appvoid/rewrite7",
+      "revision": "5b26a55d9021b2ab1e3e3ea35e4f0aba8bd5a930",
+      "recipe": "794a6de09517a367ca643fda6b9a2a511c975f0a62cb0aa3fbdd0321a6a34e80"
+    },
+    "observations": {
+      "presentations": 0,
+      "unique_samples": 0,
+      "ranges": {}
+    },
+    "coverage": {
+      "climbmix": {
+        "epoch": 0,
+        "shard": 0,
+        "shards": 6543,
+        "rows_read": 0,
+        "prepared_position": null,
+        "documents_emitted": 0
+      },
+      "rewrite6": {
+        "epoch": 0,
+        "shard": 0,
+        "shards": 1,
+        "rows_read": 0,
+        "prepared_position": null,
+        "documents_emitted": 0
+      },
+      "ultra_style": {
+        "epoch": 0,
+        "shard": 0,
+        "shards": 552,
+        "rows_read": 0,
+        "prepared_position": null,
+        "documents_emitted": 0
+      },
+      "ultra_qa": {
+        "epoch": 0,
+        "shard": 0,
+        "shards": 616,
+        "rows_read": 0,
+        "prepared_position": null,
+        "documents_emitted": 0
+      }
+    }
+  }
+}

unified/ablations/unified-v1-blt-none-s42/inference/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:df2b648db355f91327dbe8e2c259e618ff4861a1b186e89909fd2de52a2bab5e
+size 41714820

unified/ablations/unified-v1-blt-none-s42/source/__init__.py

@@ -0,0 +1 @@
+"""Self-contained SparkBET-9M training runtime and Hugging Face export helpers."""

unified/ablations/unified-v1-blt-none-s42/source/audio_codec.py

@@ -0,0 +1,101 @@
+"""Phase-preserving spectral Braille codec, version 1.
+
+Each quantized spectral byte is one Braille cell (its eight dots are eight bits).
+Two binary raster planes, both <=80x80: 8-bit log magnitude and 8-bit phase.
+This is a lossy, deterministic codec, not a neural audio tokenizer.
+"""
+import io,json,math
+import numpy as np
+from scipy import signal
+from PIL import Image
+from braille_encoder import binary_to_braille,local_contrast,floyd_steinberg
+RATE=16000
+NFFT=128
+HOP=64
+CHUNK=384  # 24 ms; 65 bins x 7 frames per plane fits in 80x48 pixels.
+DB=80.0
+
+def canonical_audio(value):
+    if isinstance(value,dict) and 'array' in value:
+        x=np.asarray(value['array'],dtype=np.float32);sr=int(value['sampling_rate'])
+    else:
+        import soundfile as sf
+        raw=value.get('bytes') if isinstance(value,dict) else None
+        path=value.get('path') if isinstance(value,dict) else value
+        x,sr=sf.read(io.BytesIO(raw) if raw is not None else path,dtype='float32',always_2d=False)
+    if x.ndim==2:x=x.mean(axis=1)
+    if x.ndim!=1 or not len(x) or sr<=0 or not np.isfinite(x).all():raise ValueError('Invalid waveform')
+    if len(x)>sr*120:raise ValueError('Audio exceeds 120-second decoding bound')
+    if sr!=RATE:
+        g=math.gcd(sr,RATE);x=signal.resample_poly(x,RATE//g,sr//g).astype(np.float32)
+    return np.clip(x,-1,1).astype(np.float32)
+
+def byte_plane(values,width=40):
+    raw=np.asarray(values,dtype=np.uint8).reshape(-1)
+    rows=(len(raw)+width-1)//width
+    if rows*4>80 or width*2>80:raise ValueError('Spectral plane exceeds 80x80')
+    padded=np.pad(raw,(0,rows*width-len(raw)))
+    return '\n'.join(''.join(chr(0x2800+int(v)) for v in row) for row in padded.reshape(rows,width))
+
+def read_plane(text,count):
+    lines=text.splitlines()
+    if not lines or len(lines)>20 or any(len(r)!=40 for r in lines):raise ValueError('Malformed Braille plane shape')
+    raw=[ord(c)-0x2800 for row in lines for c in row]
+    if any(v<0 or v>255 for v in raw) or len(raw)!=math.ceil(count/40)*40:raise ValueError('Malformed Braille payload')
+    if any(raw[count:]):raise ValueError('Nonzero spectral padding')
+    return np.array(raw[:count],dtype=np.uint8)
+
+def encode_tile(wave):
+    x=np.asarray(wave,dtype=np.float32)
+    if x.ndim!=1 or not 1<=len(x)<=CHUNK or not np.isfinite(x).all():raise ValueError('Invalid tile')
+    n=len(x);padded=np.pad(x,(0,CHUNK-n))
+    _,_,z=signal.stft(padded,fs=RATE,window='hann',nperseg=NFFT,noverlap=NFFT-HOP,
+                      nfft=NFFT,boundary='zeros',padded=True)
+    mag=np.abs(z);scale=max(float(mag.max()),1e-12)
+    # Code 0 is exact silence; 1..255 represent [-80,0] dB relative to scale.
+    db=20*np.log10(np.maximum(mag/scale,1e-12))
+    qm=np.where(db < -DB,0,1+np.rint((np.clip(db,-DB,0)+DB)*254/DB)).astype(np.uint8)
+    qp=(np.rint((np.angle(z)+np.pi)*(256/(2*np.pi))).astype(np.int32)%256).astype(np.uint8)
+    qp[qm==0]=128
+    meta=dict(v=1,sr=RATE,n=n,fft=NFFT,hop=HOP,frames=z.shape[1],scale=scale,db=DB)
+    return '<audio_tile>'+json.dumps(meta,separators=(',',':'))+'\nMAG\n'+byte_plane(qm)+'\nPHASE\n'+byte_plane(qp)+'\n</audio_tile>'
+
+def decode_tile(text):
+    if not text.startswith('<audio_tile>') or not text.endswith('\n</audio_tile>'):raise ValueError('Incomplete audio tile')
+    header,planes=text[len('<audio_tile>'):].split('\nMAG\n',1)
+    meta=json.loads(header)
+    if (meta['v'],meta['sr'],meta['fft'],meta['hop'],meta['frames'],meta['db'])!=(1,RATE,NFFT,HOP,7,DB):raise ValueError('Unsupported codec parameters')
+    if not 1<=meta['n']<=CHUNK or not 0<float(meta['scale'])<=2:raise ValueError('Invalid audio metadata')
+    magtxt,phasetxt=planes[:-len('\n</audio_tile>')].split('\nPHASE\n')
+    shape=(NFFT//2+1,meta['frames']);count=math.prod(shape)
+    qm=read_plane(magtxt,count).reshape(shape).astype(np.float64)
+    qp=read_plane(phasetxt,count).reshape(shape).astype(np.float64)
+    mag=np.where(qm==0,0,float(meta['scale'])*10**(((qm-1)*DB/254-DB)/20))
+    phase=qp*(2*np.pi/256)-np.pi
+    z=mag*np.exp(1j*phase)
+    # Real-valued signal: DC and Nyquist bins must be real.
+    z[0]=z[0].real;z[-1]=z[-1].real
+    _,wave=signal.istft(z,fs=RATE,window='hann',nperseg=NFFT,noverlap=NFFT-HOP,nfft=NFFT,boundary=True)
+    return np.clip(wave[:meta['n']],-1,1).astype(np.float32),RATE
+
+def encode_audio(wave):
+    return [encode_tile(wave[i:i+CHUNK]) for i in range(0,len(wave),CHUNK)]
+
+def decode_audio(tiles):
+    if not tiles:raise ValueError('No tiles')
+    return np.concatenate([decode_tile(t)[0] for t in tiles]),RATE
+
+def overview(wave,grid=80):
+    """Whole-clip log spectrum for semantic tasks; never used as reversible audio."""
+    x=np.asarray(wave,dtype=np.float32);x=np.pad(x,(0,max(0,256-len(x))))
+    _,_,z=signal.stft(x,fs=RATE,nperseg=256,noverlap=128,boundary='zeros')
+    power=20*np.log10(np.maximum(np.abs(z),1e-6))
+    hi=float(power.max());image=np.clip((power-(hi-70))/70,0,1)
+    # Frequency high at top; time left to right. No saliency crop or phase resizing.
+    im=Image.fromarray(np.uint8((1-image[::-1])*255)).resize((grid,grid),Image.Resampling.BOX).convert('RGBA')
+    enhanced,alpha=local_contrast(im,amount=0)
+    return binary_to_braille(floyd_steinberg(enhanced,alpha),alpha)
+
+def snr(reference,decoded):
+    x=np.asarray(reference,dtype=float);y=np.asarray(decoded,dtype=float)
+    return float(10*np.log10((np.sum(x*x)+1e-20)/(np.sum((x-y)**2)+1e-20)))

unified/ablations/unified-v1-blt-none-s42/source/bet_model.py

@@ -0,0 +1,488 @@
+"""SparkBET-9M: min-spark-style looped core with BET phase conditioning."""
+import math, hashlib
+from dataclasses import dataclass, asdict
+import torch
+from torch import nn
+import torch.nn.functional as F
+from torch.utils.checkpoint import checkpoint
+
+SEQ_LEN = 1024
+MAX_LOOPS = 8
+PAD_ID, BOS_ID, EOS_ID = 256, 257, 258
+VOCAB_SIZE = 259
+EXPECTED_PARAM_COUNT = 9_353_876
+EXPECTED_ARCH_SHAPE_SHA256 = "c85415fc50d03a23f89ccd870ebf53a3a00c5a61017c13a28f6444fdd71b72b9"
+_USE_GRAD_CHECKPOINTING = False
+_STATE_NOISE_SIGMA = 0.0
+
+
+def set_gradient_checkpointing(enabled):
+    global _USE_GRAD_CHECKPOINTING
+    _USE_GRAD_CHECKPOINTING = bool(enabled)
+
+
+def set_state_noise_sigma(value):
+    global _STATE_NOISE_SIGMA
+    value = float(value)
+    if value < 0:
+        raise ValueError("state noise sigma must be >= 0")
+    _STATE_NOISE_SIGMA = value
+
+
+@dataclass(frozen=True)
+class BETConfig:
+    vocab_size: int = VOCAB_SIZE
+    hidden_size: int = 324
+    intermediate_size: int = 864
+    prelude_layers: int = 1
+    body_blocks: int = 6
+    coda_layers: int = 1
+    num_heads: int = 6
+    num_kv_heads: int = 2
+    head_dim: int = 54
+    lora_rank: int = 16
+    hyper_lanes: int = 2
+    max_seq_len: int = SEQ_LEN
+    max_loops: int = MAX_LOOPS
+    rope_theta: float = 10_000.0
+    rms_eps: float = 1e-6
+    ddl_beta_init: float = 1.0
+    ddl_k_eps: float = 1e-2
+    ddl_v_sigmoid_scale: float = 4.0
+
+    @property
+    def q_dim(self):
+        return self.num_heads * self.head_dim
+
+    @property
+    def kv_dim(self):
+        return self.num_kv_heads * self.head_dim
+
+    @property
+    def qkv_dim(self):
+        return self.q_dim + 2 * self.kv_dim
+
+
+CFG = BETConfig()
+
+
+class RMSNorm(nn.Module):
+    def __init__(self, dim, eps=1e-6):
+        super().__init__()
+        self.weight = nn.Parameter(torch.ones(dim))
+        self.eps = eps
+
+    def forward(self, x):
+        dtype = x.dtype
+        y = x.float()
+        y = y * torch.rsqrt(y.pow(2).mean(-1, keepdim=True) + self.eps)
+        return (y * self.weight.float()).to(dtype)
+
+
+def rope_cos_sin(position_ids, dim, theta, dtype):
+    inv = 1.0 / (theta ** (torch.arange(0, dim, 2, device=position_ids.device, dtype=torch.float32) / dim))
+    f = position_ids.float().unsqueeze(-1) * inv
+    return f.cos().unsqueeze(1).to(dtype), f.sin().unsqueeze(1).to(dtype)
+
+
+def apply_rope(x, cos, sin):
+    # RoPE tables may be prepared before the first autocast linear, when the
+    # embedding stream is FP32. Cast them to the projected Q/K dtype here so
+    # attention remains FP16 on every CUDA profile instead of being promoted.
+    cos, sin = cos.to(dtype=x.dtype), sin.to(dtype=x.dtype)
+    xe, xo = x[..., 0::2], x[..., 1::2]
+    return torch.stack((xe * cos - xo * sin, xe * sin + xo * cos), dim=-1).flatten(-2)
+
+
+def attention_mask_and_positions(input_ids, attention_mask=None):
+    b, t = input_ids.shape
+    if attention_mask is None:
+        pos = torch.arange(t, device=input_ids.device).view(1, t).expand(b, t)
+        return None, pos
+    mask = attention_mask.to(device=input_ids.device, dtype=torch.bool)
+    if mask.shape != input_ids.shape:
+        raise ValueError(f"attention_mask shape {tuple(mask.shape)} != input_ids {tuple(input_ids.shape)}")
+    pos = (mask.long().cumsum(-1) - 1).clamp_min(0)
+    if bool(mask.all()):
+        return None, pos
+    causal = torch.ones((t, t), device=input_ids.device, dtype=torch.bool).tril()[None, None]
+    allowed = causal & mask[:, None, None, :]
+    return allowed, pos
+
+
+class Attention(nn.Module):
+    """6Q/2KV GQA with per-head QK RMSNorm before RoPE."""
+    def __init__(self, c):
+        super().__init__()
+        if c.hidden_size != c.q_dim:
+            raise ValueError("hidden_size must equal num_heads * head_dim")
+        if c.num_heads % c.num_kv_heads:
+            raise ValueError("num_heads must be divisible by num_kv_heads")
+        self.qkv = nn.Linear(c.hidden_size, c.qkv_dim, bias=False)
+        self.out = nn.Linear(c.q_dim, c.hidden_size, bias=False)
+        self.qn = RMSNorm(c.head_dim, c.rms_eps)
+        self.kn = RMSNorm(c.head_dim, c.rms_eps)
+        self.nh, self.nkv, self.hd = c.num_heads, c.num_kv_heads, c.head_dim
+        self.q_dim, self.kv_dim = c.q_dim, c.kv_dim
+
+    def forward(self, x, cos, sin, qkv_delta=None, attn_mask=None):
+        b, t, _ = x.shape
+        qkv = self.qkv(x)
+        if qkv_delta is not None:
+            if qkv_delta.shape != qkv.shape:
+                raise RuntimeError("phase LoRA QKV delta shape mismatch")
+            qkv = qkv + qkv_delta
+        q, k, v = qkv.split([self.q_dim, self.kv_dim, self.kv_dim], dim=-1)
+        q = q.view(b, t, self.nh, self.hd).transpose(1, 2)
+        k = k.view(b, t, self.nkv, self.hd).transpose(1, 2)
+        v = v.view(b, t, self.nkv, self.hd).transpose(1, 2)
+        q = apply_rope(self.qn(q), cos, sin)
+        k = apply_rope(self.kn(k), cos, sin)
+        if self.nkv != self.nh:
+            repeat = self.nh // self.nkv
+            k = k.repeat_interleave(repeat, dim=1)
+            v = v.repeat_interleave(repeat, dim=1)
+        if attn_mask is None:
+            z = F.scaled_dot_product_attention(q, k, v, is_causal=True, dropout_p=0.0)
+        else:
+            z = F.scaled_dot_product_attention(q, k, v, attn_mask=attn_mask, dropout_p=0.0)
+        return self.out(z.transpose(1, 2).contiguous().view(b, t, self.q_dim))
+
+
+class SwiGLU(nn.Module):
+    def __init__(self, c):
+        super().__init__()
+        self.gate_up = nn.Linear(c.hidden_size, 2 * c.intermediate_size, bias=False)
+        self.down = nn.Linear(c.intermediate_size, c.hidden_size, bias=False)
+
+    def forward(self, x):
+        gate, up = self.gate_up(x).chunk(2, dim=-1)
+        return self.down(F.silu(gate) * up)
+
+
+class DeepDeltaResidual(nn.Module):
+    """Scalar Deep-Delta residual update used only in the shared recurrent body."""
+    def __init__(self, c):
+        super().__init__()
+        self.k_eps = c.ddl_k_eps
+        self.v_sigmoid_scale = c.ddl_v_sigmoid_scale
+        self.beta_init = c.ddl_beta_init
+        self.beta = nn.Linear(c.hidden_size, 1, bias=True)
+        self.v_proj = nn.Linear(c.hidden_size, 1, bias=True)
+
+    @torch.no_grad()
+    def reset_beta_bias(self):
+        p = min(max(self.beta_init, 0.0), 2.0) / 2.0
+        p = min(max(p, 1e-6), 1.0 - 1e-6)
+        self.beta.bias.fill_(math.log(p) - math.log(1.0 - p))
+
+    def forward(self, x, *, k_in, context):
+        d = k_in.size(-1)
+        eps_rms = (self.k_eps * self.k_eps) / d
+        k_rms = F.rms_norm(k_in, [d], eps=eps_rms)
+        scale = 1.0 / math.sqrt(d)
+        beta = 2.0 * torch.sigmoid(self.beta(context).float())
+        proj = torch.sum(k_rms * x, dim=-1, keepdim=True, dtype=torch.float32) * scale
+        v = torch.sigmoid(self.v_proj(x).float()) * self.v_sigmoid_scale
+        delta = ((beta * (v - proj)) * scale).to(dtype=x.dtype)
+        return x + delta * k_rms
+
+
+class PlainBlock(nn.Module):
+    def __init__(self, c):
+        super().__init__()
+        self.attn_norm = RMSNorm(c.hidden_size, c.rms_eps)
+        self.attn = Attention(c)
+        self.ffn_norm = RMSNorm(c.hidden_size, c.rms_eps)
+        self.ffn = SwiGLU(c)
+
+    def forward(self, x, cos, sin, attn_mask):
+        x = x + self.attn(self.attn_norm(x), cos, sin, attn_mask=attn_mask)
+        return x + self.ffn(self.ffn_norm(x))
+
+
+class ContinuousLoopConditioner(nn.Module):
+    def __init__(self, d):
+        super().__init__()
+        self.net = nn.Sequential(nn.Linear(8, d), nn.SiLU(), nn.Linear(d, 2 * d))
+
+    @staticmethod
+    def features(t, dt, device, dtype):
+        return torch.tensor([
+            t, dt,
+            math.sin(math.pi * t), math.cos(math.pi * t),
+            math.sin(2 * math.pi * t), math.cos(2 * math.pi * t),
+            math.log(max(dt, 1e-6)), math.log(max(1.0 - t + dt, 1e-6)),
+        ], device=device, dtype=dtype)
+
+    def parameters_for(self, t, dt, device, dtype):
+        f = self.features(t, dt, device, dtype)
+        scale, shift = self.net(f).chunk(2, dim=-1)
+        return f, scale, shift
+
+    @staticmethod
+    def modulate(x, scale, shift):
+        return x * (1.0 + 0.1 * scale) + 0.1 * shift
+
+
+class PhaseLoRA(nn.Module):
+    def __init__(self, c):
+        super().__init__()
+        self.down = nn.Linear(c.hidden_size, c.lora_rank, bias=False)
+        self.up = nn.Linear(c.lora_rank, c.qkv_dim, bias=False)
+        self.gate = nn.Linear(8, c.lora_rank, bias=True)
+
+    def forward(self, x, phase_features):
+        gate = (2.0 * torch.sigmoid(self.gate(phase_features).float())).to(dtype=x.dtype)
+        return self.up(self.down(x) * gate)
+
+
+class LoopedBlock(nn.Module):
+    def __init__(self, c):
+        super().__init__()
+        self.attn_norm = RMSNorm(c.hidden_size, c.rms_eps)
+        self.attn = Attention(c)
+        self.phase_lora = PhaseLoRA(c)
+        self.ddl_attn = DeepDeltaResidual(c)
+        self.ffn_norm = RMSNorm(c.hidden_size, c.rms_eps)
+        self.ffn = SwiGLU(c)
+        self.ddl_ffn = DeepDeltaResidual(c)
+
+    def forward(self, x, cos, sin, phase_scale, phase_shift, phase_features, attn_mask):
+        attn_context = ContinuousLoopConditioner.modulate(x, phase_scale, phase_shift)
+        qkv_delta = self.phase_lora(attn_context, phase_features)
+        x_norm = self.attn_norm(attn_context)
+        x = self.ddl_attn(
+            x,
+            k_in=self.attn(x_norm, cos, sin, qkv_delta=qkv_delta, attn_mask=attn_mask),
+            context=x_norm,
+        )
+        ffn_context = ContinuousLoopConditioner.modulate(x, phase_scale, phase_shift)
+        x_norm = self.ffn_norm(ffn_context)
+        return self.ddl_ffn(x, k_in=self.ffn(x_norm), context=x_norm)
+
+
+class LoopHyperConnection(nn.Module):
+    """Two persistent loop lanes with per-budget read/mix/write scalars."""
+    def __init__(self, c):
+        super().__init__()
+        self.k = int(c.hyper_lanes)
+        self.max_loops = int(c.max_loops)
+        if self.k != 2:
+            raise ValueError("SparkBET is defined for two Hyper-Connection lanes")
+        shape = (self.max_loops, self.max_loops)
+        self.alpha = nn.Parameter(torch.zeros(*shape, self.k))
+        self.mix = nn.Parameter(torch.zeros(*shape, self.k, self.k))
+        self.beta = nn.Parameter(torch.zeros(*shape, self.k))
+        self.reset_parameters()
+
+    @torch.no_grad()
+    def reset_parameters(self):
+        self.alpha.zero_(); self.mix.zero_(); self.beta.zero_()
+        eye = torch.eye(self.k, device=self.mix.device, dtype=self.mix.dtype)
+        for budget in range(1, self.max_loops + 1):
+            b = budget - 1
+            for i in range(budget):
+                self.alpha[b, i, i % self.k] = 1.0
+                self.mix[b, i].copy_(eye)
+                self.beta[b, i].fill_(1.0)
+
+    def init_lanes(self, prelude_state):
+        return prelude_state.unsqueeze(0).expand(self.k, *prelude_state.shape)
+
+    def read(self, lanes, loops, iteration):
+        a = self.alpha[loops - 1, iteration].to(dtype=lanes.dtype)
+        return torch.einsum("k,kbtd->btd", a, lanes)
+
+    def write(self, lanes, branch_delta, loops, iteration):
+        m = self.mix[loops - 1, iteration].to(dtype=lanes.dtype)
+        b = self.beta[loops - 1, iteration].to(dtype=lanes.dtype)
+        mixed = torch.einsum("kj,jbtd->kbtd", m, lanes)
+        return mixed + b[:, None, None, None] * branch_delta.unsqueeze(0)
+
+    @staticmethod
+    def pool(lanes):
+        return lanes.mean(dim=0)
+
+
+class SparkBET(nn.Module):
+    """Prelude -> six shared recurrent blocks -> coda, with exact loop budgets 1..8."""
+    def __init__(self, c=CFG):
+        super().__init__()
+        self.c = c
+        self.embed = nn.Embedding(c.vocab_size, c.hidden_size)
+        self.prelude = nn.ModuleList([PlainBlock(c) for _ in range(c.prelude_layers)])
+        self.body = nn.ModuleList([LoopedBlock(c) for _ in range(c.body_blocks)])
+        self.time_cond = ContinuousLoopConditioner(c.hidden_size)
+        self.loop_hyper = LoopHyperConnection(c)
+        self.coda = nn.ModuleList([PlainBlock(c) for _ in range(c.coda_layers)])
+        self.final_norm = RMSNorm(c.hidden_size, c.rms_eps)
+        self.apply(self._generic_init)
+        self._mechanism_init()
+
+    @staticmethod
+    def _generic_init(m):
+        if isinstance(m, nn.Linear):
+            nn.init.normal_(m.weight, 0.0, 0.02)
+            if m.bias is not None: nn.init.zeros_(m.bias)
+        elif isinstance(m, nn.Embedding):
+            nn.init.normal_(m.weight, 0.0, 0.02)
+
+    @torch.no_grad()
+    def _mechanism_init(self):
+        nn.init.zeros_(self.time_cond.net[-1].weight)
+        nn.init.zeros_(self.time_cond.net[-1].bias)
+        self.loop_hyper.reset_parameters()
+        for block in self.body:
+            nn.init.zeros_(block.phase_lora.up.weight)
+            nn.init.zeros_(block.phase_lora.gate.weight)
+            nn.init.zeros_(block.phase_lora.gate.bias)
+        for block in [*self.prelude, *self.body, *self.coda]:
+            nn.init.zeros_(block.attn.out.weight)
+            nn.init.zeros_(block.ffn.down.weight)
+        for block in self.body:
+            block.ddl_attn.reset_beta_bias(); block.ddl_ffn.reset_beta_bias()
+
+    def _run_plain(self, block, x, cos, sin, attn_mask):
+        if _USE_GRAD_CHECKPOINTING and self.training:
+            return checkpoint(block, x, cos, sin, attn_mask, use_reentrant=False)
+        return block(x, cos, sin, attn_mask)
+
+    def _run_looped(self, block, x, cos, sin, scale, shift, features, attn_mask):
+        if _USE_GRAD_CHECKPOINTING and self.training:
+            return checkpoint(block, x, cos, sin, scale, shift, features, attn_mask, use_reentrant=False)
+        return block(x, cos, sin, scale, shift, features, attn_mask)
+
+    def _readout(self, x, cos, sin, attn_mask):
+        h = x
+        for block in self.coda:
+            h = self._run_plain(block, h, cos, sin, attn_mask)
+        h = self.final_norm(h)
+        return F.linear(h, self.embed.weight)
+
+    @staticmethod
+    def _schedule(step_sizes):
+        if isinstance(step_sizes, int):
+            n = int(step_sizes)
+            step_sizes = uniform_steps(n)
+        if not step_sizes:
+            raise ValueError("empty refinement schedule")
+        values = [float(v) for v in step_sizes]
+        if any(v <= 0 for v in values):
+            raise ValueError("refinement strides must be positive")
+        if abs(sum(values) - 1.0) > 1e-5:
+            raise ValueError("refinement strides must sum to 1")
+        return values
+
+    def _prepare(self, input_ids, step_sizes, attention_mask=None):
+        steps = self._schedule(step_sizes)
+        loops = len(steps)
+        if loops > self.c.max_loops:
+            raise ValueError(f"loops {loops} > configured max_loops {self.c.max_loops}")
+        if input_ids.shape[1] > self.c.max_seq_len:
+            raise ValueError("context exceeds max_seq_len")
+        x = self.embed(input_ids)
+        attn_mask, pos = attention_mask_and_positions(input_ids, attention_mask)
+        cos, sin = rope_cos_sin(pos, self.c.head_dim, self.c.rope_theta, x.dtype)
+        for block in self.prelude:
+            x = self._run_plain(block, x, cos, sin, attn_mask)
+        lanes = self.loop_hyper.init_lanes(x)
+        shared_noise = torch.randn_like(x) if self.training and _STATE_NOISE_SIGMA > 0 else None
+        return steps, lanes, shared_noise, cos, sin, attn_mask
+
+    def _advance(self, lanes, steps, iteration, elapsed, shared_noise, cos, sin, attn_mask):
+        dt = steps[iteration]
+        t_mid = elapsed + 0.5 * dt
+        features, scale, shift = self.time_cond.parameters_for(t_mid, dt, lanes.device, lanes.dtype)
+        branch_input = self.loop_hyper.read(lanes, len(steps), iteration)
+        if shared_noise is not None:
+            t_end = elapsed + dt
+            sigma = _STATE_NOISE_SIGMA * max(0.0, 1.0 - t_end)
+            if sigma:
+                branch_input = branch_input + sigma * shared_noise
+        h = branch_input
+        for block in self.body:
+            h = self._run_looped(block, h, cos, sin, scale, shift, features, attn_mask)
+        return self.loop_hyper.write(lanes, h - branch_input, len(steps), iteration)
+
+    def _run_recurrence(self, input_ids, step_sizes, attention_mask=None, collect_states=False):
+        steps, lanes, noise, cos, sin, attn_mask = self._prepare(input_ids, step_sizes, attention_mask)
+        states = [] if collect_states else None
+        elapsed = 0.0
+        for i, dt in enumerate(steps):
+            lanes = self._advance(lanes, steps, i, elapsed, noise, cos, sin, attn_mask)
+            elapsed += dt
+            if collect_states: states.append(self.loop_hyper.pool(lanes))
+        return self.loop_hyper.pool(lanes), states, cos, sin, attn_mask
+
+    def forward(self, input_ids, step_sizes=None, attention_mask=None):
+        if step_sizes is None: step_sizes = uniform_steps(self.c.max_loops)
+        x, _, cos, sin, attn_mask = self._run_recurrence(input_ids, step_sizes, attention_mask, False)
+        return self._readout(x, cos, sin, attn_mask)
+
+    def forward_loop_exits(self, input_ids, step_sizes=None, attention_mask=None):
+        if step_sizes is None: step_sizes = uniform_steps(self.c.max_loops)
+        _, states, cos, sin, attn_mask = self._run_recurrence(input_ids, step_sizes, attention_mask, True)
+        return [self._readout(h, cos, sin, attn_mask) for h in states]
+
+    def count_params(self):
+        return sum(p.numel() for p in self.parameters())
+
+
+# Preserve the old trainer/import name while changing the implementation.
+BETFog = SparkBET
+
+
+def uniform_steps(n):
+    n = int(n)
+    if not 1 <= n <= MAX_LOOPS:
+        raise ValueError(f"loop budget must be in [1,{MAX_LOOPS}]")
+    return [1.0 / n] * n
+
+
+def architecture_shape_signature(model):
+    lines = [f"{k}:{tuple(v.shape)}:{v.dtype}" for k, v in model.state_dict().items()]
+    return hashlib.sha256("\n".join(lines).encode()).hexdigest()
+
+
+@torch.no_grad()
+def verify_architecture(model, rank=0):
+    expected = dict(
+        vocab_size=259, hidden_size=324, intermediate_size=864,
+        prelude_layers=1, body_blocks=6, coda_layers=1,
+        num_heads=6, num_kv_heads=2, head_dim=54,
+        lora_rank=16, hyper_lanes=2, max_seq_len=1024, max_loops=8,
+        rope_theta=10_000.0, rms_eps=1e-6,
+        ddl_beta_init=1.0, ddl_k_eps=1e-2, ddl_v_sigmoid_scale=4.0,
+    )
+    actual = asdict(model.c)
+    for k, v in expected.items():
+        if actual[k] != v:
+            raise AssertionError(f"Architecture drift: {k}={actual[k]} expected {v}")
+    if model.count_params() != EXPECTED_PARAM_COUNT:
+        raise AssertionError(f"Parameter drift: {model.count_params():,} != {EXPECTED_PARAM_COUNT:,}")
+    sig = architecture_shape_signature(model)
+    if sig != EXPECTED_ARCH_SHAPE_SHA256:
+        raise AssertionError(f"Architecture SHA drift: {sig} != {EXPECTED_ARCH_SHAPE_SHA256}")
+    if not torch.allclose(model.time_cond.net[-1].weight, torch.zeros_like(model.time_cond.net[-1].weight)):
+        raise AssertionError("time conditioner must start as identity")
+    for i, block in enumerate(model.body):
+        if not torch.allclose(block.phase_lora.up.weight, torch.zeros_like(block.phase_lora.up.weight)):
+            raise AssertionError(f"phase LoRA {i} up projection must start zero")
+        for name, ddl in (("attn", block.ddl_attn), ("ffn", block.ddl_ffn)):
+            beta = (2.0 * torch.sigmoid(ddl.beta.bias.float())).item()
+            if abs(beta - 1.0) > 1e-6:
+                raise AssertionError(f"body {i} {name} DDL beta init={beta}")
+    hc = model.loop_hyper
+    eye = torch.eye(hc.k, device=hc.mix.device, dtype=hc.mix.dtype)
+    for budget in range(1, model.c.max_loops + 1):
+        for i in range(budget):
+            expected_alpha = torch.zeros_like(hc.alpha[budget - 1, i]); expected_alpha[i % hc.k] = 1
+            if not torch.allclose(hc.alpha[budget - 1, i], expected_alpha): raise AssertionError("Hyper alpha init drift")
+            if not torch.allclose(hc.mix[budget - 1, i], eye): raise AssertionError("Hyper mix init drift")
+            if not torch.allclose(hc.beta[budget - 1, i], torch.ones_like(hc.beta[budget - 1, i])): raise AssertionError("Hyper beta init drift")
+    if rank == 0:
+        print("[verify] SparkBET architecture PASSED")
+        print(f"[verify] params: {model.count_params():,}")
+        print(f"[verify] physical blocks: 1 prelude + 6 shared body + 1 coda; L8 applications=50")
+        print(f"[verify] architecture SHA256: {sig}")

unified/ablations/unified-v1-blt-none-s42/source/braille_encoder.py

@@ -0,0 +1,1532 @@
+#!/usr/bin/env python3
+"""
+Supported input sources
+-----------------------
+1. Single image file
+2. Folder of images
+3. CSV / TSV with an image path column
+4. JSON / JSONL with an image/path field
+5. Parquet with an image path column
+6. Hugging Face dataset repo via datasets.load_dataset(...)
+7. Hugging Face dataset already saved locally via datasets.load_from_disk(...)
+
+Supported output formats
+------------------------
+- txt      : single-image Braille text
+- jsonl    : dataset rows
+- csv      : dataset rows
+- parquet  : dataset rows
+- hf-disk  : datasets.Dataset.save_to_disk(...)
+- hf-push  : datasets.Dataset.push_to_hub(...)
+
+Dependencies
+------------
+Core:
+    pip install pillow numpy opencv-python
+
+Dataset features:
+    pip install datasets pandas pyarrow huggingface_hub
+
+Single image usage
+------------------
+    python smart_braille_encoder_v12.py input.png
+    python smart_braille_encoder_v12.py input.png --save-debug
+
+Dataset usage
+-------------
+    python smart_braille_encoder_v12.py ./images --dataset-out out.jsonl
+    python smart_braille_encoder_v12.py metadata.csv --image-column image_path --dataset-out out.parquet
+    python smart_braille_encoder_v12.py --hf-dataset user/dataset --split train --dataset-out out.jsonl
+    python smart_braille_encoder_v12.py --hf-dataset user/dataset --split train --hf-out braille_ds
+    python smart_braille_encoder_v12.py --hf-dataset user/dataset --split train --push-to-hub user/my-braille-dataset
+
+Notes
+-----
+- The exported dataset includes:
+    braille_text
+    braille_lines
+    braille_codepoints
+    braille_flat_codepoints
+    decision metadata
+- This makes it easy to train your own “ASCII tokenized” / Braille-tokenized models.
+"""
+
+from __future__ import annotations
+
+import argparse
+import io
+import json
+from pathlib import Path
+from typing import Any
+
+import cv2
+import numpy as np
+from PIL import Image, ImageDraw, ImageFilter
+
+
+# ============================================================
+# Utility
+# ============================================================
+
+IMAGE_EXTENSIONS = {
+    ".png", ".jpg", ".jpeg", ".webp", ".bmp", ".gif", ".tif", ".tiff"
+}
+
+
+def clamp(v: int, lo: int, hi: int) -> int:
+    return max(lo, min(v, hi))
+
+
+def robust_norm_masked(
+    x: np.ndarray,
+    mask: np.ndarray | None = None,
+    low: float = 1.0,
+    high: float = 99.0,
+) -> np.ndarray:
+    x = np.asarray(x, dtype=np.float32)
+
+    if mask is None:
+        vals = x.ravel()
+    else:
+        vals = x[mask.astype(bool)]
+
+    if vals.size == 0:
+        return np.zeros_like(x, dtype=np.float32)
+
+    lo = np.percentile(vals, low)
+    hi = np.percentile(vals, high)
+
+    out = np.clip((x - lo) / (hi - lo + 1e-6), 0.0, 1.0)
+
+    if mask is not None:
+        out *= mask.astype(np.float32)
+
+    return out.astype(np.float32)
+
+
+def is_image_path(path: str | Path) -> bool:
+    return Path(path).suffix.lower() in IMAGE_EXTENSIONS
+
+
+def jsonable(value: Any) -> Any:
+    if isinstance(value, (str, int, float, bool)) or value is None:
+        return value
+    if isinstance(value, Path):
+        return str(value)
+    if isinstance(value, (list, dict)):
+        return value
+    if isinstance(value, tuple):
+        return list(value)
+    if isinstance(value, np.generic):
+        return value.item()
+    return str(value)
+
+
+# ============================================================
+# Load / alpha handling
+# ============================================================
+
+def load_image(path: str | Path) -> Image.Image:
+    return Image.open(path).convert("RGBA")
+
+
+def alpha_mask(
+    image: Image.Image,
+    alpha_threshold: int = 8,
+) -> np.ndarray:
+    rgba = np.asarray(image.convert("RGBA"), dtype=np.uint8)
+    return rgba[..., 3] >= alpha_threshold
+
+
+def visible_bbox(
+    image: Image.Image,
+    alpha_threshold: int = 8,
+    padding_ratio: float = 0.02,
+) -> tuple[int, int, int, int]:
+    mask = alpha_mask(image, alpha_threshold)
+    h, w = mask.shape
+    ys, xs = np.where(mask)
+
+    if xs.size == 0:
+        return (0, 0, w, h)
+
+    x0 = int(xs.min())
+    y0 = int(ys.min())
+    x1 = int(xs.max()) + 1
+    y1 = int(ys.max()) + 1
+
+    bw = x1 - x0
+    bh = y1 - y0
+    pad = int(round(max(bw, bh) * padding_ratio))
+
+    return (
+        max(0, x0 - pad),
+        max(0, y0 - pad),
+        min(w, x1 + pad),
+        min(h, y1 + pad),
+    )
+
+
+# ============================================================
+# Attention analysis
+# ============================================================
+
+def compute_maps(
+    image: Image.Image,
+    alpha_threshold: int = 8,
+) -> dict:
+    rgba = np.asarray(image.convert("RGBA"), dtype=np.float32)
+
+    rgb = rgba[..., :3]
+    alpha_u8 = rgba[..., 3]
+    alpha = alpha_u8 / 255.0
+
+    visible = alpha_u8 >= alpha_threshold
+    visible_f = visible.astype(np.float32)
+
+    gray = (
+        0.299 * rgb[..., 0]
+        + 0.587 * rgb[..., 1]
+        + 0.114 * rgb[..., 2]
+    ).astype(np.float32)
+
+    h, w = gray.shape
+
+    if not np.any(visible):
+        z = np.zeros((h, w), dtype=np.float32)
+        return {
+            "attention": z,
+            "edge_energy": z,
+            "colorfulness": z,
+            "visible": visible,
+            "alpha": alpha,
+            "gray": gray,
+        }
+
+    gx = cv2.Sobel(gray, cv2.CV_32F, 1, 0, ksize=3)
+    gy = cv2.Sobel(gray, cv2.CV_32F, 0, 1, ksize=3)
+    edge_energy = np.hypot(gx, gy)
+
+    edges = robust_norm_masked(edge_energy, visible, 5, 99.5)
+
+    local_contrast = np.zeros_like(gray)
+    for sigma in (2, 8, 24):
+        blur = cv2.GaussianBlur(gray, (0, 0), sigma)
+        local_contrast += np.abs(gray - blur)
+    local_contrast /= 3.0
+    local_contrast = robust_norm_masked(local_contrast, visible, 5, 99)
+
+    median_luma = np.median(gray[visible])
+    luma_surprise = robust_norm_masked(np.abs(gray - median_luma), visible, 5, 99)
+
+    median_rgb = np.median(rgb[visible], axis=0)
+    color_distance = np.linalg.norm(rgb - median_rgb, axis=2)
+    color_surprise = robust_norm_masked(color_distance, visible, 5, 99)
+
+    bins = 32
+    bin_id = np.clip((gray / 256.0 * bins).astype(np.int32), 0, bins - 1)
+    hist = np.bincount(bin_id[visible].ravel(), minlength=bins).astype(np.float32)
+    prob = hist / (hist.sum() + 1e-9)
+    rarity = -np.log(prob[bin_id] + 1e-9)
+    rarity = robust_norm_masked(rarity, visible, 1, 99)
+
+    raw = (
+        0.18 * edges
+        + 0.20 * local_contrast
+        + 0.26 * luma_surprise
+        + 0.20 * color_surprise
+        + 0.16 * rarity
+    )
+    raw *= visible_f
+
+    blurred_raw = cv2.GaussianBlur(raw, (0, 0), 10)
+    blurred_vis = cv2.GaussianBlur(visible_f, (0, 0), 10)
+    coherent = blurred_raw / (blurred_vis + 1e-6)
+    coherent = robust_norm_masked(coherent, visible, 1, 99.5)
+
+    yy, xx = np.mgrid[0:h, 0:w]
+    cx = w / 2.0
+    cy = h / 2.0
+    sx = w * 0.48
+    sy = h * 0.40
+
+    center = np.exp(
+        -(((xx - cx) ** 2) / (2 * sx ** 2) + ((yy - cy) ** 2) / (2 * sy ** 2))
+    )
+
+    attention = coherent * (0.84 + 0.16 * center)
+    attention *= visible_f
+    attention = robust_norm_masked(attention, visible, 0, 99.5)
+
+    rgb_u8 = np.clip(rgb, 0, 255).astype(np.uint8)
+    hsv = cv2.cvtColor(rgb_u8, cv2.COLOR_RGB2HSV).astype(np.float32)
+
+    saturation = hsv[..., 1] / 255.0
+    value = hsv[..., 2] / 255.0
+    colorfulness = saturation * (0.35 + 0.65 * value)
+    colorfulness *= visible_f
+    colorfulness = robust_norm_masked(colorfulness, visible, 2, 99)
+
+    return {
+        "attention": attention.astype(np.float32),
+        "edge_energy": (edge_energy * visible_f).astype(np.float32),
+        "colorfulness": colorfulness.astype(np.float32),
+        "visible": visible,
+        "alpha": alpha.astype(np.float32),
+        "gray": gray,
+    }
+
+
+# ============================================================
+# Saliency geometry
+# ============================================================
+
+def salient_mask(
+    attention: np.ndarray,
+    visible: np.ndarray,
+    percentile: float = 87.0,
+) -> np.ndarray:
+    vals = attention[visible]
+    if vals.size == 0:
+        return np.zeros_like(attention, dtype=np.uint8)
+
+    threshold = np.percentile(vals, percentile)
+    mask = ((attention >= threshold) & visible).astype(np.uint8)
+    kernel = np.ones((5, 5), np.uint8)
+    mask = cv2.morphologyEx(mask, cv2.MORPH_CLOSE, kernel)
+    mask *= visible.astype(np.uint8)
+    return mask
+
+
+def bbox_from_mask(
+    mask: np.ndarray,
+    fallback: tuple[int, int, int, int],
+) -> tuple[int, int, int, int]:
+    ys, xs = np.where(mask > 0)
+    if xs.size == 0:
+        return fallback
+
+    return (
+        int(xs.min()),
+        int(ys.min()),
+        int(xs.max()) + 1,
+        int(ys.max()) + 1,
+    )
+
+
+def border_spread_in_visible_frame(
+    mask: np.ndarray,
+    visible_box: tuple[int, int, int, int],
+    border_ratio: float = 0.08,
+) -> dict:
+    x0, y0, x1, y1 = visible_box
+    local = mask[y0:y1, x0:x1]
+    h, w = local.shape
+
+    if h <= 0 or w <= 0:
+        return {k: 0.0 for k in ("left", "right", "top", "bottom", "tl", "tr", "bl", "br")}
+
+    total = float(local.sum()) + 1e-9
+
+    bx = max(1, int(w * border_ratio))
+    by = max(1, int(h * border_ratio))
+    ch = max(1, int(h * 0.18))
+    cw = max(1, int(w * 0.18))
+
+    return {
+        "left": float(local[:, :bx].sum() / total),
+        "right": float(local[:, w - bx:].sum() / total),
+        "top": float(local[:by, :].sum() / total),
+        "bottom": float(local[h - by:, :].sum() / total),
+        "tl": float(local[:ch, :cw].sum() / total),
+        "tr": float(local[:ch, w - cw:].sum() / total),
+        "bl": float(local[h - ch:, :cw].sum() / total),
+        "br": float(local[h - ch:, w - cw:].sum() / total),
+    }
+
+
+def features_span_visible_frame(
+    mask: np.ndarray,
+    salient_box: tuple[int, int, int, int],
+    visible_box: tuple[int, int, int, int],
+    side_presence_threshold: float = 0.012,
+    bbox_span_threshold: float = 0.88,
+) -> tuple[bool, dict]:
+    vx0, vy0, vx1, vy1 = visible_box
+    sx0, sy0, sx1, sy1 = salient_box
+
+    vw = max(vx1 - vx0, 1)
+    vh = max(vy1 - vy0, 1)
+
+    span_x = (sx1 - sx0) / vw
+    span_y = (sy1 - sy0) / vh
+
+    spread = border_spread_in_visible_frame(mask, visible_box)
+
+    all_sides = (
+        spread["left"] >= side_presence_threshold
+        and spread["right"] >= side_presence_threshold
+        and spread["top"] >= side_presence_threshold
+        and spread["bottom"] >= side_presence_threshold
+    )
+
+    corner_hits = sum(
+        spread[k] >= side_presence_threshold * 0.65
+        for k in ("tl", "tr", "bl", "br")
+    )
+
+    spans = span_x >= bbox_span_threshold and span_y >= bbox_span_threshold and all_sides
+    touches_four_corners = corner_hits == 4
+
+    return bool(spans or touches_four_corners), {
+        "span_x": float(span_x),
+        "span_y": float(span_y),
+        "corner_hits": int(corner_hits),
+        **spread,
+    }
+
+
+# ============================================================
+# Colorful center detector
+# ============================================================
+
+def detect_central_color_feature(
+    maps: dict,
+    visible_box: tuple[int, int, int, int],
+) -> dict | None:
+    colorfulness = maps["colorfulness"]
+    attention = maps["attention"]
+    visible = maps["visible"]
+
+    vals = colorfulness[visible]
+    if vals.size == 0:
+        return None
+
+    threshold = max(float(np.percentile(vals, 72)), 0.22)
+
+    att_vals = attention[visible]
+    att_threshold = float(np.percentile(att_vals, 55)) if att_vals.size else 0.0
+
+    seed = ((colorfulness >= threshold) & visible).astype(np.uint8)
+    support = ((colorfulness >= threshold * 0.62) & (attention >= att_threshold) & visible).astype(np.uint8)
+
+    seed = cv2.morphologyEx(seed, cv2.MORPH_CLOSE, np.ones((5, 5), np.uint8))
+    seed = cv2.dilate(seed, np.ones((3, 3), np.uint8), iterations=1)
+
+    candidate = ((seed > 0) & (support > 0)).astype(np.uint8)
+
+    n, labels, stats, centroids = cv2.connectedComponentsWithStats(candidate, connectivity=8)
+    if n <= 1:
+        return None
+
+    vx0, vy0, vx1, vy1 = visible_box
+    vis_w = max(vx1 - vx0, 1)
+    vis_h = max(vy1 - vy0, 1)
+    frame_cx = (vx0 + vx1) / 2.0
+    frame_cy = (vy0 + vy1) / 2.0
+
+    visible_area = float(visible.sum()) + 1e-9
+    total_color_mass = float((colorfulness * visible.astype(np.float32)).sum()) + 1e-9
+
+    best = None
+
+    for label in range(1, n):
+        area = int(stats[label, cv2.CC_STAT_AREA])
+        if area < max(8, int(visible_area * 0.002)):
+            continue
+
+        component = labels == label
+        cx, cy = centroids[label]
+
+        dx = (cx - frame_cx) / (vis_w / 2.0 + 1e-9)
+        dy = (cy - frame_cy) / (vis_h / 2.0 + 1e-9)
+        center_distance = float(np.sqrt(dx * dx + dy * dy))
+
+        if center_distance > 0.72:
+            continue
+
+        mean_color = float(colorfulness[component].mean())
+        mean_attention = float(attention[component].mean())
+        color_mass = float(colorfulness[component].sum())
+        dominance = color_mass / total_color_mass
+        area_ratio = area / visible_area
+
+        center_score = np.exp(-(center_distance ** 2) / (2 * 0.42 ** 2))
+        score = (
+            0.34 * mean_color
+            + 0.24 * mean_attention
+            + 0.22 * center_score
+            + 0.14 * min(dominance * 2.0, 1.0)
+            + 0.06 * min(area_ratio * 8.0, 1.0)
+        )
+
+        if best is None or score > best["score"]:
+            best = {
+                "score": float(score),
+                "mask": component.astype(np.uint8),
+                "centroid": (float(cx), float(cy)),
+                "center_distance": center_distance,
+                "mean_color": mean_color,
+                "mean_attention": mean_attention,
+                "dominance": float(dominance),
+                "area_ratio": float(area_ratio),
+            }
+
+    if best is None:
+        return None
+    if best["score"] < 0.44:
+        return None
+    if best["mean_color"] < 0.28:
+        return None
+    if best["mean_attention"] < 0.20:
+        return None
+
+    return best
+
+
+def color_outline_crop(
+    feature: dict,
+    image_size: tuple[int, int],
+    visible_box: tuple[int, int, int, int],
+) -> tuple[tuple[int, int, int, int], dict]:
+    mask = feature["mask"].astype(np.uint8)
+    ys, xs = np.where(mask > 0)
+
+    if xs.size == 0:
+        return visible_box, {
+            "outline_layers": 0,
+            "layer_px": 0,
+            "object_scale": 1.0,
+            "expanded_mask": mask,
+        }
+
+    x0 = int(xs.min())
+    y0 = int(ys.min())
+    x1 = int(xs.max()) + 1
+    y1 = int(ys.max()) + 1
+
+    obj_side = max(x1 - x0, y1 - y0)
+
+    vx0, vy0, vx1, vy1 = visible_box
+    visible_side = max(vx1 - vx0, vy1 - vy0)
+
+    object_scale = obj_side / max(visible_side, 1)
+    layers = 3 if object_scale < 0.30 else 2
+
+    layer_px = max(2, int(round(obj_side * 0.045)))
+    kernel_size = layer_px * 2 + 1
+
+    kernel = cv2.getStructuringElement(cv2.MORPH_ELLIPSE, (kernel_size, kernel_size))
+
+    expanded = mask.copy()
+    for _ in range(layers):
+        expanded = cv2.dilate(expanded, kernel, iterations=1)
+
+    eys, exs = np.where(expanded > 0)
+
+    ex0 = int(exs.min())
+    ey0 = int(eys.min())
+    ex1 = int(exs.max()) + 1
+    ey1 = int(eys.max()) + 1
+
+    side = max(ex1 - ex0, ey1 - ey0)
+
+    img_w, img_h = image_size
+    visible_w = vx1 - vx0
+    visible_h = vy1 - vy0
+    max_useful_side = max(visible_w, visible_h)
+
+    side = max(2, min(side, max_useful_side, min(img_w, img_h)))
+
+    cx = (ex0 + ex1) / 2.0
+    cy = (ey0 + ey1) / 2.0
+
+    sx0 = clamp(int(round(cx - side / 2)), 0, img_w - side)
+    sy0 = clamp(int(round(cy - side / 2)), 0, img_h - side)
+
+    return (
+        sx0,
+        sy0,
+        sx0 + side,
+        sy0 + side,
+    ), {
+        "outline_layers": int(layers),
+        "layer_px": int(layer_px),
+        "object_scale": float(object_scale),
+        "expanded_mask": expanded,
+    }
+
+
+# ============================================================
+# Generic v6 crop geometry / evaluation
+# ============================================================
+
+def expanded_square_bbox(
+    salient_box: tuple[int, int, int, int],
+    visible_box: tuple[int, int, int, int],
+    image_size: tuple[int, int],
+    margin_ratio: float = 0.12,
+) -> tuple[int, int, int, int]:
+    img_w, img_h = image_size
+    sx0, sy0, sx1, sy1 = salient_box
+    vx0, vy0, vx1, vy1 = visible_box
+
+    bw = sx1 - sx0
+    bh = sy1 - sy0
+    side = max(bw, bh)
+
+    side = int(round(side * (1.0 + 2.0 * margin_ratio)))
+
+    visible_w = vx1 - vx0
+    visible_h = vy1 - vy0
+    max_useful_side = max(visible_w, visible_h)
+
+    side = min(side, max_useful_side, min(img_w, img_h))
+    side = max(side, 2)
+
+    cx = (sx0 + sx1) / 2.0
+    cy = (sy0 + sy1) / 2.0
+
+    x0 = int(round(cx - side / 2))
+    y0 = int(round(cy - side / 2))
+
+    x0 = clamp(x0, 0, img_w - side)
+    y0 = clamp(y0, 0, img_h - side)
+
+    return (x0, y0, x0 + side, y0 + side)
+
+
+def evaluate_crop(
+    crop_box: tuple[int, int, int, int],
+    visible_box: tuple[int, int, int, int],
+    attention: np.ndarray,
+    edge_energy: np.ndarray,
+    visible: np.ndarray,
+) -> dict:
+    x0, y0, x1, y1 = crop_box
+    vx0, vy0, vx1, vy1 = visible_box
+
+    visible_f = visible.astype(np.float32)
+
+    saliency_weighted = attention * visible_f
+    total_saliency = float(saliency_weighted.sum()) + 1e-9
+    crop_saliency = float(saliency_weighted[y0:y1, x0:x1].sum())
+
+    vals = attention[visible]
+    strong_threshold = np.percentile(vals, 90) if vals.size else 1.0
+
+    strong = attention * (attention >= strong_threshold) * visible_f
+    total_strong = float(strong.sum()) + 1e-9
+    crop_strong = float(strong[y0:y1, x0:x1].sum())
+
+    weighted_edges = edge_energy * (0.35 + 0.65 * attention) * visible_f
+    total_detail = float(weighted_edges.sum()) + 1e-9
+    crop_detail = float(weighted_edges[y0:y1, x0:x1].sum())
+
+    total_visible = float(visible_f.sum()) + 1e-9
+    crop_visible = float(visible_f[y0:y1, x0:x1].sum())
+    visible_coverage = crop_visible / total_visible
+
+    visible_w = vx1 - vx0
+    visible_h = vy1 - vy0
+    base_side = max(visible_w, visible_h)
+    crop_side = max(x1 - x0, y1 - y0)
+    resolution_gain = base_side / max(crop_side, 1)
+
+    return {
+        "saliency_coverage": float(crop_saliency / total_saliency),
+        "strong_saliency_coverage": float(crop_strong / total_strong),
+        "detail_coverage": float(crop_detail / total_detail),
+        "visible_pixel_coverage": float(visible_coverage),
+        "resolution_gain": float(resolution_gain),
+        "crop_ratio_of_visible_frame": float(crop_side / max(base_side, 1)),
+    }
+
+
+def should_crop_v6(metrics: dict) -> tuple[bool, float, float]:
+    saliency_loss = 1.0 - metrics["saliency_coverage"]
+    strong_loss = 1.0 - metrics["strong_saliency_coverage"]
+    detail_loss = 1.0 - metrics["detail_coverage"]
+
+    gain = np.log2(max(metrics["resolution_gain"], 1.0))
+    benefit = 1.30 * gain
+    cost = 1.30 * strong_loss + 0.85 * saliency_loss + 0.95 * detail_loss
+
+    if metrics["strong_saliency_coverage"] < 0.93:
+        return False, float(benefit), float(cost)
+    if metrics["detail_coverage"] < 0.72:
+        return False, float(benefit), float(cost)
+    if metrics["resolution_gain"] < 1.12:
+        return False, float(benefit), float(cost)
+
+    return bool(benefit > cost), float(benefit), float(cost)
+
+
+def should_crop_color_feature(
+    metrics: dict,
+    feature: dict,
+) -> tuple[bool, float, float]:
+    saliency_loss = 1.0 - metrics["saliency_coverage"]
+    strong_loss = 1.0 - metrics["strong_saliency_coverage"]
+    detail_loss = 1.0 - metrics["detail_coverage"]
+
+    gain = np.log2(max(metrics["resolution_gain"], 1.0))
+
+    center_bonus = np.exp(-(feature["center_distance"] ** 2) / 0.18)
+    feature_bonus = (
+        0.34 * feature["mean_color"]
+        + 0.28 * feature["mean_attention"]
+        + 0.22 * min(feature["dominance"] * 2.0, 1.0)
+        + 0.16 * center_bonus
+    )
+
+    benefit = 1.30 * gain + 0.45 * feature_bonus
+    cost = 1.22 * strong_loss + 0.72 * saliency_loss + 0.82 * detail_loss
+
+    if metrics["strong_saliency_coverage"] < 0.91:
+        return False, float(benefit), float(cost)
+    if metrics["detail_coverage"] < 0.66:
+        return False, float(benefit), float(cost)
+    if metrics["resolution_gain"] < 1.07:
+        return False, float(benefit), float(cost)
+
+    return bool(benefit > cost), float(benefit), float(cost)
+
+
+# ============================================================
+# View selection
+# ============================================================
+
+def select_view(
+    image: Image.Image,
+    alpha_threshold: int = 8,
+    saliency_percentile: float = 87.0,
+    margin_ratio: float = 0.12,
+) -> dict:
+    w, h = image.size
+
+    vis_box = visible_bbox(image, alpha_threshold=alpha_threshold)
+    maps = compute_maps(image, alpha_threshold=alpha_threshold)
+
+    attention = maps["attention"]
+    edge_energy = maps["edge_energy"]
+    visible = maps["visible"]
+
+    if not np.any(visible):
+        return {
+            "mode": "empty",
+            "box": vis_box,
+            "visible_box": vis_box,
+            "attention": attention,
+            "mask": np.zeros((h, w), dtype=np.uint8),
+            "bbox": vis_box,
+            "maps": maps,
+            "spread": {},
+            "metrics": None,
+            "color_feature": None,
+            "outline_info": None,
+            "reason": "image has no visible pixels",
+        }
+
+    mask = salient_mask(attention, visible, percentile=saliency_percentile)
+    sal_box = bbox_from_mask(mask, fallback=vis_box)
+
+    spans, spread = features_span_visible_frame(mask, sal_box, vis_box)
+
+    if spans:
+        return {
+            "mode": "visible_full",
+            "box": vis_box,
+            "visible_box": vis_box,
+            "attention": attention,
+            "mask": mask,
+            "bbox": sal_box,
+            "maps": maps,
+            "spread": spread,
+            "metrics": None,
+            "color_feature": None,
+            "outline_info": None,
+            "reason": "v6 full-frame rule: salient features span the visible frame / all four sides or corners",
+        }
+
+    color_feature = detect_central_color_feature(maps, vis_box)
+    if color_feature is not None:
+        color_box, outline_info = color_outline_crop(color_feature, image.size, vis_box)
+
+        color_metrics = evaluate_crop(color_box, vis_box, attention, edge_energy, visible)
+        color_ok, benefit, cost = should_crop_color_feature(color_metrics, color_feature)
+
+        color_metrics["benefit"] = benefit
+        color_metrics["cost"] = cost
+
+        if color_ok:
+            return {
+                "mode": "color_outline_crop",
+                "box": color_box,
+                "visible_box": vis_box,
+                "attention": attention,
+                "mask": mask,
+                "bbox": sal_box,
+                "maps": maps,
+                "spread": spread,
+                "metrics": color_metrics,
+                "color_feature": color_feature,
+                "outline_info": outline_info,
+                "reason": f'central colorful feature detected after v6 full-frame check; kept {outline_info["outline_layers"]} outer outline layers',
+            }
+
+    candidate = expanded_square_bbox(sal_box, vis_box, image.size, margin_ratio=margin_ratio)
+    metrics = evaluate_crop(candidate, vis_box, attention, edge_energy, visible)
+    crop_ok, benefit, cost = should_crop_v6(metrics)
+
+    metrics["benefit"] = benefit
+    metrics["cost"] = cost
+
+    if crop_ok:
+        return {
+            "mode": "crop",
+            "box": candidate,
+            "visible_box": vis_box,
+            "attention": attention,
+            "mask": mask,
+            "bbox": sal_box,
+            "maps": maps,
+            "spread": spread,
+            "metrics": metrics,
+            "color_feature": color_feature,
+            "outline_info": None,
+            "reason": "v6 saliency/detail crop accepted because resolution gain exceeds weighted information loss",
+        }
+
+    return {
+        "mode": "visible_full",
+        "box": vis_box,
+        "visible_box": vis_box,
+        "attention": attention,
+        "mask": mask,
+        "bbox": sal_box,
+        "maps": maps,
+        "spread": spread,
+        "metrics": metrics,
+        "color_feature": color_feature,
+        "outline_info": None,
+        "reason": "transparent margins removed for free; v6 crop test rejected further cropping",
+    }
+
+
+# ============================================================
+# Fit selected box to 64x64
+# ============================================================
+
+def fit_box_to_square(
+    image: Image.Image,
+    box: tuple[int, int, int, int],
+    size: int = 64,
+) -> Image.Image:
+    region = image.crop(box).convert("RGBA")
+    w, h = region.size
+
+    scale = min(size / max(w, 1), size / max(h, 1))
+
+    nw = max(1, int(round(w * scale)))
+    nh = max(1, int(round(h * scale)))
+
+    reduced = region.resize((nw, nh), Image.Resampling.LANCZOS)
+
+    canvas = Image.new("RGBA", (size, size), (0, 0, 0, 0))
+
+    ox = (size - nw) // 2
+    oy = (size - nh) // 2
+
+    canvas.alpha_composite(reduced, (ox, oy))
+    return canvas
+
+
+# ============================================================
+# Braille conversion
+# ============================================================
+
+BRAILLE_BITS = {
+    (0, 0): 0,
+    (0, 1): 1,
+    (0, 2): 2,
+    (1, 0): 3,
+    (1, 1): 4,
+    (1, 2): 5,
+    (0, 3): 6,
+    (1, 3): 7,
+}
+
+
+def local_contrast(
+    rgba_image: Image.Image,
+    amount: float = 1.5,
+    blur_radius: float = 2.4,
+) -> tuple[np.ndarray, np.ndarray]:
+    rgba = np.asarray(rgba_image.convert("RGBA"), dtype=np.float32)
+    rgb = rgba[..., :3]
+    alpha = rgba[..., 3]
+
+    gray = (
+        0.299 * rgb[..., 0]
+        + 0.587 * rgb[..., 1]
+        + 0.114 * rgb[..., 2]
+    ).astype(np.float32)
+
+    gray_img = Image.fromarray(np.clip(gray, 0, 255).astype(np.uint8), mode="L")
+    local = np.asarray(gray_img.filter(ImageFilter.GaussianBlur(blur_radius)), dtype=np.float32)
+
+    enhanced = np.clip(gray + amount * (gray - local), 0, 255)
+    return enhanced, alpha
+
+
+def floyd_steinberg(
+    arr: np.ndarray,
+    alpha: np.ndarray,
+    alpha_threshold: int = 8,
+) -> np.ndarray:
+    work = arr.astype(np.float32).copy()
+
+    h, w = work.shape
+    out = np.full((h, w), 255, dtype=np.uint8)
+    visible = alpha >= alpha_threshold
+
+    for y in range(h):
+        for x in range(w):
+            if not visible[y, x]:
+                continue
+
+            old = work[y, x]
+            new = 255.0 if old >= 128 else 0.0
+            out[y, x] = int(new)
+
+            err = old - new
+
+            if x + 1 < w and visible[y, x + 1]:
+                work[y, x + 1] += err * 7 / 16
+            if y + 1 < h and x > 0 and visible[y + 1, x - 1]:
+                work[y + 1, x - 1] += err * 3 / 16
+            if y + 1 < h and visible[y + 1, x]:
+                work[y + 1, x] += err * 5 / 16
+            if y + 1 < h and x + 1 < w and visible[y + 1, x + 1]:
+                work[y + 1, x + 1] += err * 1 / 16
+
+    return out
+
+
+def binary_to_braille(
+    binary: np.ndarray,
+    alpha: np.ndarray,
+    alpha_threshold: int = 8,
+) -> str:
+    """
+    64x64 binary source grid -> 32x16 Unicode Braille.
+    """
+    h, w = binary.shape
+
+    if h % 4 != 0 or w % 2 != 0:
+        raise ValueError("Braille grid must be divisible by 4 vertically and 2 horizontally.")
+
+    lines = []
+
+    for cy in range(h // 4):
+        line = []
+        for cx in range(w // 2):
+            code = 0
+            any_visible = False
+
+            for sy in range(4):
+                for sx in range(2):
+                    yy = cy * 4 + sy
+                    xx = cx * 2 + sx
+
+                    if alpha[yy, xx] < alpha_threshold:
+                        continue
+
+                    any_visible = True
+
+                    if binary[yy, xx] < 128:
+                        code |= 1 << BRAILLE_BITS[(sx, sy)]
+
+            if not any_visible:
+                line.append(" ")
+            else:
+                line.append(chr(0x2800 + code))
+
+        lines.append("".join(line))
+
+    return "\n".join(lines)
+
+
+def braille_metadata(braille_text: str) -> dict:
+    lines = braille_text.splitlines()
+    codepoints_2d = [[ord(ch) for ch in line] for line in lines]
+    flat_codepoints = [cp for row in codepoints_2d for cp in row]
+
+    return {
+        "braille_lines": lines,
+        "braille_width_chars": max((len(line) for line in lines), default=0),
+        "braille_height_chars": len(lines),
+        "braille_codepoints": codepoints_2d,
+        "braille_flat_codepoints": flat_codepoints,
+    }
+
+
+# ============================================================
+# Full pipeline
+# ============================================================
+
+def encode_image(
+    image_path: str | Path | None = None,
+    image: Image.Image | None = None,
+    grid_size: int = 64,
+    alpha_threshold: int = 8,
+) -> dict:
+    if image is None:
+        if image_path is None:
+            raise ValueError("Either image_path or image must be provided.")
+        image = load_image(image_path)
+    else:
+        image = image.convert("RGBA")
+
+    decision = select_view(image, alpha_threshold=alpha_threshold)
+
+    small = fit_box_to_square(image, decision["box"], size=grid_size)
+
+    enhanced, alpha = local_contrast(small)
+    binary = floyd_steinberg(enhanced, alpha, alpha_threshold=alpha_threshold)
+    braille = binary_to_braille(binary, alpha, alpha_threshold=alpha_threshold)
+
+    return {
+        "image": image,
+        "decision": decision,
+        "small": small,
+        "enhanced": enhanced,
+        "alpha_small": alpha,
+        "binary": binary,
+        "braille": braille,
+        "grid_size": grid_size,
+    }
+
+
+# ============================================================
+# Debug output
+# ============================================================
+
+def save_debug(result: dict, out_dir: Path) -> None:
+    out_dir.mkdir(parents=True, exist_ok=True)
+
+    image = result["image"]
+    d = result["decision"]
+    maps = d["maps"]
+
+    overlay = image.convert("RGBA").copy()
+    draw = ImageDraw.Draw(overlay)
+
+    draw.rectangle(d["visible_box"], outline=(0, 128, 255, 255), width=5)
+    draw.rectangle(d["bbox"], outline=(0, 255, 0, 255), width=3)
+    draw.rectangle(d["box"], outline=(255, 0, 0, 255), width=4)
+
+    overlay.save(out_dir / "decision_overlay.png")
+
+    def save_map(filename: str, arr: np.ndarray):
+        Image.fromarray(np.clip(arr * 255, 0, 255).astype(np.uint8), mode="L").save(out_dir / filename)
+
+    save_map("attention_map.png", maps["attention"])
+
+    weighted_detail = (
+        maps["edge_energy"]
+        * (0.35 + 0.65 * maps["attention"])
+        * maps["visible"].astype(np.float32)
+    )
+
+    if weighted_detail.max() > 0:
+        weighted_detail = weighted_detail / (weighted_detail.max() + 1e-9)
+
+    save_map("weighted_detail_map.png", weighted_detail)
+    save_map("colorfulness_map.png", maps["colorfulness"])
+
+    Image.fromarray(d["mask"] * 255, mode="L").save(out_dir / "salient_mask.png")
+
+    if d["color_feature"] is not None:
+        Image.fromarray(d["color_feature"]["mask"] * 255, mode="L").save(out_dir / "central_color_feature.png")
+
+    if d["outline_info"] is not None and "expanded_mask" in d["outline_info"]:
+        Image.fromarray(d["outline_info"]["expanded_mask"] * 255, mode="L").save(out_dir / "color_outline_expanded.png")
+
+    result["small"].save(out_dir / "selected_64x64.png")
+    Image.fromarray(result["binary"], mode="L").save(out_dir / "binary_64x64.png")
+    Image.fromarray(np.clip(result["alpha_small"], 0, 255).astype(np.uint8), mode="L").save(out_dir / "alpha_64x64.png")
+    (out_dir / "braille.txt").write_text(result["braille"], encoding="utf-8")
+
+
+# ============================================================
+# Dataset ingestion helpers
+# ============================================================
+
+def image_from_value(value: Any, base_dir: Path | None = None) -> tuple[Image.Image, str | None]:
+    """
+    Supports:
+    - local file path string
+    - PIL Image
+    - numpy array
+    - dicts like {"path": ...} or {"bytes": ...}
+    """
+    if isinstance(value, Image.Image):
+        return value.convert("RGBA"), None
+
+    if isinstance(value, np.ndarray):
+        return Image.fromarray(value).convert("RGBA"), None
+
+    if isinstance(value, dict):
+        if "bytes" in value and value["bytes"] is not None:
+            raw = value["bytes"]
+            return Image.open(io.BytesIO(raw)).convert("RGBA"), value.get("path")
+        for key in ("path", "file_name", "filename", "filepath", "image_path"):
+            if key in value and value[key]:
+                p = Path(value[key])
+                if base_dir is not None and not p.is_absolute():
+                    p = base_dir / p
+                return load_image(p), str(p)
+
+    if isinstance(value, (str, Path)):
+        p = Path(value)
+        if base_dir is not None and not p.is_absolute():
+            p = base_dir / p
+        return load_image(p), str(p)
+
+    raise ValueError(f"Unsupported image value type: {type(value)}")
+
+
+def iter_folder_images(folder: Path):
+    for path in sorted(folder.rglob("*")):
+        if path.is_file() and is_image_path(path):
+            yield {
+                "id": path.stem,
+                "image_value": str(path),
+                "source": str(path),
+                "extra": {},
+            }
+
+
+def iter_tabular_records(path: Path, image_column: str):
+    import pandas as pd
+
+    suffix = path.suffix.lower()
+
+    if suffix == ".csv":
+        df = pd.read_csv(path)
+    elif suffix == ".tsv":
+        df = pd.read_csv(path, sep="\t")
+    elif suffix == ".parquet":
+        df = pd.read_parquet(path)
+    elif suffix == ".json":
+        raw = json.loads(path.read_text(encoding="utf-8"))
+        if isinstance(raw, dict):
+            raw = raw.get("data", raw.get("items", raw))
+            if isinstance(raw, dict):
+                raw = [raw]
+        df = pd.DataFrame(raw)
+    elif suffix == ".jsonl":
+        rows = [json.loads(line) for line in path.read_text(encoding="utf-8").splitlines() if line.strip()]
+        df = pd.DataFrame(rows)
+    else:
+        raise ValueError(f"Unsupported tabular file: {path}")
+
+    if image_column not in df.columns:
+        fallback_cols = [c for c in df.columns if c.lower() in {"image", "image_path", "path", "file_name", "filename"}]
+        if not fallback_cols:
+            raise ValueError(f'Image column "{image_column}" not found.')
+        image_column = fallback_cols[0]
+
+    base_dir = path.parent
+
+    for idx, row in df.iterrows():
+        rec = row.to_dict()
+        yield {
+            "id": str(rec.get("id", idx)),
+            "image_value": rec[image_column],
+            "source": f"{path}:{idx}",
+            "base_dir": base_dir,
+            "extra": rec,
+        }
+
+
+def iter_hf_dataset_repo(dataset_name: str, split: str, config_name: str | None):
+    try:
+        from datasets import load_dataset
+    except Exception as e:
+        raise RuntimeError("datasets is required for --hf-dataset. Install with: pip install datasets") from e
+
+    kwargs = {}
+    if config_name:
+        kwargs["name"] = config_name
+
+    ds = load_dataset(dataset_name, **kwargs, split=split)
+
+    image_column = None
+    for c in ds.column_names:
+        if c.lower() in {"image", "img"}:
+            image_column = c
+            break
+    if image_column is None:
+        for c in ds.column_names:
+            if "image" in c.lower():
+                image_column = c
+                break
+    if image_column is None:
+        raise ValueError("Could not find an image column in the HF dataset.")
+
+    for idx, row in enumerate(ds):
+        yield {
+            "id": str(row.get("id", idx)),
+            "image_value": row[image_column],
+            "source": f"{dataset_name}:{split}:{idx}",
+            "extra": dict(row),
+        }
+
+
+def iter_hf_local_disk(path: Path):
+    try:
+        from datasets import load_from_disk
+    except Exception as e:
+        raise RuntimeError("datasets is required for --hf-load-from-disk. Install with: pip install datasets") from e
+
+    ds = load_from_disk(str(path))
+
+    if hasattr(ds, "keys"):
+        if "train" in ds:
+            ds = ds["train"]
+        else:
+            first_key = next(iter(ds.keys()))
+            ds = ds[first_key]
+
+    image_column = None
+    for c in ds.column_names:
+        if c.lower() in {"image", "img"}:
+            image_column = c
+            break
+    if image_column is None:
+        for c in ds.column_names:
+            if "image" in c.lower():
+                image_column = c
+                break
+    if image_column is None:
+        raise ValueError("Could not find an image column in the local HF dataset.")
+
+    for idx, row in enumerate(ds):
+        yield {
+            "id": str(row.get("id", idx)),
+            "image_value": row[image_column],
+            "source": f"{path}:{idx}",
+            "extra": dict(row),
+        }
+
+
+def pick_input_iterator(
+    input_path: str | None,
+    hf_dataset: str | None,
+    hf_load_from_disk: str | None,
+    image_column: str,
+):
+    if hf_dataset:
+        return iter_hf_dataset_repo(hf_dataset, split="train", config_name=None)
+
+    if hf_load_from_disk:
+        return iter_hf_local_disk(Path(hf_load_from_disk))
+
+    if input_path is None:
+        raise ValueError("You must provide an input path or a Hugging Face dataset source.")
+
+    path = Path(input_path)
+
+    if path.is_dir():
+        return iter_folder_images(path)
+
+    if path.is_file() and is_image_path(path):
+        return [{
+            "id": path.stem,
+            "image_value": str(path),
+            "source": str(path),
+            "extra": {},
+        }]
+
+    if path.is_file() and path.suffix.lower() in {".csv", ".tsv", ".json", ".jsonl", ".parquet"}:
+        return iter_tabular_records(path, image_column=image_column)
+
+    raise ValueError(f"Unsupported input source: {input_path}")
+
+
+# ============================================================
+# Dataset row building
+# ============================================================
+
+def make_dataset_row(
+    item: dict,
+    grid_size: int,
+    alpha_threshold: int,
+) -> dict:
+    image, resolved_path = image_from_value(item["image_value"], base_dir=item.get("base_dir"))
+    result = encode_image(image=image, grid_size=grid_size, alpha_threshold=alpha_threshold)
+
+    d = result["decision"]
+    braille = result["braille"]
+    meta = braille_metadata(braille)
+
+    width, height = result["image"].size
+
+    row = {
+        "id": item.get("id"),
+        "source": item.get("source"),
+        "image": resolved_path if resolved_path is not None else None,
+        "original_width": width,
+        "original_height": height,
+        "grid_size": grid_size,
+        "braille_text": braille,
+        **meta,
+        "mode": d["mode"],
+        "reason": d["reason"],
+        "visible_box": list(map(int, d["visible_box"])),
+        "selected_box": list(map(int, d["box"])),
+        "salient_box": list(map(int, d["bbox"])),
+        "metrics": {k: jsonable(v) for k, v in (d["metrics"] or {}).items()},
+        "outline_info": {k: jsonable(v) for k, v in (d["outline_info"] or {}).items() if k != "expanded_mask"},
+    }
+
+    extra = item.get("extra", {})
+    for k, v in extra.items():
+        if k not in row:
+            row[k] = jsonable(v)
+
+    return row
+
+
+def build_dataset_rows(
+    iterator,
+    grid_size: int,
+    alpha_threshold: int,
+    limit: int | None = None,
+):
+    rows = []
+    for idx, item in enumerate(iterator):
+        if limit is not None and idx >= limit:
+            break
+        rows.append(make_dataset_row(item, grid_size=grid_size, alpha_threshold=alpha_threshold))
+    return rows
+
+
+# ============================================================
+# Dataset export
+# ============================================================
+
+def export_jsonl(rows: list[dict], out_path: Path):
+    with out_path.open("w", encoding="utf-8") as f:
+        for row in rows:
+            f.write(json.dumps(row, ensure_ascii=False) + "\n")
+
+
+def export_csv(rows: list[dict], out_path: Path):
+    import pandas as pd
+
+    serializable_rows = []
+    for row in rows:
+        new_row = {}
+        for k, v in row.items():
+            if isinstance(v, (list, dict)):
+                new_row[k] = json.dumps(v, ensure_ascii=False)
+            else:
+                new_row[k] = v
+        serializable_rows.append(new_row)
+
+    pd.DataFrame(serializable_rows).to_csv(out_path, index=False)
+
+
+def export_parquet(rows: list[dict], out_path: Path):
+    import pandas as pd
+
+    serializable_rows = []
+    for row in rows:
+        new_row = {}
+        for k, v in row.items():
+            if isinstance(v, (list, dict)):
+                new_row[k] = json.dumps(v, ensure_ascii=False)
+            else:
+                new_row[k] = v
+        serializable_rows.append(new_row)
+
+    pd.DataFrame(serializable_rows).to_parquet(out_path, index=False)
+
+
+def rows_to_hf_dataset(rows: list[dict]):
+    try:
+        from datasets import Dataset, Image as HFImage
+    except Exception as e:
+        raise RuntimeError("datasets is required for HF export. Install with: pip install datasets") from e
+
+    ds = Dataset.from_list(rows)
+
+    # If the dataset has an image-path column with local files, cast it.
+    if "image" in ds.column_names:
+        image_values = ds["image"]
+        if any(v for v in image_values if isinstance(v, str)):
+            try:
+                ds = ds.cast_column("image", HFImage())
+            except Exception:
+                pass
+
+    return ds
+
+
+def export_hf_disk(rows: list[dict], out_dir: Path):
+    ds = rows_to_hf_dataset(rows)
+    ds.save_to_disk(str(out_dir))
+    return ds
+
+
+def push_hf_dataset(rows: list[dict], repo_id: str, private: bool = False):
+    ds = rows_to_hf_dataset(rows)
+    ds.push_to_hub(repo_id, private=private)
+    return ds
+
+
+# ============================================================
+# CLI
+# ============================================================
+
+def main():
+    parser = argparse.ArgumentParser(
+        description="v12 attention-based RGBA -> 64x64 -> Unicode Braille, with HF-friendly dataset export"
+    )
+
+    parser.add_argument("input", nargs="?", help="Image file, folder, or table file")
+    parser.add_argument("-o", "--output", default="braille_output.txt", help="Single-image text output path")
+    parser.add_argument("--grid-size", type=int, default=64, help="Default: 64")
+    parser.add_argument("--alpha-threshold", type=int, default=8)
+    parser.add_argument("--save-debug", action="store_true")
+    parser.add_argument("--debug-dir", default="braille_debug")
+
+    # Dataset / HF options
+    parser.add_argument("--dataset-out", help="Write dataset rows to .jsonl / .csv / .parquet")
+    parser.add_argument("--hf-out", help="Save as datasets.Dataset.save_to_disk(...) folder")
+    parser.add_argument("--push-to-hub", help="Push the generated dataset to Hugging Face Hub")
+    parser.add_argument("--private", action="store_true", help="Use with --push-to-hub")
+    parser.add_argument("--image-column", default="image", help='For CSV/JSON/Parquet inputs. Default: "image"')
+    parser.add_argument("--limit", type=int, default=None, help="Optional row limit for dataset mode")
+
+    # HF input
+    parser.add_argument("--hf-dataset", help="Load a Hugging Face dataset repo, e.g. user/dataset")
+    parser.add_argument("--hf-load-from-disk", help="Load a dataset previously saved with save_to_disk(...)")
+    parser.add_argument("--split", default="train", help="HF split. Default: train")
+    parser.add_argument("--config-name", default=None, help="Optional HF config/subset name")
+
+    args = parser.parse_args()
+
+    # -----------------------------------------
+    # Single-image mode
+    # -----------------------------------------
+    if (
+        args.input
+        and Path(args.input).is_file()
+        and is_image_path(args.input)
+        and not args.dataset_out
+        and not args.hf_out
+        and not args.push_to_hub
+        and not args.hf_dataset
+        and not args.hf_load_from_disk
+    ):
+        result = encode_image(
+            image_path=args.input,
+            grid_size=args.grid_size,
+            alpha_threshold=args.alpha_threshold,
+        )
+
+        Path(args.output).write_text(result["braille"], encoding="utf-8")
+
+        d = result["decision"]
+
+        print(result["braille"])
+        print()
+        print("Mode:", d["mode"])
+        print("Reason:", d["reason"])
+        print("Visible box:", d["visible_box"])
+        print("Selected box:", d["box"])
+
+        if d["outline_info"] is not None:
+            print("Color outline layers:", d["outline_info"]["outline_layers"])
+            print("Pixels per outline layer:", d["outline_info"]["layer_px"])
+
+        if d["metrics"] is not None:
+            m = d["metrics"]
+            print("Strong saliency retained:", f'{m["strong_saliency_coverage"] * 100:.1f}%')
+            print("Saliency retained:", f'{m["saliency_coverage"] * 100:.1f}%')
+            print("Weighted detail retained:", f'{m["detail_coverage"] * 100:.1f}%')
+            print("Visible pixels retained:", f'{m["visible_pixel_coverage"] * 100:.1f}%')
+            print("Resolution gain:", f'{m["resolution_gain"]:.2f}x')
+
+        print("Saved:", args.output)
+
+        if args.save_debug:
+            save_debug(result, Path(args.debug_dir))
+            print("Debug files:", args.debug_dir)
+
+        return
+
+    # -----------------------------------------
+    # Dataset mode
+    # -----------------------------------------
+    if args.hf_dataset:
+        iterator = iter_hf_dataset_repo(args.hf_dataset, split=args.split, config_name=args.config_name)
+    elif args.hf_load_from_disk:
+        iterator = iter_hf_local_disk(Path(args.hf_load_from_disk))
+    else:
+        iterator = pick_input_iterator(
+            input_path=args.input,
+            hf_dataset=None,
+            hf_load_from_disk=None,
+            image_column=args.image_column,
+        )
+
+    rows = build_dataset_rows(
+        iterator,
+        grid_size=args.grid_size,
+        alpha_threshold=args.alpha_threshold,
+        limit=args.limit,
+    )
+
+    if args.dataset_out:
+        out_path = Path(args.dataset_out)
+        suffix = out_path.suffix.lower()
+        if suffix == ".jsonl":
+            export_jsonl(rows, out_path)
+        elif suffix == ".csv":
+            export_csv(rows, out_path)
+        elif suffix == ".parquet":
+            export_parquet(rows, out_path)
+        else:
+            raise ValueError("dataset-out must end with .jsonl, .csv, or .parquet")
+        print("Saved dataset rows:", out_path)
+
+    ds = None
+
+    if args.hf_out:
+        ds = export_hf_disk(rows, Path(args.hf_out))
+        print("Saved HF dataset to disk:", args.hf_out)
+
+    if args.push_to_hub:
+        ds = push_hf_dataset(rows, args.push_to_hub, private=args.private)
+        print("Pushed HF dataset:", args.push_to_hub)
+
+    if not args.dataset_out and not args.hf_out and not args.push_to_hub:
+        # Default dataset fallback: write JSONL if not explicitly specified.
+        out_path = Path("braille_dataset.jsonl")
+        export_jsonl(rows, out_path)
+        print("Saved dataset rows:", out_path)
+
+    print("Rows:", len(rows))
+    if rows:
+        print("Braille size:", f'{rows[0]["braille_width_chars"]}x{rows[0]["braille_height_chars"]} chars')
+        print("Source grid:", f'{rows[0]["grid_size"]}x{rows[0]["grid_size"]}')
+
+
+if __name__ == "__main__":
+    main()

unified/ablations/unified-v1-blt-none-s42/source/canvas_flow.py

@@ -0,0 +1,1025 @@
+"""CanvasFlow Lab: independently implemented discrete-flow canvas experiments.
+
+No recurrent-depth weight sharing. Image locations are arbitrary masks. Text is
+an ordered byte lane; geometry supplies capacity coordinates, the final renderer
+supplies exact word-wrapped coordinates. See the notebook for the distinction.
+"""
+from __future__ import annotations
+import argparse, contextlib, copy, hashlib, html, json, math, os, random, re
+import tempfile, time, warnings, pickle, inspect
+from dataclasses import asdict, dataclass, replace
+from pathlib import Path
+import numpy as np
+from PIL import Image, ImageDraw
+import torch
+from torch import nn
+import torch.nn.functional as F
+
+FREE, TEXT, IMAGE, RESERVED, UNKNOWN = range(5)
+EOS, PAD, MASK = 256, 257, 258
+BYTE_VOCAB = 259
+SCHEMA = 1
+FAMILIES = ('shapes', 'tree', 'house', 'fish', 'rocket', 'chart', 'waves', 'terrain', 'network', 'texture')
+POLICIES = ('row', 'columns', 'reverse_columns')
+DOTS = ((0, 0), (1, 0), (2, 0), (0, 1), (1, 1), (2, 1), (3, 0), (3, 1))
+
+def digest(obj):
+    return hashlib.sha256(json.dumps(obj, sort_keys=True).encode()).hexdigest()
+
+def atomic_json(path, obj):
+    path = Path(path); path.parent.mkdir(parents=True, exist_ok=True)
+    tmp = path.with_suffix(path.suffix + '.tmp')
+    tmp.write_text(json.dumps(obj, indent=2, allow_nan=False)); os.replace(tmp, path)
+
+def seed_all(seed):
+    random.seed(seed); np.random.seed(seed); torch.manual_seed(seed)
+    if torch.cuda.is_available(): torch.cuda.manual_seed_all(seed)
+
+def pack_braille(bitmap):
+    a = np.asarray(bitmap, dtype=bool)
+    if a.ndim != 2 or a.shape[0] % 4 or a.shape[1] % 2:
+        raise ValueError('Braille bitmap must have height divisible by 4 and width by 2')
+    out = np.zeros((a.shape[0] // 4, a.shape[1] // 2), np.uint8)
+    for bit, (y, x) in enumerate(DOTS): out |= a[y::4, x::2].astype(np.uint8) << bit
+    return out
+
+def unpack_braille(cells):
+    a = np.asarray(cells, np.uint8); out = np.zeros((a.shape[0] * 4, a.shape[1] * 2), bool)
+    for bit, (y, x) in enumerate(DOTS): out[y::4, x::2] = ((a >> bit) & 1).astype(bool)
+    return out
+
+def dilate(mask, radius=1):
+    mask = np.asarray(mask, bool)
+    if radius < 0: raise ValueError('negative margin')
+    p = np.pad(mask, radius)
+    out = np.zeros_like(mask)
+    for dy in range(2 * radius + 1):
+        for dx in range(2 * radius + 1): out |= p[dy:dy+mask.shape[0], dx:dx+mask.shape[1]]
+    return out
+
+def flow_runs(allowed, policy='columns', region_order=None):
+    """Horizontal runs linked only through unambiguous overlap across rows.
+
+    Split/merge events start new regions: left/right columns beside an obstacle
+    do not collapse into one region merely because they connect below it.
+    region_order can explicitly supply any permutation of the resulting regions.
+    """
+    allowed = np.asarray(allowed, bool); runs = []; previous = []; next_region = 0
+    for y, row in enumerate(allowed):
+        edges = np.flatnonzero(np.diff(np.r_[False, row, False].astype(np.int8)))
+        current = [(y, int(a), int(b)) for a, b in zip(edges[::2], edges[1::2])]
+        overlap = [[j for j, p in enumerate(previous) if max(a, p[1]) < min(b, p[2])]
+                   for _, a, b in current]
+        uses = [sum(j in js for js in overlap) for j in range(len(previous))]
+        tagged = []
+        for r, js in zip(current, overlap):
+            if len(js) == 1 and uses[js[0]] == 1: region = previous[js[0]][3]
+            else: region = next_region; next_region += 1
+            tagged.append((*r, region))
+        runs.extend(tagged); previous = tagged
+    if policy == 'row' and region_order is None: return runs
+    first = {}
+    for y, a, b, region in runs: first.setdefault(region, (y, a))
+    if region_order is not None:
+        if sorted(region_order) != sorted(first): raise ValueError('region_order must be a permutation')
+        rank = {r: i for i, r in enumerate(region_order)}
+    else:
+        if policy not in POLICIES: raise ValueError(policy)
+        sign = -1 if policy == 'reverse_columns' else 1
+        # Regions beginning on the same row are columns; merging regions follow.
+        order = sorted(first, key=lambda r: (first[r][0], sign * first[r][1]))
+        rank = {r: i for i, r in enumerate(order)}
+    return sorted(runs, key=lambda r: (rank[r[3]], r[0], r[1]))
+
+def capacity_map(layout, length, policy='columns', region_order=None):
+    """Content-independent, provisional coordinates. NEVER derived from targets."""
+    coords = np.zeros((length, 2), np.int16); regions = np.zeros(length, np.int16)
+    valid = np.zeros(length, bool); k = 0
+    for y, a, b, region in flow_runs(np.asarray(layout) == TEXT, policy, region_order):
+        n = min(b - a, length - k)
+        if n <= 0: break
+        coords[k:k+n, 0] = y; coords[k:k+n, 1] = np.arange(a, a+n)
+        regions[k:k+n] = region + 1; valid[k:k+n] = True; k += n
+    return coords, regions, valid
+
+def wrap_words(text, layout, policy='columns', region_order=None):
+    """ASCII wrapper. Preserves normalized byte order and never crosses obstacles.
+
+    A word is moved intact to the next run that can hold it. Only words longer
+    than EVERY run are split. Whitespace is normalized explicitly. Overflow is
+    returned, never silently lost. Returned mapping is the exact rendered map.
+    """
+    text.encode('ascii', errors='strict')
+    normalized = ' '.join(text.split())
+    runs = flow_runs(np.asarray(layout) == TEXT, policy, region_order)
+    coords, indices, rendered = [], [], []
+    if not runs: return dict(text=normalized, chars='', coords=[], indices=[], overflow=normalized)
+    longest = max(b-a for _, a, b, _ in runs)
+    ri = 0; cursor = runs[0][1]; consumed = 0
+    for wi, word in enumerate(normalized.split(' ')):
+        if not word: continue
+        piece = (' ' if wi else '') + word
+        # Keep the actual separating space in the byte-to-cell map.
+        if len(piece) <= longest:
+            while ri < len(runs) and runs[ri][2] - cursor < len(piece):
+                ri += 1
+                if ri < len(runs): cursor = runs[ri][1]
+            if ri == len(runs): break
+        for ch in piece:
+            while ri < len(runs) and cursor >= runs[ri][2]:
+                ri += 1
+                if ri < len(runs): cursor = runs[ri][1]
+            if ri == len(runs): break
+            coords.append((runs[ri][0], cursor)); indices.append(consumed)
+            rendered.append(ch); cursor += 1; consumed += 1
+        if ri == len(runs): break
+    return dict(text=normalized, chars=''.join(rendered), coords=coords,
+                indices=indices, overflow=normalized[consumed:])
+
+def draw_picture(rng, family, side=64):
+    """Original procedural binary graphics, not downloaded or generated captions."""
+    im = Image.new('L', (side, side), 0); d = ImageDraw.Draw(im)
+    p = lambda x: int(x * side / 64)
+    def line(points, width=2): d.line([(p(x), p(y)) for x,y in points], fill=255, width=max(1,p(width)))
+    if family == 'tree':
+        d.polygon([(p(32),p(3)),(p(8),p(42)),(p(56),p(42))], fill=255)
+        d.rectangle((p(28),p(39),p(36),p(61)),fill=255)
+    elif family == 'house':
+        d.polygon([(p(4),p(28)),(p(32),p(4)),(p(60),p(28))],fill=255)
+        d.rectangle((p(12),p(28),p(52),p(60)),outline=255,width=2)
+        d.rectangle((p(27),p(39),p(38),p(60)),outline=255,width=2)
+        d.rectangle((p(17),p(34),p(23),p(42)),fill=255)
+    elif family == 'fish':
+        d.ellipse((p(5),p(15),p(48),p(48)), fill=255)
+        d.polygon([(p(43),p(31)),(p(61),p(15)),(p(61),p(49))],fill=255)
+        d.ellipse((p(13),p(24),p(18),p(29)),fill=0)
+    elif family == 'rocket':
+        d.polygon([(p(32),p(2)),(p(43),p(24)),(p(43),p(48)),(p(21),p(48)),(p(21),p(24))],fill=255)
+        d.ellipse((p(27),p(23),p(37),p(33)),fill=0)
+        line([(24,50),(20,62)]); line([(32,50),(32,61)]); line([(40,50),(44,62)])
+    elif family == 'chart':
+        line([(5,4),(5,58),(61,58)],1)
+        for j in range(5):
+            x = 10+j*10; h = int(rng.integers(5,48))
+            d.rectangle((p(x),p(57-h),p(x+6),p(57)),fill=255)
+    elif family == 'waves':
+        for j in range(4):
+            yy = 10+j*14 + np.sin(np.linspace(0, rng.uniform(5,15),side))*rng.uniform(2,5)
+            d.line(list(zip(range(side),[p(v) for v in yy])),fill=255,width=1)
+    elif family == 'terrain':
+        heights = rng.integers(12,42,9)
+        line([(i*8,int(y)) for i,y in enumerate(heights)])
+        line([(0,57),(64,57)],1); d.ellipse((p(44),p(2),p(55),p(13)),outline=255,width=2)
+    elif family == 'network':
+        pts = rng.integers(8,56,(7,2))
+        for a,b in zip(pts, np.roll(pts,1,axis=0)): line([a,b],1)
+        for x,y in pts: d.ellipse((p(x-3),p(y-3),p(x+3),p(y+3)),fill=255)
+    elif family == 'texture':
+        x,y=np.meshgrid(np.arange(side),np.arange(side))
+        arr = ((x//int(rng.integers(3,10)) + y//int(rng.integers(3,10)))%2)*255
+        return arr.astype(np.uint8)>0
+    else:
+        for _ in range(int(rng.integers(2,6))):
+            x,y = rng.integers(2,40,2); w,h = rng.integers(8,23,2)
+            box=(p(x),p(y),p(x+w),p(y+h))
+            if rng.random()<.5: d.ellipse(box,outline=255,width=2)
+            else: d.rectangle(box,outline=255,width=2)
+    if rng.random()<.5: im=im.transpose(Image.Transpose.FLIP_LEFT_RIGHT)
+    # Shift the drawing within its source crop, not just the canvas placement.
+    a=np.asarray(im)>127
+    return np.roll(a, tuple(rng.integers(-3,4,2)),axis=(0,1))
+
+def source_text(rng, families, scene_id):
+    subjects = ', '.join(families) if families else 'an empty page'
+    verbs = ('examined','described','compared','sketched','measured','revisited')
+    names = ('Mira','Leo','Ada','Noah','Iris','Omar','Lina','Theo')
+    name = str(rng.choice(names)); verb = str(rng.choice(verbs)); n=int(rng.integers(2,40))
+    types = [
+        f'{name} {verb} {subjects}. The picture belongs to study {scene_id}. The notes stay in reading order even when the page has several separate regions.',
+        f'Observation {scene_id}: {subjects}. We counted {n} marks in the draft and added {n+3} in the revision. The new total is {2*n+3}. This is a worked arithmetic example, not a count of visible objects.',
+        f'Field notes by {name}. First inspect {subjects}. Next compare the outline and the empty spaces. Finally write a short explanation. Keep every word away from the drawing.',
+        f'{name} opened a notebook at sunrise. A study of {subjects} filled part of the page. The remaining space held a short story about a careful observer and a surprising discovery.',
+        f'Record {scene_id}. Subject: {subjects}. Author: {name}. Revision: {n}. Status: reviewed. The diagram and the text are two views of the same small experiment.',
+    ]
+    text = str(rng.choice(types))
+    if rng.random()<.35: text += ' ' + str(rng.choice(types))
+    return text
+
+def make_example(index, split='train', size=128, text_len=512, prompt_len=192, seed=731, encoding_override=None):
+    split_code={'train':0,'val':1,'test':2,'ood':3}[split]
+    rng=np.random.default_rng(np.random.SeedSequence([seed,split_code,index]))
+    kind=('text','image','mixed')[index%3]
+    encoding=int(rng.random()<.25)  # 0=Braille bytes, 1=ASCII image glyphs
+    if encoding_override is not None:encoding=int(encoding_override)
+    policy=POLICIES[int(rng.integers(0,3))]
+    image=np.zeros((size,size),np.uint8); occupied=np.zeros((size,size),bool)
+    families=[]; objects=[]
+    if kind!='text':
+        count=int(rng.integers(1,4 if split!='ood' else 6))
+        for j in range(count):
+            family=str(rng.choice(FAMILIES)); families.append(family)
+            source_side=int(rng.choice([32,64,64,64,96,128]))
+            bitmap=draw_picture(rng,family,source_side)
+            if rng.random()<.3: bitmap=np.rot90(bitmap,int(rng.integers(1,4)))
+            cells=pack_braille(bitmap)
+            if encoding:
+                block=bitmap.reshape(source_side//4,4,source_side//2,2).mean(axis=(1,3))
+                ramp=np.frombuffer(b' .:-=+*#%@',np.uint8)
+                cells=ramp[np.round(block*(len(ramp)-1)).astype(int)]
+                ink=cells!=32
+            else: ink=cells!=0
+            # 64x64 source pixels -> 16x32 Braille cells. Small test canvases resize.
+            h=min(cells.shape[0],max(4,size//2)); w=min(cells.shape[1],max(4,size//2))
+            if (h,w)!=cells.shape:
+                cells=np.asarray(Image.fromarray(cells).resize((w,h),Image.Resampling.NEAREST))
+                ink=cells!=(32 if encoding else 0)
+            y=int(rng.integers(0,size-h+1)); x=int(rng.integers(0,size-w+1))
+            if split=='ood': # Unseen edge contacts and diagonal chained placement.
+                y=int((size-h)*j/max(1,count-1)); x=int((size-w)*(count-1-j)/max(1,count-1))
+            box=bool(rng.random()<.25)
+            mask=np.ones_like(ink) if box else ink
+            sub=image[y:y+h,x:x+w]; sub[mask]=cells[mask]
+            occupied[y:y+h,x:x+w]|=mask
+            objects.append(dict(family=family,y=y,x=x,h=h,w=w,box=box))
+    margin=int(rng.integers(1,4)); blocked=dilate(occupied,margin)
+    layout=np.full((size,size),FREE,np.uint8)
+    if kind!='image':
+        if objects and rng.random()<.8:
+            # Put the reading region around an image wherever it happens to be.
+            # Otherwise short 512-byte text at row zero would rarely test wrapping.
+            focus=objects[int(rng.integers(len(objects)))]; pad=int(rng.integers(4,17))
+            y0=max(0,focus['y']-2); y1=min(size,focus['y']+focus['h']+pad)
+            x0=max(0,focus['x']-pad); x1=min(size,focus['x']+focus['w']+pad)
+        elif rng.random()<.5:
+            y0=int(rng.integers(0,max(1,size//2))); y1=size
+            x0=int(rng.integers(0,max(1,size//3))); x1=size-int(rng.integers(0,max(1,size//4)))
+        else: y0,x0,y1,x1=0,0,size,size
+        layout[y0:y1,x0:x1]=TEXT
+    # Additional gutters can create disconnected text regions without image boxes.
+    if kind!='image' and rng.random()<.4:
+        mid=int(rng.integers(size//3,2*size//3)); layout[:,mid:mid+1]=RESERVED
+    layout[blocked]=RESERVED; layout[occupied]=IMAGE
+    text=source_text(rng,families,f'{split}-{index}') if kind!='image' else ''
+    text=text[:text_len-1]
+    if len(text)==text_len-1 and ' ' in text: text=text.rsplit(' ',1)[0]
+    # Short canvases have a real capacity constraint. Determine a fitting prefix.
+    rendered=wrap_words(text,layout,policy)
+    if rendered['overflow']: text=rendered['chars'].rstrip()
+    tokens=np.full(text_len,PAD,np.int16); b=text.encode('ascii')
+    tokens[:len(b)]=list(b); tokens[len(b)]=EOS
+    geometry='; '.join(f"{o['family']} at {o['x']},{o['y']}" for o in objects)
+    prompt=f"{kind}; {'ASCII' if encoding else 'Braille'}; order {policy}; margin {margin}; {geometry}. Write notes."[:prompt_len]
+    p=np.full(prompt_len,PAD,np.int16); p[:len(prompt)]=list(prompt.encode('ascii'))
+    coords,regions,capacity=capacity_map(layout,text_len,policy)
+    return dict(image=image,layout=layout,text=tokens,prompt=p,coords=coords,
+                regions=regions,capacity=capacity,encoding=np.int64(encoding),order=np.int64(POLICIES.index(policy)),
+                meta=dict(index=index,split=split,kind=kind,families=families,objects=objects,
+                          text=text,prompt=prompt,policy=policy,margin=margin))
+
+ARRAY_KEYS=('image','layout','text','prompt','coords','regions','capacity','encoding','order')
+
+def build_data(root, count=4096, size=128, text_len=512, prompt_len=192, seed=731):
+    root=Path(root); root.mkdir(parents=True,exist_ok=True)
+    generator_source='\n'.join(inspect.getsource(fn) for fn in
+        (pack_braille,dilate,flow_runs,capacity_map,wrap_words,draw_picture,source_text,make_example))
+    config=dict(schema=SCHEMA,count=count,size=size,text_len=text_len,prompt_len=prompt_len,seed=seed,
+                generator_sha=digest(dict(source=generator_source,families=FAMILIES,policies=POLICIES,dots=DOTS)))
+    manifest=root/'manifest.json'
+    if manifest.exists():
+        old=json.loads(manifest.read_text())
+        if old['config']!=config: raise ValueError('Dataset config/code mismatch; choose a fresh data directory')
+        for name,sha in old['sha256'].items():
+            if hashlib.sha256((root/name).read_bytes()).hexdigest()!=sha: raise ValueError(f'Data checksum mismatch: {name}')
+        return old
+    checksums={}; counts={'train':count,'val':max(32,count//16),'test':max(32,count//16),'ood':max(32,count//16)}
+    for split,n in counts.items():
+        arrays=None; metadata=[]
+        for i in range(n):
+            ex=make_example(i,split,size,text_len,prompt_len,seed)
+            if arrays is None: arrays={k:np.empty((n,*np.asarray(ex[k]).shape),np.asarray(ex[k]).dtype) for k in ARRAY_KEYS}
+            for k in ARRAY_KEYS: arrays[k][i]=ex[k]
+            metadata.append(ex['meta'])
+        for k,a in arrays.items():
+            name=f'{split}_{k}.npy'; tmp=root/(name+'.tmp')
+            with open(tmp,'wb') as f: np.save(f,a)
+            os.replace(tmp,root/name); checksums[name]=hashlib.sha256((root/name).read_bytes()).hexdigest()
+        name=f'{split}_meta.json'; atomic_json(root/name,metadata)
+        checksums[name]=hashlib.sha256((root/name).read_bytes()).hexdigest()
+        print(f'data {split}: {n} examples',flush=True)
+    result=dict(config=config,counts=counts,sha256=checksums,fingerprint=digest(checksums))
+    atomic_json(manifest,result); return result
+
+class CanvasData:
+    def __init__(self, root, split='train', device='cpu', preload=False):
+        root=Path(root); self.manifest=json.loads((root/'manifest.json').read_text())
+        self.data={k:np.load(root/f'{split}_{k}.npy',mmap_mode='r') for k in ARRAY_KEYS}
+        self.n=len(self.data['text']); self.device=device
+        self.raster_cache=None
+        if preload:
+            self.data={k:torch.from_numpy(np.array(v)).to(device) for k,v in self.data.items()}
+    def batch(self, indices, variant='full'):
+        idx=np.asarray(indices)%self.n
+        out={k:(v[torch.as_tensor(idx,device=v.device)].long() if torch.is_tensor(v)
+                else torch.as_tensor(np.array(v[idx]),device=self.device).long()) for k,v in self.data.items()}
+        if variant=='raster':
+            if self.raster_cache is None:
+                # Build ONCE, outside steady-state timings after the first batch.
+                layouts=self.data['layout'].cpu().numpy() if torch.is_tensor(self.data['layout']) else self.data['layout']
+                maps=[capacity_map(x,out['text'].shape[1],'row') for x in layouts]
+                self.raster_cache=tuple(torch.as_tensor(np.stack([x[j] for x in maps]),device=self.device).long() for j in (0,1))
+            out['coords']=self.raster_cache[0][torch.as_tensor(idx,device=self.device)]
+            out['regions']=self.raster_cache[1][torch.as_tensor(idx,device=self.device)]
+        return out
+
+def decode_text(tokens):
+    values=[]
+    for t in np.asarray(tokens).reshape(-1):
+        if t==EOS: break
+        if 0<=t<256: values.append(int(t))
+    return bytes(values).decode('ascii',errors='replace')
+
+def page_html(image, layout, text, encoding=0, policy='columns', title='Canvas'):
+    rendered=wrap_words(text,layout,policy); size=len(layout)
+    page=np.full((size,size),' ',dtype='<U1')
+    for y,x in np.argwhere(layout==IMAGE):
+        v=int(image[y,x]); page[y,x]=chr(0x2800+v) if not encoding else chr(v if 32<=v<127 else 63)
+    for (y,x),ch in zip(rendered['coords'],rendered['chars']): page[y,x]=ch
+    body='\n'.join(''.join(row) for row in page)
+    return f'<section><h2>{html.escape(title)}</h2><pre>{html.escape(body)}</pre><p>Overflow: {html.escape(rendered["overflow"])}</p></section>'
+
+def gallery(root, output, n=9, split='val'):
+    root=Path(root); meta=json.loads((root/f'{split}_meta.json').read_text())
+    images=np.load(root/f'{split}_image.npy',mmap_mode='r'); layouts=np.load(root/f'{split}_layout.npy',mmap_mode='r')
+    enc=np.load(root/f'{split}_encoding.npy',mmap_mode='r')
+    parts=[page_html(images[i],layouts[i],meta[i]['text'],int(enc[i]),meta[i]['policy'],meta[i]['prompt']) for i in range(min(n,len(meta)))]
+    Path(output).write_text('<!doctype html><meta charset="utf-8"><style>body{background:#111827;color:#e5e7eb;font:14px system-ui}section{margin:24px}pre{font:10px/12px monospace;white-space:pre;border:1px solid #475569;display:inline-block;padding:12px}h2{font-size:15px}</style>'+''.join(parts))
+
+@dataclass
+class Config:
+    size: int=128
+    text_len: int=512
+    prompt_len: int=192
+    dim: int=256
+    depth: int=6
+    heads: int=4
+    patch: int=4
+    local_dim: int=32
+    variant: str='full'
+    objective: str='dfm'
+    checkpoint: bool=False
+    def validate(self):
+        if self.size%self.patch or self.dim%self.heads or self.dim%2:
+            raise ValueError('Incompatible canvas/patch/head dimensions')
+        if self.variant not in ('full','no_flow','raster','patch8','mdlm'): raise ValueError(self.variant)
+        if self.objective not in ('dfm','mdlm'): raise ValueError(self.objective)
+
+def positional(t, dim):
+    freq=torch.exp(-math.log(10000)*torch.arange(dim//2,device=t.device).float()/max(1,dim//2-1))
+    phase=t.float()[...,None]*freq
+    return torch.cat((phase.sin(),phase.cos()),-1)
+
+class Block(nn.Module):
+    def __init__(self, dim, heads):
+        super().__init__(); self.heads=heads
+        self.n1=nn.LayerNorm(dim); self.n2=nn.LayerNorm(dim)
+        self.qkv=nn.Linear(dim,dim*3,bias=False); self.proj=nn.Linear(dim,dim,bias=False)
+        self.up=nn.Linear(dim,dim*8,bias=False); self.down=nn.Linear(dim*4,dim,bias=False)
+    def forward(self,h):
+        b,n,d=h.shape
+        q,k,v=self.qkv(self.n1(h)).reshape(b,n,3,self.heads,d//self.heads).permute(2,0,3,1,4).unbind(0)
+        # Fused SDPA chooses a supported kernel. No 16K x 16K full-canvas mask.
+        z=F.scaled_dot_product_attention(q,k,v,dropout_p=0.0,is_causal=False)
+        h=h+self.proj(z.transpose(1,2).reshape(b,n,d))
+        a,g=self.up(self.n2(h)).chunk(2,-1)
+        return h+self.down(F.silu(a)*g)
+
+class CanvasModel(nn.Module):
+    """Two modes, same implementation: small layout planner or content denoiser.
+
+    Content uses a single shared byte embedding and transformer for the spatial
+    patches, linguistic byte lane, and byte prompt. Every block has unique weights.
+    """
+    def __init__(self,c,planner=False):
+        super().__init__(); c.validate(); self.c=c; self.planner=planner
+        self.byte=nn.Embedding(BYTE_VOCAB,c.dim)
+        self.local=nn.Linear(c.dim,c.local_dim,bias=False)
+        self.occupancy=nn.Embedding(5,c.local_dim)
+        self.to_patch=nn.Linear(c.local_dim*c.patch*c.patch,c.dim)
+        self.to_cells=nn.Linear(c.dim,c.local_dim*c.patch*c.patch)
+        self.x=nn.Embedding(c.size,c.dim); self.y=nn.Embedding(c.size,c.dim)
+        self.region=nn.Embedding(257,c.dim)
+        self.modality=nn.Embedding(4,c.dim)
+        self.format=nn.Embedding(2,c.dim)
+        self.time=nn.Sequential(nn.Linear(c.dim,c.dim),nn.SiLU(),nn.Linear(c.dim,c.dim))
+        self.blocks=nn.ModuleList([Block(c.dim,c.heads) for _ in range(c.depth)])
+        self.norm=nn.LayerNorm(c.dim)
+        self.local_norm=nn.LayerNorm(c.local_dim)
+        self.local_conv=nn.Conv2d(c.local_dim,c.local_dim,3,padding=1,groups=c.local_dim)
+        self.image_head=nn.Linear(c.local_dim,4 if planner else 256)
+        self.text_head=nn.Linear(c.dim,258,bias=False) if not planner else None
+        self.order_head=nn.Linear(c.dim,3) if planner else None
+        gy,gx=torch.meshgrid(torch.arange(c.size//c.patch)*c.patch+c.patch//2,
+                            torch.arange(c.size//c.patch)*c.patch+c.patch//2,indexing='ij')
+        self.register_buffer('gx',gx.flatten(),persistent=False); self.register_buffer('gy',gy.flatten(),persistent=False)
+        self.apply(self._init)
+        for block in self.blocks:
+            nn.init.normal_(block.proj.weight,std=.02/math.sqrt(2*c.depth))
+            nn.init.normal_(block.down.weight,std=.02/math.sqrt(2*c.depth))
+    @staticmethod
+    def _init(m):
+        if isinstance(m,(nn.Linear,nn.Embedding)):
+            nn.init.normal_(m.weight,std=.02)
+            if isinstance(m,nn.Linear) and m.bias is not None: nn.init.zeros_(m.bias)
+    def forward(self,image,text,layout,prompt,coords,regions,encoding,t):
+        c=self.c; b=image.shape[0]; s=c.size; p=c.patch; g=s//p
+        local=self.local(self.byte(image))+self.occupancy(layout)
+        patches=local.reshape(b,g,p,g,p,c.local_dim).permute(0,1,3,2,4,5).reshape(b,g*g,-1)
+        spatial=self.to_patch(patches)+self.x(self.gx)+self.y(self.gy)+self.modality.weight[0]+self.format(encoding)[:,None]
+        words=self.byte(text)+self.modality.weight[1]
+        if not self.planner:
+            words=words+self.y(coords[...,0])+self.x(coords[...,1])
+            if c.variant!='no_flow':
+                flow=positional(torch.arange(c.text_len,device=image.device),c.dim)
+                words=words+flow[None]
+            words=words+self.region(regions.clamp(0,256))
+        else:
+            words=words+positional(torch.arange(c.text_len,device=image.device),c.dim)[None]
+        condition=self.byte(prompt)+self.modality.weight[2]
+        condition=condition+positional(torch.arange(c.prompt_len,device=image.device),c.dim)[None]
+        h=torch.cat((spatial,words,condition),1)+self.time(positional(t*1000,c.dim))[:,None]
+        for block in self.blocks:
+            if c.checkpoint and self.training:
+                from torch.utils.checkpoint import checkpoint
+                h=checkpoint(block,h,use_reentrant=False)
+            else: h=block(h)
+        h=self.norm(h)
+        z=self.to_cells(h[:,:g*g]).reshape(b,g,g,p,p,c.local_dim).permute(0,1,3,2,4,5).reshape(b,s,s,c.local_dim)
+        z=z+local
+        z=z+self.local_conv(z.permute(0,3,1,2)).permute(0,2,3,1)
+        img=self.image_head(self.local_norm(z))
+        if self.planner:
+            return img,self.order_head(h[:,-c.prompt_len:].mean(1))
+        return img,self.text_head(h[:,g*g:g*g+c.text_len])
+
+def stratified_time(batch,device):
+    # Forward time: 0=all masked, 1=data. A random shift stratifies the batch.
+    return ((torch.rand((),device=device)+torch.arange(batch,device=device)/batch)%1)*.998+.001
+
+def conditioning(batch):
+    """Task dropout: joint / image->text / text->image / arbitrary infill."""
+    img=batch['image']; txt=batch['text']; device=img.device; b=len(img)
+    mode=torch.randint(0,4,(b,),device=device)
+    has_image=(batch['layout']==IMAGE).flatten(1).any(-1)
+    has_text=(batch['layout']==TEXT).flatten(1).any(-1)
+    # Do not ask a text-only page to generate an absent image (or vice versa).
+    mode=torch.where(((mode==2)&~has_image)|((mode==1)&~has_text),0,mode)
+    ki=(mode==1)[:,None,None].expand_as(img).clone()
+    kt=(mode==2)[:,None].expand_as(txt).clone()
+    ki|=(mode==3)[:,None,None] & (torch.rand(img.shape,device=device)<.35)
+    kt|=(mode==3)[:,None] & (torch.rand(txt.shape,device=device)<.35)
+    ki&=batch['layout']==IMAGE
+    kt|=(~has_text)[:,None]
+    return ki,kt
+
+def masked_ce(logits,target,masked,eligible,t,objective,weights=None):
+    """Explicit modality-normalized posterior CE, optionally 1/(1-t) weighted.
+
+    MDLM mode uses the absorbing linear-path time weight; optional class/PAD
+    balancing makes the overall canvas objective a surrogate, not a reported ELBO.
+    """
+    ce=F.cross_entropy(logits.float().reshape(-1,logits.shape[-1]),target.reshape(-1),reduction='none').reshape_as(target)
+    w=eligible.float() if weights is None else eligible.float()*weights
+    den=w.flatten(1).sum(-1).clamp_min(1)
+    value=(ce*w*masked).flatten(1).sum(-1)/den
+    if objective=='mdlm': value=value/(1-t)
+    return value.mean()
+
+def make_loss(model,batch,planner=False,objective='dfm',fixed_t=None):
+    b=len(batch['image']); dev=batch['image'].device
+    t=stratified_time(b,dev) if fixed_t is None else torch.full((b,),fixed_t,device=dev)
+    ki,kt=conditioning(batch); active=batch['layout']==IMAGE
+    im=torch.where(ki,batch['image'],MASK); txt=torch.where(kt,batch['text'],MASK)
+    if planner:
+        # Observed symbols constrain occupancy; unknown cells carry no GT class.
+        known=ki.clone()
+        known|=torch.rand(active.shape,device=dev)<.10
+        noise=(torch.rand(active.shape,device=dev)>t[:,None,None]) & ~known
+        layout=torch.where(noise,UNKNOWN,batch['layout'])
+        logits,order=model(im,txt,layout,batch['prompt'],batch['coords'],batch['regions'],batch['encoding'],t)
+        weights=torch.tensor([1.,1.,6.,2.],device=dev)[batch['layout']]
+        loss=masked_ce(logits,batch['layout'],noise,~known,t,objective,weights)
+        loss=loss+.15*F.cross_entropy(order.float(),batch['order'])
+        return loss
+    mi=(torch.rand(active.shape,device=dev)>t[:,None,None])&~ki&active
+    mt=(torch.rand(txt.shape,device=dev)>t[:,None])&~kt
+    im=torch.where(mi,MASK,batch['image']); im=torch.where(active,im,PAD)
+    txt=torch.where(mt,MASK,batch['text'])
+    il,tl=model(im,txt,batch['layout'],batch['prompt'],batch['coords'],batch['regions'],batch['encoding'],t)
+    text_weight=torch.where(batch['text']==PAD,.05,1.)
+    li=masked_ce(il,batch['image'],mi,active&~ki,t,objective)
+    lt=masked_ce(tl,batch['text'],mt,~kt,t,objective,text_weight)
+    return li+lt
+
+def autocast(device,enabled=True):
+    return torch.autocast('cuda',dtype=torch.bfloat16,enabled=enabled) if str(device).startswith('cuda') else contextlib.nullcontext()
+
+def sample_categorical(logits,temperature=1.,greedy=False):
+    if greedy or temperature==0: return logits.argmax(-1)
+    probs=(logits.float()/temperature).softmax(-1)
+    return torch.multinomial(probs.reshape(-1,probs.shape[-1]),1).reshape(probs.shape[:-1])
+
+@torch.no_grad()
+def sample_layout(model,batch,steps=8,known_layout=None,known_image=None,known_text=None,greedy=False):
+    if steps<1: raise ValueError('At least one sampling step is required')
+    c=model.c; dev=batch['image'].device; b=len(batch['image'])
+    # A known layout uses UNKNOWN at unconstrained cells, including free cells.
+    lay=torch.full_like(batch['layout'],UNKNOWN) if known_layout is None else known_layout.clone()
+    locked=lay!=UNKNOWN
+    im=torch.full_like(batch['image'],MASK) if known_image is None else known_image
+    txt=torch.full_like(batch['text'],MASK) if known_text is None else known_text
+    was=model.training; model.eval()
+    for i in range(steps):
+        t=torch.full((b,),i/steps,device=dev)
+        with autocast(dev): logits,order=model(im,txt,lay,batch['prompt'],batch['coords'],batch['regions'],batch['encoding'],t)
+        proposals=sample_categorical(logits,greedy=greedy)
+        # Integrated linear-path reveal hazard: (t_next-t)/(1-t).
+        reveal=(torch.rand(lay.shape,device=dev)<1/(steps-i))&(lay==UNKNOWN)&~locked
+        lay=torch.where(reveal,proposals,lay)
+    model.train(was)
+    return lay,order.argmax(-1)
+
+def remap_batch(batch,layout,order,variant='full'):
+    result=dict(batch); result['layout']=layout
+    maps=[capacity_map(l.cpu().numpy(),batch['text'].shape[1],
+                      'row' if variant=='raster' else POLICIES[int(o)]) for l,o in zip(layout,order)]
+    result['coords']=torch.as_tensor(np.stack([m[0] for m in maps]),device=layout.device).long()
+    result['regions']=torch.as_tensor(np.stack([m[1] for m in maps]),device=layout.device).long()
+    result['capacity']=torch.as_tensor(np.stack([m[2] for m in maps]),device=layout.device)
+    return result
+
+@torch.no_grad()
+def sample_content(model,batch,steps=8,known_image=None,known_text=None,greedy=False):
+    if steps<1: raise ValueError('At least one sampling step is required')
+    dev=batch['image'].device; b=len(batch['image']); active=batch['layout']==IMAGE
+    im=torch.full_like(batch['image'],MASK) if known_image is None else known_image.clone()
+    im=torch.where(active,im,PAD)
+    txt=torch.full_like(batch['text'],MASK) if known_text is None else known_text.clone()
+    no_text=~(batch['layout']==TEXT).flatten(1).any(-1)
+    empty=torch.full_like(txt,PAD); empty[:,0]=EOS
+    # A layout with no text region has an empty new text lane. Explicit supplied
+    # text is preserved so overflow can be reported rather than silently erased.
+    if known_text is None:
+        txt=torch.where(no_text[:,None],empty,txt)
+    else:
+        entirely_unknown=(known_text==MASK).all(-1)
+        txt=torch.where((no_text&entirely_unknown)[:,None],empty,txt)
+    was=model.training; model.eval()
+    for i in range(steps):
+        t=torch.full((b,),i/steps,device=dev)
+        with autocast(dev): il,tl=model(im,txt,batch['layout'],batch['prompt'],batch['coords'],batch['regions'],batch['encoding'],t)
+        # Image glyphs are printable ASCII; text remains unrestricted UTF-8 bytes.
+        allowed=torch.arange(256,device=dev)
+        il=il.float().masked_fill((batch['encoding']==1)[:,None,None,None]&((allowed<32)|(allowed>126)), -torch.inf)
+        tl=tl.float()  # Preserve all 256 text bytes, EOS and PAD.
+        ip=sample_categorical(il,greedy=greedy); tp=sample_categorical(tl,greedy=greedy)
+        ri=(torch.rand(im.shape,device=dev)<1/(steps-i))&(im==MASK)&active
+        rt=(torch.rand(txt.shape,device=dev)<1/(steps-i))&(txt==MASK)
+        im=torch.where(ri,ip,im); txt=torch.where(rt,tp,txt)
+    model.train(was)
+    return torch.where(active,im,0),txt
+
+def rng_state():
+    return dict(torch=torch.get_rng_state(),cuda=torch.cuda.get_rng_state_all() if torch.cuda.is_available() else [],
+                numpy=np.random.get_state(),python=random.getstate())
+
+def restore_rng(state):
+    torch.set_rng_state(state['torch'].cpu()); np.random.set_state(state['numpy']); random.setstate(state['python'])
+    if state['cuda']: torch.cuda.set_rng_state_all([x.cpu() for x in state['cuda']])
+
+def cpu_tree(x):
+    if torch.is_tensor(x): return x.detach().cpu().clone()
+    if isinstance(x,dict): return {k:cpu_tree(v) for k,v in x.items()}
+    if isinstance(x,list): return [cpu_tree(v) for v in x]
+    if isinstance(x,tuple): return tuple(cpu_tree(v) for v in x)
+    return x
+
+def save_checkpoint(path,model,opt,step,signature,extra):
+    path=Path(path); path.parent.mkdir(parents=True,exist_ok=True)
+    payload=dict(model=cpu_tree(model.state_dict()),optimizer=cpu_tree(opt.state_dict()),step=step,
+                 rng=rng_state(),signature=signature,extra=extra,config=asdict(model.c),planner=model.planner)
+    tmp=path.with_suffix('.tmp'); torch.save(payload,tmp)
+    # Previous generation survives an interrupted write or corrupt last file.
+    if path.exists(): os.replace(path,path.with_suffix('.previous.pt'))
+    os.replace(tmp,path)
+
+def read_checkpoint(path,signature=None):
+    path=Path(path); errors=[]
+    for candidate in (path,path.with_suffix('.previous.pt')):
+        if not candidate.exists(): continue
+        try: value=torch.load(candidate,map_location='cpu',weights_only=False)
+        except (OSError,RuntimeError,EOFError,ValueError,IndexError,pickle.UnpicklingError) as e:
+            errors.append(str(e)); continue
+        if signature is not None and value['signature']!=signature:
+            raise ValueError('Checkpoint config/data/code/runtime differs. Use a new run directory.')
+        return value
+    raise RuntimeError(f'No readable checkpoint: {errors}')
+
+def finite_tensors(tensors):
+    tensors=[x for x in tensors if x.numel()]
+    if not tensors: return True
+    # One host synchronization, not one per tensor.
+    return bool(torch.stack([torch.isfinite(x).all() for x in tensors]).all())
+
+def finite_optimizer(opt):
+    return finite_tensors([v for s in opt.state.values() for v in s.values() if torch.is_tensor(v)])
+
+def batch_indices(step,batch_size,accum,n,seed):
+    # A full deterministic permutation per epoch; resume depends only on accepted step.
+    ids=np.arange(step*batch_size*accum,(step+1)*batch_size*accum)
+    result=np.empty_like(ids)
+    for epoch in np.unique(ids//n):
+        sel=ids//n==epoch
+        perm=np.random.default_rng(np.random.SeedSequence([seed,int(epoch)])).permutation(n)
+        result[sel]=perm[ids[sel]%n]
+    return result.reshape(accum,batch_size)
+
+def synchronize(device):
+    if str(device).startswith('cuda'): torch.cuda.synchronize()
+
+def train(root,run,c,steps=300,batch_size=4,accum=1,lr=3e-4,seed=42,planner=False,
+          compile_model=False,preload=False,save_every=50,resume=True,device=None,
+          stop_after=None,inject_nonfinite_at=None,inject_poststep_at=None):
+    """Single-GPU training. No train-time diffusion rollouts; one time per sample.
+
+    stop_after is for exact-resume tests, and does not alter the LR schedule.
+    Fallback changes are counted, logged, and invalidate strict untouched comparisons.
+    """
+    device=device or ('cuda' if torch.cuda.is_available() else 'cpu')
+    run=Path(run); run.mkdir(parents=True,exist_ok=True); seed_all(seed)
+    if str(device).startswith('cuda'):
+        if not torch.cuda.is_bf16_supported(): raise RuntimeError('This profile requires CUDA BF16 support')
+        torch.set_float32_matmul_precision('high')
+    data=CanvasData(root,'train',device,preload)
+    c.validate(); model=CanvasModel(c,planner).to(device)
+    opt=torch.optim.AdamW(model.parameters(),lr=lr,betas=(.9,.95),weight_decay=.1,
+                         fused=str(device).startswith('cuda'))
+    # Architecture-dependent parameter initialization must not change paired
+    # task/corruption draws across ablations with different projection sizes.
+    seed_all(seed+1_000_000)
+    signature=digest(dict(config=asdict(c),planner=planner,data=data.manifest['fingerprint'],steps=steps,
+                          batch_size=batch_size,accum=accum,lr=lr,seed=seed,torch=torch.__version__,
+                          device_type=str(device).split(':')[0],compile=compile_model,
+                          code=hashlib.sha256(Path(__file__).read_bytes()).hexdigest()))
+    last=run/'last.pt'; start=0; retries=0; rollbacks=0; lr_mult=1.; ema=0.
+    if resume and last.exists():
+        state=read_checkpoint(last,signature); model.load_state_dict(state['model']); opt.load_state_dict(state['optimizer'])
+        restore_rng(state['rng']); start=state['step']; extra=state['extra']
+        retries=extra['retries']; rollbacks=extra['rollbacks']; lr_mult=extra['lr_mult']; ema=extra['grad_ema']
+        if start==steps and (run/'summary.json').exists():
+            saved=json.loads((run/'summary.json').read_text())
+            if saved.get('signature')==signature:
+                print(f'Already complete: {run}; retaining measured timing summary',flush=True)
+                return model,saved
+    elif last.exists(): raise FileExistsError('Refusing to overwrite existing run; use a fresh directory')
+    else:
+        save_checkpoint(last,model,opt,0,signature,dict(retries=0,rollbacks=0,lr_mult=1.,grad_ema=0.))
+    forward=torch.compile(model,dynamic=False) if compile_model else model
+    compile_active=compile_model
+    print(json.dumps(dict(run=str(run),planner=planner,params=sum(p.numel() for p in model.parameters()),
+                          attention_tokens=(c.size//c.patch)**2+c.text_len+c.prompt_len,device=str(device),start=start)),flush=True)
+    step=start; wall=time.perf_counter(); timings=[]; injected=False; injected_post=False
+    log=run/'train.jsonl'
+    if str(device).startswith('cuda'): torch.cuda.reset_peak_memory_stats()
+    while step<steps and (stop_after is None or step<stop_after):
+        ids=batch_indices(step,batch_size,accum,data.n,seed)
+        batches=[data.batch(i,c.variant) for i in ids]
+        replay=rng_state(); base_lr=lr*min(1.,(step+1)/min(30,max(1,steps//10)))
+        base_lr*=.1+.9*.5*(1+math.cos(math.pi*step/max(1,steps)))
+        accepted=False; attempt_records=[]; synchronize(device); begin=time.perf_counter()
+        try:
+            for attempt,(amp,backoff,clip) in enumerate(((True,1.,1.),(True,.25,.5),(False,.1,.25))):
+                restore_rng(replay); opt.zero_grad(set_to_none=True); loss_value=0.
+                for group in opt.param_groups: group['lr']=base_lr*lr_mult*backoff
+                good=True
+                for batch in batches:
+                    with autocast(device,amp): loss=make_loss(forward,batch,planner,c.objective)/accum
+                    if not bool(torch.isfinite(loss)):
+                        good=False; break
+                    loss_value+=float(loss.detach()); loss.backward()
+                if inject_nonfinite_at==step and not injected:
+                    next(p for p in model.parameters() if p.grad is not None).grad.flatten()[0]=torch.inf
+                    injected=True
+                params=[p for p in model.parameters() if p.grad is not None]
+                good=good and finite_tensors([p.grad for p in params])
+                norm=float(torch.nn.utils.clip_grad_norm_(params,clip)) if good else float('inf')
+                spike=math.isfinite(norm) and norm>max(100.,20*ema)
+                if not good or not math.isfinite(norm) or (spike and attempt==0):
+                    retries+=1; attempt_records.append(dict(attempt=attempt,reason='spike' if spike else 'nonfinite'))
+                    continue
+                opt.step()
+                if inject_poststep_at==step and not injected_post:
+                    with torch.no_grad(): next(model.parameters()).flatten()[0]=torch.nan
+                    injected_post=True
+                if not finite_tensors(list(model.parameters())) or not finite_optimizer(opt):
+                    raise FloatingPointError('Nonfinite parameter or optimizer state after update')
+                # Reduced LR affects this update only; all fallback events are counted.
+                accepted=True; ema=norm if ema==0 else .95*ema+.05*norm; break
+            if not accepted: raise FloatingPointError('All precision/clip/LR fallbacks exhausted')
+        except torch.OutOfMemoryError:
+            opt.zero_grad(set_to_none=True)
+            raise RuntimeError(f'CUDA OOM. Last known-good checkpoint: {last}. Reduce the common batch or enable checkpointing in a NEW suite; no examples were silently skipped.')
+        except FloatingPointError as exc:
+            rollbacks+=1
+            if rollbacks>3: raise RuntimeError(f'Recovery budget exhausted: {exc}') from exc
+            state=read_checkpoint(last,signature); model.load_state_dict(state['model']); opt.load_state_dict(state['optimizer'])
+            restore_rng(state['rng']); step=state['step']; ema=state['extra']['grad_ema']; lr_mult*=.25
+            with open(log,'a') as f: f.write(json.dumps(dict(event='rollback',step=step,reason=str(exc),lr_mult=lr_mult))+'\n')
+            continue
+        except Exception as exc:
+            if compile_active and (type(exc).__module__.startswith(('torch._dynamo','torch._inductor'))):
+                compile_active=False; forward=model; restore_rng(replay)
+                with open(log,'a') as f: f.write(json.dumps(dict(event='compile_fallback',reason=str(exc)[:300]))+'\n')
+                continue
+            raise
+        synchronize(device); elapsed=time.perf_counter()-begin
+        step+=1
+        if step>start+3: timings.append(elapsed)
+        record=dict(step=step,loss=loss_value,grad_norm=norm,seconds=elapsed,lr=opt.param_groups[0]['lr'],
+                    retries=retries,rollbacks=rollbacks,attempts=attempt_records)
+        with open(log,'a') as f: f.write(json.dumps(record,allow_nan=False)+'\n')
+        if step==1 or step%25==0: print(json.dumps(record),flush=True)
+        if step%save_every==0 or step==steps or step==stop_after:
+            save_checkpoint(last,model,opt,step,signature,dict(retries=retries,rollbacks=rollbacks,lr_mult=lr_mult,grad_ema=ema))
+    summary=dict(config=asdict(c),planner=planner,seed=seed,step=step,target_steps=steps,params=sum(p.numel() for p in model.parameters()),
+                 accepted_pages=step*batch_size*accum,batch_size=batch_size,accum=accum,
+                 retries=retries,rollbacks=rollbacks,unchanged_recipe=retries==0 and rollbacks==0,
+                 resume_from=start,session_seconds=time.perf_counter()-wall,
+                 median_step_seconds=float(np.median(timings)) if timings else None,
+                 pages_per_second=batch_size*accum/float(np.median(timings)) if timings else None,
+                 peak_gpu_GB=torch.cuda.max_memory_allocated()/1e9 if str(device).startswith('cuda') else None,
+                 compile_requested=compile_model,compile_active=compile_active,signature=signature,
+                 data_fingerprint=data.manifest['fingerprint'],device=str(device),torch=torch.__version__)
+    atomic_json(run/'summary.json',summary)
+    return model,summary
+
+def load_model(path,device=None):
+    device=device or ('cuda' if torch.cuda.is_available() else 'cpu')
+    ck=read_checkpoint(path); c=Config(**ck['config']); model=CanvasModel(c,ck['planner']).to(device)
+    model.load_state_dict(ck['model']); model.eval(); return model
+
+@torch.no_grad()
+def evaluate(model,data,planner_model=None,batches=4,batch_size=2,steps=8):
+    """Loss probes and true rollout metrics. Oracle geometry is labeled explicitly."""
+    was=model.training; model.eval(); device=next(model.parameters()).device
+    losses=[]; text_hits=0; text_count=0; image_hits=0; image_count=0
+    dot_correct=0; dot_count=0; known_checks=0; known_total=0; overflow=0
+    intersections=0; unions=0; generated_pages=0; invalid_padding=0; elapsed=0.
+    prior=rng_state(); seed_all(2029)
+    try:
+        for i in range(batches):
+            b=data.batch(np.arange(i*batch_size,(i+1)*batch_size),model.c.variant)
+            for t in (.1,.5,.9):
+                # A common unweighted probe is comparable across training objectives.
+                with autocast(device): losses.append(float(make_loss(model,b,False,'dfm',t)))
+            if planner_model is not None:
+                synchronize(device); begin=time.perf_counter()
+                layout,order=sample_layout(planner_model,b,steps)
+                used=remap_batch(b,layout,order,model.c.variant)
+            else:
+                synchronize(device); begin=time.perf_counter(); used=b; layout=b['layout']; order=b['order']
+            img,txt=sample_content(model,used,steps)
+            synchronize(device); elapsed+=time.perf_counter()-begin
+            active=b['layout']==IMAGE; predicted=layout==IMAGE
+            intersections+=int((active&predicted).sum()); unions+=int((active|predicted).sum())
+            image_hits+=int(((img==b['image'])&active&predicted).sum()); image_count+=int(active.sum())
+            mask=b['text']!=PAD; text_hits+=int(((txt==b['text'])&mask).sum()); text_count+=int(mask.sum())
+            for j in range(len(img)):
+                truth=b['image'][j].cpu().numpy().astype(np.uint8); guess=img[j].cpu().numpy().astype(np.uint8)
+                if int(b['encoding'][j])==0:
+                    bits=unpack_braille(truth)^unpack_braille(guess)
+                    eval_mask=np.repeat(np.repeat((active[j]|predicted[j]).cpu().numpy(),4,0),2,1)
+                    dot_correct+=int((~bits&eval_mask).sum()); dot_count+=int(eval_mask.sum())
+                pol='row' if model.c.variant=='raster' else POLICIES[int(order[j])]
+                wrapped=wrap_words(decode_text(txt[j].cpu().numpy()).replace('\ufffd','?'),layout[j].cpu().numpy(),pol)
+                overflow+=int(bool(wrapped['overflow'])); generated_pages+=1
+                ts=txt[j].cpu().numpy(); end=np.flatnonzero(ts==EOS)
+                invalid_padding+=int(bool(np.any(ts[:end[0] if len(end) else len(ts)]==PAD)))
+            # A separate arbitrary partial-content test verifies clamping exactly.
+            ki=torch.rand(b['image'].shape,device=device)<.2; kt=torch.rand(b['text'].shape,device=device)<.2
+            known_i=torch.where(ki,b['image'],MASK); known_t=torch.where(kt,b['text'],MASK)
+            ci,ct=sample_content(model,b,min(steps,2),known_i,known_t)
+            ki&=active
+            known_checks+=int(((ci==b['image'])&ki).sum()+((ct==b['text'])&kt).sum())
+            known_total+=int(ki.sum()+kt.sum())
+    finally: restore_rng(prior); model.train(was)
+    return dict(geometry='predicted' if planner_model is not None else 'oracle_diagnostic',sampling_steps=steps,
+                content_nfe=steps,planner_nfe=steps if planner_model is not None else 0,
+                common_masked_ce=float(np.mean(losses)),text_byte_accuracy=text_hits/max(1,text_count),
+                image_cell_accuracy=image_hits/max(1,image_count),image_occupancy_iou=intersections/max(1,unions),
+                braille_dot_accuracy=dot_correct/max(1,dot_count),known_preservation=known_checks/max(1,known_total),
+                overflow_page_rate=overflow/max(1,generated_pages),early_pad_page_rate=invalid_padding/max(1,generated_pages),
+                generation_seconds_per_page=elapsed/max(1,generated_pages),pages=generated_pages,
+                metric_note='Exact target accuracy is a paired diagnostic; multiple valid texts/layouts exist. This is not perplexity or FID.')
+
+def benchmark(root,c,device='cuda',batches=(1,2,4,8),repeats=5):
+    """Actual forward+backward+Adam timing, including steady-state memory.
+
+    A disposable model keeps tuning updates out of the experiment checkpoints.
+    Run per candidate variant; choose ONE common batch for controlled comparisons.
+    """
+    results=[]
+    for batch_size in batches:
+        model=opt=data=batch=None
+        try:
+            seed_all(42); model=CanvasModel(c).to(device)
+            opt=torch.optim.AdamW(model.parameters(),lr=1e-4,fused=str(device).startswith('cuda'))
+            data=CanvasData(root,'train',device); batch=data.batch(np.arange(batch_size),c.variant)
+            if str(device).startswith('cuda'): torch.cuda.reset_peak_memory_stats()
+            samples=[]
+            for j in range(repeats+2):
+                synchronize(device); begin=time.perf_counter(); opt.zero_grad(set_to_none=True)
+                with autocast(device): loss=make_loss(model,batch,objective=c.objective)
+                loss.backward(); torch.nn.utils.clip_grad_norm_(model.parameters(),1.); opt.step(); synchronize(device)
+                if j>=2: samples.append(time.perf_counter()-begin)
+            med=float(np.median(samples))
+            results.append(dict(batch_size=batch_size,seconds=med,pages_per_second=batch_size/med,
+                                peak_GB=torch.cuda.max_memory_allocated()/1e9 if str(device).startswith('cuda') else None))
+        except torch.OutOfMemoryError:
+            results.append(dict(batch_size=batch_size,oom=True)); break
+        finally:
+            del model,opt,data,batch
+            if str(device).startswith('cuda'): torch.cuda.empty_cache()
+    return results
+
+def generate_page(model,planner,prompt,layout=None,image=None,image_known=None,
+                  text=None,text_tokens=None,fixed_text=None,encoding=0,policy=None,
+                  steps=8,planner_steps=8,seed=123,greedy=False,region_order=None):
+    """Public generation API; no source example or ground truth is needed.
+
+    layout: full explicit occupancy, or UNKNOWN at unplanned cells.
+    image_known: arbitrary cell mask; zero-valued known Braille cells stay known.
+    text: optional complete ASCII text, or text_tokens with MASK for unknown bytes.
+    fixed_text: {(y,x): ASCII character}; immutable canvas text layer, separate
+      from the newly generated text lane. Planner is forbidden to overwrite it.
+    policy: optional reading order override; otherwise planner predicts one.
+    """
+    c=model.c; dev=next(model.parameters()).device; s=c.size
+    if policy is not None and policy not in POLICIES: raise ValueError(policy)
+    if encoding not in (0,1): raise ValueError('encoding must be 0 Braille or 1 ASCII')
+    lay=np.full((s,s),UNKNOWN,np.int64) if layout is None else np.array(layout,dtype=np.int64,copy=True)
+    if lay.shape!=(s,s) or np.any((lay<0)|(lay>UNKNOWN)): raise ValueError('Invalid layout')
+    im=np.zeros((s,s),np.int64) if image is None else np.asarray(image,dtype=np.int64)
+    known=np.zeros((s,s),bool) if image_known is None else np.asarray(image_known,bool)
+    if im.shape!=(s,s) or known.shape!=(s,s) or np.any((im<0)|(im>255)): raise ValueError('Invalid image')
+    if np.any(known&(lay!=UNKNOWN)&(lay!=IMAGE)): raise ValueError('Known image conflicts with explicit occupancy')
+    lay[known]=IMAGE
+    fixed_text=fixed_text or {}
+    for (y,x),char in fixed_text.items():
+        if not (0<=y<s and 0<=x<s and len(char)==1 and 32<=ord(char)<127): raise ValueError('Invalid fixed text cell')
+        if known[y,x] or lay[y,x]==IMAGE: raise ValueError('Text/image overlap in user constraints')
+        lay[y,x]=RESERVED
+    if fixed_text:
+        context=''.join(fixed_text[p] for p in sorted(fixed_text))
+        prompt=f'{prompt} Existing text: {context}'
+    prompt.encode('ascii',errors='strict')
+    if len(prompt)>c.prompt_len: raise ValueError(f'Prompt exceeds {c.prompt_len} ASCII bytes; shorten it explicitly')
+    p=np.full(c.prompt_len,PAD,np.int64); p[:len(prompt)]=list(prompt.encode('ascii'))
+    kt=np.full(c.text_len,MASK,np.int64)
+    if text is not None and text_tokens is not None: raise ValueError('Pass text OR text_tokens')
+    if text is not None:
+        raw=text.encode('ascii',errors='strict')
+        if len(raw)>=c.text_len: raise ValueError('Text exceeds lane capacity including EOS')
+        kt[:]=PAD; kt[:len(raw)]=list(raw); kt[len(raw)]=EOS
+    if text_tokens is not None:
+        kt=np.asarray(text_tokens,np.int64)
+        if kt.shape!=(c.text_len,) or np.any((kt<0)|(kt>MASK)): raise ValueError('Invalid text_tokens')
+    tensor=lambda a:torch.as_tensor(a,device=dev).long()[None]
+    b=dict(image=tensor(im),layout=tensor(lay),text=tensor(kt),prompt=tensor(p),
+           coords=torch.zeros(1,c.text_len,2,dtype=torch.long,device=dev),
+           regions=torch.zeros(1,c.text_len,dtype=torch.long,device=dev),encoding=tensor(np.asarray(encoding)),
+           order=tensor(np.asarray(POLICIES.index(policy or 'columns'))))
+    old=rng_state(); seed_all(seed)
+    try:
+        known_i=tensor(np.where(known,im,MASK)); known_t=tensor(kt)
+        if np.any(lay==UNKNOWN):
+            if planner is None: raise ValueError('Unknown occupancy requires a trained planner')
+            planned,order=sample_layout(planner,b,planner_steps,b['layout'],known_i,known_t,greedy)
+        else: planned,order=b['layout'],b['order']
+        if policy is not None: order.fill_(POLICIES.index(policy))
+        mapped=remap_batch(b,planned,order,c.variant)
+        if region_order is not None:
+            cc,rr,_=capacity_map(planned[0].cpu().numpy(),c.text_len,POLICIES[int(order[0])],region_order)
+            mapped['coords']=tensor(cc); mapped['regions']=tensor(rr)
+        generated,tokens=sample_content(model,mapped,steps,known_i,known_t,greedy)
+    finally: restore_rng(old)
+    out_image=generated[0].cpu().numpy().astype(np.uint8); out_layout=planned[0].cpu().numpy().astype(np.uint8)
+    out_text=decode_text(tokens[0].cpu().numpy()).replace('\ufffd','?')
+    out_policy='row' if c.variant=='raster' else POLICIES[int(order[0])]
+    rendered=wrap_words(out_text,out_layout,out_policy,region_order)
+    # Final canvas retains fixed text cells exactly; byte indices apply to new lane.
+    canvas=np.full((s,s),' ',dtype='<U1')
+    for y,x in np.argwhere(out_layout==IMAGE): canvas[y,x]=chr(0x2800+int(out_image[y,x])) if encoding==0 else chr(int(out_image[y,x]))
+    for (y,x),char in zip(rendered['coords'],rendered['chars']): canvas[y,x]=char
+    for (y,x),char in fixed_text.items(): canvas[y,x]=char
+    return dict(canvas=canvas,image=out_image,layout=out_layout,text=out_text,
+                tokens=tokens[0].cpu().numpy(),flow_map=rendered,policy=out_policy,
+                known_image_preserved=bool(np.array_equal(out_image[known],im[known])),
+                content_nfe=steps,planner_nfe=planner_steps if np.any(lay==UNKNOWN) else 0)
+
+def self_test():
+    """Small CPU tests check contracts; they are not CUDA throughput evidence."""
+    torch.set_num_threads(min(4,os.cpu_count() or 1)); seed_all(3); results=[]
+    def ok(name): results.append(name); print('PASS',name,flush=True)
+    cells=np.arange(256,dtype=np.uint8).reshape(16,16)
+    assert np.array_equal(pack_braille(unpack_braille(cells)),cells)
+    for bit,(y,x) in enumerate(DOTS): assert unpack_braille(np.array([[1<<bit]],np.uint8))[y,x]
+    ok('all 256 Braille dot patterns round trip')
+    layout=np.full((24,24),TEXT,np.uint8); layout[:16,8:16]=IMAGE
+    runs=flow_runs(layout==TEXT,'columns')
+    assert runs[1][0]==1 and runs[1][1]==0 # Column-first, not raster hopping.
+    text='This is a test with several words beside a tall image.'
+    wrapped=wrap_words(text,layout,'columns')
+    assert wrapped['chars']==text and not wrapped['overflow']
+    assert len(set(wrapped['coords']))==len(wrapped['coords'])
+    assert all(layout[y,x]==TEXT for y,x in wrapped['coords'])
+    tiny=np.full((3,4),TEXT,np.uint8)
+    w=wrap_words('abcdefghijklmnopqrst',tiny); assert w['chars']+w['overflow']=='abcdefghijklmnopqrst'
+    assert wrap_words('hello',np.full((3,3),IMAGE))['overflow']=='hello'
+    ok('geometry regions, exact byte order, long-word splitting, explicit overflow')
+    for split in ('train','val','test','ood'):
+        for i in range(9):
+            e=make_example(i,split,size=32,text_len=64,prompt_len=96)
+            assert np.array_equal(e['coords'],capacity_map(e['layout'],64,e['meta']['policy'])[0])
+            w=wrap_words(e['meta']['text'],e['layout'],e['meta']['policy'])
+            assert not w['overflow'] and all(e['layout'][y,x]==TEXT for y,x in w['coords'])
+    assert make_example(0,'train')['meta']!=make_example(0,'val')['meta']
+    ok('deterministic split streams, arbitrary masks, no target-derived coordinates')
+    examples=[make_example(i,'train',16,32,64) for i in range(3)]
+    batch={k:torch.tensor(np.stack([e[k] for e in examples])).long() for k in ARRAY_KEYS}
+    config=Config(size=16,text_len=32,prompt_len=64,dim=32,heads=2,depth=2,patch=4,local_dim=8)
+    for planner in (False,True):
+        for variant in (('full','no_flow','raster','patch8','mdlm') if not planner else ('full',)):
+            c=replace(config,variant=variant,patch=8 if variant=='patch8' else 4,objective='mdlm' if variant=='mdlm' else 'dfm')
+            model=CanvasModel(c,planner); loss=make_loss(model,batch,planner,c.objective); loss.backward()
+            assert torch.isfinite(loss) and finite_tensors([p.grad for p in model.parameters() if p.grad is not None])
+            assert len({id(b.qkv.weight) for b in model.blocks})==c.depth
+    ok('all five ablations and planner forward/backward; unique block weights')
+    logits=torch.zeros(1,4,2); target=torch.zeros(1,4,dtype=torch.long)
+    mask=torch.tensor([[True,True,False,False]]); eligible=torch.ones_like(mask); t=torch.tensor([.5])
+    assert abs(float(masked_ce(logits,target,mask,eligible,t,'dfm'))-.5*math.log(2))<1e-6
+    assert abs(float(masked_ce(logits,target,mask,eligible,t,'mdlm'))-math.log(2))<1e-6
+    ok('DFM posterior CE and absorbing-path time weight analytic check')
+    model=CanvasModel(config); planner=CanvasModel(config,True)
+    for n in (1,4):
+        known_i=torch.full_like(batch['image'],MASK); active=batch['layout']==IMAGE
+        known_i[active]=batch['image'][active]
+        known_t=torch.full_like(batch['text'],MASK); known_t[:,:4]=batch['text'][:,:4]
+        img,txt=sample_content(model,batch,n,known_i,known_t)
+        assert not (img==MASK).any() and not (txt==MASK).any()
+        assert torch.equal(img[active],batch['image'][active]) and torch.equal(txt[:,:4],batch['text'][:,:4])
+        known_layout=torch.full_like(batch['layout'],UNKNOWN); known_layout[:,:,0]=RESERVED
+        out,order=sample_layout(planner,batch,n,known_layout)
+        assert not (out==UNKNOWN).any() and (out[:,:,0]==RESERVED).all()
+    ok('one/four-step sampling terminates; arbitrary known values and geometry clamped')
+    explicit=np.full((16,16),TEXT); explicit[5:8,7:11]=IMAGE
+    image=np.zeros((16,16),np.uint8); image[5:8,7:11]=255; known=explicit==IMAGE
+    result=generate_page(model,None,'mixed page',explicit,image,known,text='Hello world.',fixed_text={(15,15):'X'},steps=1)
+    assert result['known_image_preserved'] and result['canvas'][15,15]=='X' and result['text']=='Hello world.'
+    ok('public generation API preserves known image/text and fixed canvas cells')
+    with tempfile.TemporaryDirectory() as tmp:
+        root=Path(tmp)/'data'; build_data(root,count=9,size=16,text_len=32,prompt_len=64)
+        args=dict(root=root,c=config,steps=3,batch_size=2,accum=1,device='cpu',save_every=1,seed=91)
+        m1,_=train(run=Path(tmp)/'whole',**args)
+        train(run=Path(tmp)/'resume',stop_after=1,**args)
+        m2,_=train(run=Path(tmp)/'resume',**args)
+        assert all(torch.equal(v,m2.state_dict()[k]) for k,v in m1.state_dict().items())
+        a=read_checkpoint(Path(tmp)/'whole/last.pt'); b=read_checkpoint(Path(tmp)/'resume/last.pt')
+        for key in a['optimizer']['state']:
+            for field,x in a['optimizer']['state'][key].items():
+                assert torch.equal(x,b['optimizer']['state'][key][field]) if torch.is_tensor(x) else x==b['optimizer']['state'][key][field]
+        _,summary=train(run=Path(tmp)/'retry',inject_nonfinite_at=0,**args)
+        assert summary['step']==3 and summary['retries']==1
+        _,rollback=train(run=Path(tmp)/'rollback',inject_poststep_at=1,**args)
+        assert rollback['step']==3 and rollback['rollbacks']==1
+        corrupt=Path(tmp)/'rollback/last.pt'; corrupt.write_bytes(b'broken checkpoint')
+        recovered=read_checkpoint(corrupt); assert recovered['step']==2
+        try: read_checkpoint(Path(tmp)/'whole/last.pt',signature='wrong')
+        except ValueError: pass
+        else: raise AssertionError('Incompatible resume should fail')
+    ok('bit-exact interrupted resume including Adam; injected Inf replays without skips')
+    ok('post-update NaN rollback, previous-checkpoint recovery, signature rejection')
+    print(json.dumps(dict(passed=len(results),tests=results)),flush=True)
+    return results
+
+def main():
+    p=argparse.ArgumentParser(description=__doc__); sub=p.add_subparsers(dest='command',required=True)
+    sub.add_parser('selftest')
+    prep=sub.add_parser('prepare'); prep.add_argument('--root',required=True); prep.add_argument('--count',type=int,default=4096)
+    prep.add_argument('--size',type=int,default=128); prep.add_argument('--text-len',type=int,default=512)
+    gen=sub.add_parser('gallery'); gen.add_argument('--root',required=True); gen.add_argument('--output',required=True)
+    a=p.parse_args()
+    if a.command=='selftest': self_test()
+    elif a.command=='prepare': build_data(a.root,a.count,a.size,a.text_len)
+    else: gallery(a.root,a.output)
+
+if __name__=='__main__': main()

unified/ablations/unified-v1-blt-none-s42/source/causal_arch.py

@@ -0,0 +1,747 @@
+"""Audited byte architecture screen. See the notebook's fidelity table and sources.
+
+This is a scaled, from-scratch study, not a reproduction of published checkpoints.
+ByteFlow's whole-sequence Top-K mode is deliberately excluded from causal rankings.
+"""
+from __future__ import annotations
+
+import argparse
+import contextlib
+import copy
+import hashlib
+import json
+import math
+import os
+import random
+import sys
+import time
+from dataclasses import asdict, dataclass, replace
+from datetime import timedelta
+from pathlib import Path
+
+import numpy as np
+import torch
+from torch import nn
+from torch.nn import functional as F
+from torch.utils.checkpoint import checkpoint
+
+PAD, BOS, EOS, VOCAB = 256, 257, 258, 259
+SCHEMA = 3
+FINEWEB_REVISION = "9bb295ddab0e05d785b879661af7260fed5140fc"
+ARCHS = ("ouro_1pass", "evabyte", "blt", "byteflow_causal")
+
+
+@dataclass
+class ModelCfg:
+    dim: int = 640
+    ff: int = 1536
+    heads: int = 10
+    context: int = 4096
+    body_layers: int = 6
+    loops: int = 1
+    local_dim: int = 256
+    local_ff: int = 768
+    local_heads: int = 4
+    encoder_layers: int = 2
+    decoder_layers: int = 2
+    local_window: int = 512
+    query_block: int = 128
+    # Released EvaByte configuration, NOT the generic class defaults.
+    eva_window: int = 2048
+    eva_chunk: int = 16
+    eva_heads: int = 8
+    # Scaled BLT capacities. Hash sizes 3..8 and two latent slots are retained.
+    hash_buckets: int = 4096
+    hash_sizes: tuple = (3, 4, 5, 6, 7, 8)
+    cross_slots: int = 2
+    entropy_dim: int = 128
+    entropy_ff: int = 384
+    entropy_heads: int = 2
+    entropy_layers: int = 4
+    patch_size: float = 4.0
+    byteflow_bins: int = 16
+    byteflow_eps2: float = 1.0
+    byteflow_score: str = "l2"  # Appendix B trace approximation, on RAW states.
+    router_window: int = 64     # Explicit causal adaptation, not paper Top-K.
+    grad_checkpoint: bool = True
+
+    def validate(self):
+        for d, h in ((self.dim, self.heads), (self.local_dim, self.local_heads),
+                     (self.entropy_dim, self.entropy_heads)):
+            if d % h or (d // h) % 2:
+                raise ValueError("Each head must have an even integral dimension.")
+        if self.context < 2 or self.patch_size < 1 or self.loops < 1:
+            raise ValueError("Invalid context, patch size, or loop count.")
+        if self.eva_window % self.eva_chunk or self.eva_chunk < 1:
+            raise ValueError("EVA window must be divisible by chunk size.")
+        if min(self.local_window, self.query_block, self.router_window) < 1:
+            raise ValueError("Attention/router windows must be positive.")
+
+
+def seed_all(seed):
+    random.seed(seed)
+    np.random.seed(seed)
+    torch.manual_seed(seed)
+    if torch.cuda.is_available():
+        torch.cuda.manual_seed_all(seed)
+
+
+def documents(ids):
+    """BOS starts a new document; the first row position may be a continuation."""
+    return (ids == BOS).long().cumsum(-1)
+
+
+def ckpt(module, *args, enabled=False):
+    if enabled and module.training and torch.is_grad_enabled():
+        return checkpoint(module, *args, use_reentrant=False, preserve_rng_state=True)
+    return module(*args)
+
+
+class RMSNorm(nn.Module):
+    def __init__(self, dim, eps=1e-6, unit_offset=False):
+        super().__init__()
+        self.weight = nn.Parameter(torch.zeros(dim) if unit_offset else torch.ones(dim))
+        self.eps, self.unit_offset = eps, unit_offset
+
+    def forward(self, x):
+        # FP32 statistics AND multiplication; the residual stream stays FP32.
+        z = x.float()
+        w = self.weight.float() + (1.0 if self.unit_offset else 0.0)
+        return (z * torch.rsqrt(z.square().mean(-1, keepdim=True) + self.eps) * w).to(x.dtype)
+
+
+def rotary(q, k, theta):
+    t, d = q.shape[-2:]
+    with torch.autocast(q.device.type, enabled=False):
+        freq = theta ** (-torch.arange(0, d, 2, device=q.device, dtype=torch.float32) / d)
+        phase = torch.arange(t, device=q.device, dtype=torch.float32)[:, None] * freq
+        cs = torch.cat((phase, phase), -1).cos()[None, None]
+        sn = torch.cat((phase, phase), -1).sin()[None, None]
+        def rotate(x):
+            a, b = x.float().chunk(2, -1)
+            return (x.float() * cs + torch.cat((-b, a), -1) * sn).to(x.dtype)
+        return rotate(q), rotate(k)
+
+
+def exclusive(z, v):
+    # Half precision dot products here used to overflow before GradScaler could help.
+    a, b = z.float(), v.float()
+    return (a - (a * b).sum(-1, keepdim=True) * b /
+            b.square().sum(-1, keepdim=True).clamp_min(1e-12)).to(z.dtype)
+
+
+def sdpa(q, k, v, allowed):
+    # At least one allowed key per real query is guaranteed by callers. Empty
+    # padding queries receive a dummy key and are explicitly zeroed afterward.
+    return F.scaled_dot_product_attention(q, k, v, attn_mask=allowed,
+                                         dropout_p=0.0, is_causal=False)
+
+
+class Attention(nn.Module):
+    def __init__(self, d, heads, window=None, theta=100000.0,
+                 qk_norm=False, gated=False, xsa=False, query_block=128):
+        super().__init__()
+        self.d, self.h, self.hd = d, heads, d // heads
+        self.window, self.theta, self.query_block = window, theta, query_block
+        self.gated, self.xsa = gated, xsa
+        self.q = nn.Linear(d, d * (2 if gated else 1), bias=False)
+        self.k, self.v, self.o = [nn.Linear(d, d, bias=False) for _ in range(3)]
+        self.qn = RMSNorm(self.hd) if qk_norm else nn.Identity()
+        self.kn = RMSNorm(self.hd) if qk_norm else nn.Identity()
+
+    def forward(self, x, doc):
+        b, t, d = x.shape
+        qr = self.q(x)
+        qr, gate = qr.chunk(2, -1) if self.gated else (qr, None)
+        shape = lambda z: z.reshape(b, t, self.h, self.hd).transpose(1, 2)
+        q, k, v = self.qn(shape(qr)), self.kn(shape(self.k(x))), shape(self.v(x))
+        q, k = rotary(q, k, self.theta)
+        pieces = []
+        # Chunk queries; SWA also slices keys. There is no T x T SWA allocation.
+        for start in range(0, t, self.query_block):
+            end = min(t, start + self.query_block)
+            lo = max(0, start - self.window + 1) if self.window else 0
+            qi = torch.arange(start, end, device=x.device)[:, None]
+            ki = torch.arange(lo, end, device=x.device)[None, :]
+            allow = ki <= qi
+            if self.window:
+                allow = allow & (ki > qi - self.window)
+            allow = allow[None] & (doc[:, start:end, None] == doc[:, None, lo:end])
+            pieces.append(sdpa(q[:, :, start:end], k[:, :, lo:end], v[:, :, lo:end], allow[:, None]))
+        z = torch.cat(pieces, -2)
+        if self.xsa:
+            z = exclusive(z, v)
+        if gate is not None:
+            # Query-dependent bounded G1 sigmoid gate, not unbounded SiLU.
+            z = z * shape(gate).float().sigmoid().to(z.dtype)
+        return self.o(z.transpose(1, 2).reshape(b, t, d))
+
+
+class SwiGLU(nn.Module):
+    def __init__(self, d, ff):
+        super().__init__()
+        self.g = nn.Linear(d, ff, bias=False)
+        self.u = nn.Linear(d, ff, bias=False)
+        self.o = nn.Linear(ff, d, bias=False)
+
+    def forward(self, x):
+        return self.o(F.silu(self.g(x)) * self.u(x))
+
+
+class Block(nn.Module):
+    def __init__(self, d, ff, heads, *, window=None, theta=100000.0,
+                 sandwich=False, ouro=False, query_block=128, norm_eps=1e-6,
+                 unit_offset=False):
+        super().__init__()
+        self.n1, self.n2 = [RMSNorm(d, norm_eps, unit_offset) for _ in range(2)]
+        self.attn = Attention(d, heads, window, theta, ouro, ouro, ouro, query_block)
+        self.ff = SwiGLU(d, ff)
+        self.post_a = RMSNorm(d) if sandwich else nn.Identity()
+        self.post_f = RMSNorm(d) if sandwich else nn.Identity()
+
+    def forward(self, x, doc):
+        # Published Ouro: x + Norm(Attention(Norm(x))). Never Norm(x + branch).
+        x = x + self.post_a(self.attn(self.n1(x), doc))
+        return x + self.post_f(self.ff(self.n2(x)))
+
+
+def initialize(m):
+    if isinstance(m, (nn.Linear, nn.Embedding)):
+        nn.init.normal_(m.weight, std=0.02)
+        if isinstance(m, nn.Linear) and m.bias is not None:
+            nn.init.zeros_(m.bias)
+
+
+class OuroLM(nn.Module):
+    """User's prelude/shared-body/coda baseline, with corrected branch norms.
+
+    This deliberately does not add ByteDance's early-exit training objective.
+    """
+    def __init__(self, c):
+        super().__init__()
+        self.c = c
+        self.embed = nn.Embedding(VOCAB, c.dim)
+        make = lambda: Block(c.dim, c.ff, c.heads, sandwich=True, ouro=True,
+                             query_block=c.query_block)
+        self.prelude, self.coda = make(), make()
+        self.body = nn.ModuleList([make() for _ in range(c.body_layers)])
+        self.loop_norm, self.norm = RMSNorm(c.dim), RMSNorm(c.dim)
+        self.apply(initialize)
+
+    def forward(self, ids, loops=None):
+        loops = self.c.loops if loops is None else loops
+        if loops < 1:
+            raise ValueError("loops must be positive")
+        doc = documents(ids)
+        h = ckpt(self.prelude, self.embed(ids), doc, enabled=self.c.grad_checkpoint)
+        for _ in range(loops):
+            for layer in self.body:
+                h = ckpt(layer, h, doc, enabled=self.c.grad_checkpoint)
+            h = self.loop_norm(h)  # Also used at one pass; no unused DDP parameter.
+        h = ckpt(self.coda, h, doc, enabled=self.c.grad_checkpoint)
+        return {"logits": F.linear(self.norm(h), self.embed.weight), "heads": []}
+
+
+class EVAAttention(nn.Module):
+    """PyTorch reference-equation EVA for arbitrary lengths, including decode.
+
+    Exact intra-window attention + learned summaries from strictly earlier
+    windows. Document-crossing chunks are split into document-specific slots.
+    """
+    def __init__(self, c):
+        super().__init__()
+        self.c, self.h, self.hd = c, c.heads, c.dim // c.heads
+        self.q, self.k, self.v, self.o = [nn.Linear(c.dim, c.dim, bias=False) for _ in range(4)]
+        self.mu = nn.Parameter(torch.randn(1, self.h, 1, 1, self.hd).clamp(-1, 1) / math.sqrt(self.hd))
+        self.phi = nn.Parameter(torch.randn(1, self.h, 1, 1, self.hd).clamp(-1, 1) / math.sqrt(self.hd))
+
+    def summaries(self, k, v, doc):
+        b, h, t, d = k.shape
+        chunk = self.c.eva_chunk
+        # A summary cannot mix documents. A chunk boundary or BOS starts a slot.
+        starts = torch.zeros_like(doc, dtype=torch.bool)
+        starts[:, ::chunk] = True
+        starts[:, 1:] |= doc[:, 1:] != doc[:, :-1]
+        seg, counts = starts.long().cumsum(-1) - 1, starts.sum(-1)
+        n = int(counts.max().item())
+        pos = torch.arange(t, device=k.device).expand(b, t)
+        first = torch.full((b, n), t, device=k.device, dtype=torch.long)
+        first.scatter_reduce_(1, seg, pos, reduce="amin", include_self=True)
+        last = torch.full_like(first, -1)
+        last.scatter_reduce_(1, seg, pos, reduce="amax", include_self=True)
+        offsets = pos - first.gather(1, seg)
+        flat = (seg * chunk + offsets)[:, None, :, None].expand(b, h, t, d)
+        # Accumulate products in FP32, including key squared norms.
+        kk = k.float().new_zeros(b, h, n * chunk, d).scatter(2, flat, k.float())
+        vv = v.float().new_zeros(b, h, n * chunk, d).scatter(2, flat, v.float())
+        kk, vv = kk.view(b, h, n, chunk, d), vv.view(b, h, n, chunk, d)
+        valid = torch.arange(chunk, device=k.device)[None, None, None, :] <= (last-first)[:, None, :, None]
+        active = first < t
+        safe = valid.clone()
+        safe[:, :, :, 0] |= ~active[:, None, :]
+        with torch.autocast(k.device.type, enabled=False):
+            wm = ((kk * self.mu.float()).sum(-1)).masked_fill(~safe, -torch.inf).softmax(-1)
+            wp = (((kk * self.phi.float()).sum(-1) - 0.5 * kk.square().sum(-1)) /
+                  math.sqrt(d)).masked_fill(~safe, -torch.inf).softmax(-1)
+            kb = (wm[..., None] * kk).sum(-2)
+            vb = (wp[..., None] * vv).sum(-2)
+        sdoc = doc.gather(1, first.clamp_max(t - 1))
+        return kb.to(k.dtype), vb.to(v.dtype), last, sdoc, active
+
+    def forward(self, x, doc):
+        b, t, d = x.shape
+        reshape = lambda z: z.view(b, t, self.h, self.hd).transpose(1, 2)
+        q, k, v = reshape(self.q(x)), reshape(self.k(x)), reshape(self.v(x))
+        q, k = rotary(q, k, 100000.0)
+        kb, vb, last, sdoc, active = self.summaries(k, v, doc)
+        outputs = []
+        w = self.c.eva_window
+        for ws in range(0, t, w):
+            we = min(t, ws + w)
+            # Keep a zero dependency when there are no earlier slots; the
+            # mathematical EVA path is unchanged and reducer participation is safe.
+            keys = torch.cat((k[:, :, ws:we], kb), -2)
+            vals = torch.cat((v[:, :, ws:we], vb), -2)
+            for a in range(ws, we, self.c.query_block):
+                z = min(we, a + self.c.query_block)
+                qi = torch.arange(a, z, device=x.device)[:, None]
+                ki = torch.arange(ws, we, device=x.device)[None, :]
+                local = (ki <= qi)[None] & (doc[:, a:z, None] == doc[:, None, ws:we])
+                summary = (last[:, None, :] < ws) & active[:, None, :]
+                summary = summary & (doc[:, a:z, None] == sdoc[:, None, :])
+                allow = torch.cat((local, summary), -1)
+                outputs.append(sdpa(q[:, :, a:z], keys, vals, allow[:, None]))
+        h = torch.cat(outputs, -2).transpose(1, 2).reshape(b, t, d)
+        return self.o(h)
+
+
+class EVABlock(nn.Module):
+    def __init__(self, c):
+        super().__init__()
+        self.n1, self.n2 = [RMSNorm(c.dim, 1e-5, True) for _ in range(2)]
+        self.attn, self.ff = EVAAttention(c), SwiGLU(c.dim, c.ff)
+
+    def forward(self, x, doc):
+        x = x + self.attn(self.n1(x), doc)
+        return x + self.ff(self.n2(x))
+
+
+class EvaByteLM(nn.Module):
+    def __init__(self, c, depth):
+        super().__init__()
+        self.c = c
+        self.embed = nn.Embedding(VOCAB, c.dim)
+        self.layers = nn.ModuleList([EVABlock(c) for _ in range(depth)])
+        self.norm = RMSNorm(c.dim, 1e-5, True)
+        self.output = nn.Linear(c.dim, VOCAB * c.eva_heads, bias=False)
+        self.apply(initialize)
+
+    def forward(self, ids, loops=None):
+        doc, h = documents(ids), self.embed(ids)
+        for layer in self.layers:
+            h = ckpt(layer, h, doc, enabled=self.c.grad_checkpoint)
+        all_heads = self.output(self.norm(h)).view(*ids.shape, self.c.eva_heads, VOCAB)
+        return {"logits": all_heads[:, :, 0], "heads": list(all_heads.unbind(2))}
+
+
+class EntropyLM(nn.Module):
+    def __init__(self, c):
+        super().__init__()
+        self.c = c
+        self.embed = nn.Embedding(VOCAB, c.entropy_dim)
+        self.layers = nn.ModuleList([Block(c.entropy_dim, c.entropy_ff, c.entropy_heads,
+            window=c.local_window, theta=500000.0, query_block=c.query_block)
+            for _ in range(c.entropy_layers)])
+        self.norm = RMSNorm(c.entropy_dim)
+        self.output = nn.Linear(c.entropy_dim, VOCAB, bias=False)
+        self.register_buffer("threshold", torch.tensor(float("nan")))
+        self.apply(initialize)
+
+    def forward(self, ids, loops=None):
+        h, doc = self.embed(ids), documents(ids)
+        for layer in self.layers:
+            h = ckpt(layer, h, doc, enabled=self.c.grad_checkpoint)
+        return {"logits": self.output(self.norm(h)), "heads": []}
+
+    @torch.no_grad()
+    def entropy(self, ids):
+        logits = self(ids)["logits"].float()
+        lp = logits.log_softmax(-1)
+        return -(lp.exp() * lp).sum(-1)
+
+
+def segments(starts):
+    """Ragged routing metadata; never truncate excess patches."""
+    b, t = starts.shape
+    ids = starts.long().cumsum(-1) - 1
+    counts = starts.sum(-1)
+    n = int(counts.max().item())
+    pos = torch.arange(t, device=starts.device).expand(b, t)
+    first = torch.full((b, n), t, device=starts.device, dtype=torch.long)
+    first.scatter_reduce_(1, ids, pos, reduce="amin", include_self=True)
+    valid = torch.arange(n, device=starts.device)[None] < counts[:, None]
+    return ids, first.clamp_max(t - 1), valid
+
+
+def gather(states, idx):
+    return states.gather(1, idx[..., None].expand(*idx.shape, states.shape[-1]))
+
+
+class CrossAttention(nn.Module):
+    def __init__(self, d, heads, block):
+        super().__init__()
+        self.h, self.hd, self.block = heads, d // heads, block
+        self.nq, self.nkv = RMSNorm(d), RMSNorm(d)
+        self.q, self.k, self.v, self.o = [nn.Linear(d, d, bias=False) for _ in range(4)]
+
+    def forward(self, x, kv, qgroup, kgroup, valid_q=None):
+        b, t, d = x.shape
+        nk = kv.shape[1]
+        shape = lambda a: a.view(b, -1, self.h, self.hd).transpose(1, 2)
+        q, kn = shape(self.q(self.nq(x))), self.nkv(kv)
+        k, v = shape(self.k(kn)), shape(self.v(kn))
+        chunks = []
+        for a in range(0, t, self.block):
+            z = min(t, a + self.block)
+            allow = qgroup[:, a:z, None] == kgroup[:, None, :]
+            real = allow.any(-1)
+            if valid_q is not None:
+                real = real & valid_q[:, a:z]
+            # Avoid all-masked softmax on padded/no-context queries.
+            allow[:, :, 0] |= ~real
+            y = sdpa(q[:, :, a:z], k, v, allow[:, None])
+            y = y * real[:, None, :, None]
+            chunks.append(y)
+        z = torch.cat(chunks, -2).transpose(1, 2).reshape(b, t, d)
+        return x + self.o(z)
+
+
+class GlobalStack(nn.Module):
+    def __init__(self, c, depth, theta=500000.0):
+        super().__init__()
+        self.c = c
+        self.layers = nn.ModuleList([Block(c.dim, c.ff, c.heads,
+            theta=theta, query_block=c.query_block) for _ in range(depth)])
+        self.norm = RMSNorm(c.dim)
+
+    def forward(self, x, doc):
+        for layer in self.layers:
+            x = ckpt(layer, x, doc, enabled=self.c.grad_checkpoint)
+        return self.norm(x)
+
+
+class BLTLM(nn.Module):
+    """Scaled paper BLT: frozen independent patcher, ngrams, cross-attention.
+
+    Decoder access is aligned with the NEXT byte's patch. A current patch is
+    visible only when the prefix-only entropy decision has closed it.
+    """
+    def __init__(self, c, depth):
+        super().__init__()
+        self.c = c
+        self.patcher = EntropyLM(c).requires_grad_(False)
+        self.embed = nn.Embedding(VOCAB, c.local_dim)
+        self.hashes = nn.ModuleList([nn.Embedding(c.hash_buckets, c.local_dim) for _ in c.hash_sizes])
+        make = lambda: Block(c.local_dim, c.local_ff, c.local_heads,
+            window=c.local_window, theta=500000.0, query_block=c.query_block)
+        self.enc = nn.ModuleList([make() for _ in range(c.encoder_layers)])
+        self.dec = nn.ModuleList([make() for _ in range(c.decoder_layers)])
+        make_cross = lambda: CrossAttention(c.local_dim, c.local_heads, c.query_block)
+        self.enc_cross = nn.ModuleList([make_cross() for _ in self.enc])
+        self.dec_cross = nn.ModuleList([make_cross() for _ in self.dec])
+        self.pool_proj = nn.Linear(c.local_dim, c.local_dim * c.cross_slots, bias=False)
+        self.to_global = nn.Linear(c.local_dim * c.cross_slots, c.dim, bias=False)
+        self.to_local = nn.Linear(c.dim, c.local_dim * c.cross_slots, bias=False)
+        self.global_stack = GlobalStack(c, depth)
+        self.norm = RMSNorm(c.local_dim)
+        self.output = nn.Linear(c.local_dim, VOCAB, bias=False)
+        self.apply(initialize)
+
+    def train(self, mode=True):
+        super().train(mode)
+        self.patcher.eval()
+        return self
+
+    def byte_embeddings(self, x, doc):
+        b, t = x.shape
+        emb = self.embed(x)
+        # Polynomial hashes, causal and reset at document boundaries. Modular
+        # arithmetic avoids signed int64 overflow and is platform-independent.
+        for size, table in zip(self.c.hash_sizes, self.hashes):
+            if size > t:
+                emb = emb + table.weight[0] * 0
+                continue
+            h = torch.zeros_like(x)
+            valid = torch.arange(t, device=x.device)[None] >= size - 1
+            valid = valid.expand(b, t).clone()
+            for lag in range(size - 1, -1, -1):
+                shifted = F.pad(x[:, :t-lag], (lag, 0), value=0) if lag else x
+                h = (h * 257 + shifted) % self.c.hash_buckets
+                valid &= (shifted < 256)
+                if lag:
+                    prevdoc = F.pad(doc[:, :t-lag], (lag, 0), value=-1)
+                    valid &= prevdoc == doc
+            emb = emb + table(h) * valid[..., None]
+        return emb / (1 + len(self.hashes))
+
+    def forward(self, x, loops=None):
+        if not torch.isfinite(self.patcher.threshold):
+            raise RuntimeError("BLT requires a trained, calibrated entropy checkpoint; run the patcher stage first.")
+        b, t = x.shape
+        doc = documents(x)
+        with torch.no_grad():
+            e = self.patcher.entropy(x)
+            close = (e > self.patcher.threshold) | (x == BOS) | (x == EOS)
+            starts = torch.cat((torch.ones_like(close[:, :1]), close[:, :-1]), 1)
+            starts |= x == BOS
+            pid, first, valid = segments(starts)
+            n = first.shape[1]
+            pdoc = doc.gather(1, first)
+            pdoc = torch.where(valid, pdoc, -1)
+            groups = torch.arange(n, device=x.device)[None].expand(b, n)
+            pgroups = groups.repeat_interleave(self.c.cross_slots, 1)
+            pvalid = valid.repeat_interleave(self.c.cross_slots, 1)
+            # No peeking at x[t+1] to decide decoder conditioning.
+            context_id = pid + close.long() - 1
+            ok = context_id >= 0
+            context_doc = pdoc.gather(1, context_id.clamp_min(0))
+            ok &= context_doc == doc
+            context_id = torch.where(ok, context_id, -2)
+        h = self.byte_embeddings(x, doc)
+        p = None
+        for layer, cross in zip(self.enc, self.enc_cross):
+            h = ckpt(layer, h, doc, enabled=self.c.grad_checkpoint)
+            if p is None:
+                pooled = h.float().new_full((b, n, h.shape[-1]), -torch.inf)
+                pooled = pooled.scatter_reduce(1, pid[..., None].expand_as(h), h.float(),
+                                                reduce="amax", include_self=True)
+                pooled = torch.where(valid[..., None], pooled, torch.zeros_like(pooled))
+                p = self.pool_proj(pooled.to(h.dtype)).reshape(b, n * self.c.cross_slots, -1)
+            p = ckpt(cross, p, h, pgroups, pid, pvalid, enabled=self.c.grad_checkpoint)
+        g = self.global_stack(self.to_global(p.reshape(b, n, -1)), pdoc)
+        kv = self.to_local(g).reshape(b, n * self.c.cross_slots, -1)
+        for cross, layer in zip(self.dec_cross, self.dec):
+            h = ckpt(cross, h, kv, context_id, pgroups, ok, enabled=self.c.grad_checkpoint)
+            h = ckpt(layer, h, doc, enabled=self.c.grad_checkpoint)
+        return {"logits": self.output(self.norm(h)), "heads": [],
+                "patches": valid.sum().detach(), "positions": x.numel()}
+
+
+class Canon(nn.Module):
+    def __init__(self, d):
+        super().__init__()
+        # Identity initialization is stable without adding a second residual
+        # around Canon. The paper's operator replaces its input.
+        w = torch.zeros(4, d)
+        w[0] = 1
+        self.weight = nn.Parameter(w)
+
+    def forward(self, x, doc):
+        y = x * self.weight[0]
+        for lag in range(1, min(4, x.shape[1])):
+            h = F.pad(x[:, :-lag], (0, 0, lag, 0))
+            same = F.pad(doc[:, :-lag], (lag, 0), value=-1) == doc
+            y = y + h * self.weight[lag] * same[..., None]
+        return y
+
+
+class CanonBlock(nn.Module):
+    def __init__(self, c):
+        super().__init__()
+        # Main paper equations explicitly specify LayerNorm in local modules.
+        self.n1, self.n2 = nn.LayerNorm(c.local_dim), nn.LayerNorm(c.local_dim)
+        self.a = Attention(c.local_dim, c.local_heads, c.local_window,
+                           500000.0, query_block=c.query_block)
+        self.ff = SwiGLU(c.local_dim, c.local_ff)
+        self.c1, self.c2 = Canon(c.local_dim), Canon(c.local_dim)
+
+    def forward(self, x, doc):
+        x = self.c1(x + self.a(self.n1(x), doc), doc)
+        return self.c2(x + self.ff(self.n2(x)), doc)
+
+
+def coding_scores(h, eps2=1.0, mode="l2", doc=None):
+    # Detaching boundary decisions follows discrete Top-K selection. Selected
+    # encoder states still receive the normal language-model gradients.
+    with torch.no_grad(), torch.autocast(h.device.type, enabled=False):
+        x = h.detach().float()
+        b, t, d = x.shape
+        alpha = d / eps2
+        if mode == "l2":
+            return 0.5 * alpha * x.square().sum(-1)
+        if mode != "exact_smw":
+            raise ValueError(mode)
+        # Expensive diagnostic; unnormalized states are essential to Eq. 11.
+        eye = torch.eye(d, device=x.device, dtype=torch.float64).expand(b, d, d)
+        inv = eye.clone()
+        scores = []
+        for i in range(t):
+            if i and doc is not None:
+                inv = torch.where((doc[:, i] != doc[:, i-1])[:, None, None], eye, inv)
+            v = x[:, i].double().unsqueeze(-1)
+            av = inv @ v
+            q = (v.transpose(1, 2) @ av).flatten().clamp_min(0)
+            den = 1 + alpha * q
+            scores.append(0.5 * den.log())
+            inv = inv - (alpha / den)[:, None, None] * (av @ av.transpose(1, 2))
+            inv = (inv + inv.transpose(1, 2)) * 0.5
+        return torch.stack(scores, 1).float()
+
+
+def causal_promotions(scores, doc, patch_size, window):
+    """Irrevocable trailing-window rank gate. A disclosed ByteFlow adaptation.
+
+    Current position is selected if its score is in the top 1/patch_size of
+    the observed window. Ties choose earlier positions. No future rank or cap.
+    """
+    s = F.pad(scores, (window - 1, 0), value=-torch.inf).unfold(1, window, 1)
+    d = F.pad(doc, (window - 1, 0), value=-1).unfold(1, window, 1)
+    valid = d == doc[..., None]
+    rank = ((s >= scores[..., None]) & valid).sum(-1)
+    quota = (valid.sum(-1).float() / patch_size).ceil().clamp_min(1)
+    starts = rank <= quota
+    starts[:, 0] = True
+    starts[:, 1:] |= doc[:, 1:] != doc[:, :-1]
+    return starts
+
+
+def paper_promotions(scores, patch_size):
+    b, t = scores.shape
+    k = max(1, min(t, math.ceil(t / patch_size)))
+    starts = torch.zeros_like(scores, dtype=torch.bool)
+    starts[:, 0] = True
+    if k > 1:
+        # Stable sort provides a deterministic tie policy.
+        ix = torch.argsort(scores[:, 1:], dim=-1, descending=True, stable=True)[:, :k-1] + 1
+        starts.scatter_(1, ix, True)
+    return starts
+
+
+class ByteFlowLM(nn.Module):
+    def __init__(self, c, depth, paper=False):
+        super().__init__()
+        self.c, self.paper = c, paper
+        self.embed = nn.Embedding(VOCAB, c.local_dim)
+        self.enc = nn.ModuleList([CanonBlock(c) for _ in range(c.encoder_layers)])
+        self.dec = nn.ModuleList([CanonBlock(c) for _ in range(c.decoder_layers)])
+        self.to_global = nn.Linear(c.local_dim, c.dim, bias=False)
+        self.global_stack = GlobalStack(c, depth)
+        self.up = nn.ModuleList([nn.Linear(c.dim, c.local_dim, bias=False) for _ in range(c.byteflow_bins)])
+        self.norm = nn.LayerNorm(c.local_dim)
+        self.output = nn.Linear(c.local_dim, VOCAB, bias=False)
+        self.apply(initialize)
+
+    def forward(self, x, loops=None):
+        b, t = x.shape
+        doc, h = documents(x), self.embed(x)
+        for layer in self.enc:
+            h = ckpt(layer, h, doc, enabled=self.c.grad_checkpoint)
+        scores = coding_scores(h, self.c.byteflow_eps2, self.c.byteflow_score, doc)
+        if self.paper:
+            starts = paper_promotions(scores, self.c.patch_size)
+            # EOS/BOS isolation is a common data-protocol adaptation.
+            starts[:, 1:] |= doc[:, 1:] != doc[:, :-1]
+        else:
+            starts = causal_promotions(scores, doc, self.c.patch_size, self.c.router_window)
+        pid, first, valid = segments(starts)
+        pdoc = torch.where(valid, doc.gather(1, first), -1)
+        z = self.to_global(gather(h, first))
+        g = self.global_stack(z, pdoc)
+        lifted = gather(g, pid)
+        # Paper Eq. 15 is length-dependent. Causal mode fixes the horizon to
+        # the configured context so extending a prefix cannot change old bins.
+        horizon = t if self.paper else self.c.context
+        bins = (torch.arange(t, device=x.device) * self.c.byteflow_bins // horizon).clamp_max(self.c.byteflow_bins - 1)
+        # Compute by contiguous bin spans; never materialize [T, G, local_dim].
+        up = []
+        for i, proj in enumerate(self.up):
+            selected = torch.nonzero(bins == i, as_tuple=False).flatten()
+            if selected.numel():
+                up.append(proj(lifted[:, selected]))
+        s = h + torch.cat(up, 1)
+        # Unused bins at short prefixes should have zero gradients, not missing
+        # ones in DDP. This does not alter any activation.
+        s = s + sum(proj.weight.reshape(-1)[0] * 0 for proj in self.up)
+        for layer in self.dec:
+            s = ckpt(layer, s, doc, enabled=self.c.grad_checkpoint)
+        return {"logits": self.output(self.norm(s)), "heads": [],
+                "patches": valid.sum().detach(), "positions": x.numel()}
+
+
+def nparams(model, trainable=False):
+    return sum(p.numel() for p in model.parameters() if not trainable or p.requires_grad)
+
+
+def build_model(arch, c, seed=42):
+    if arch not in ARCHS + ("entropy",): raise ValueError("Unsupported architecture; recurrent ouro is removed")
+    c.validate()
+    if c.loops != 1: raise ValueError("This unified screen supports one-pass Ouro only")
+    builders = {
+        "evabyte": lambda n: EvaByteLM(c, n),
+        "blt": lambda n: BLTLM(c, n),
+        "byteflow_causal": lambda n: ByteFlowLM(c, n, False),
+        "byteflow_paper": lambda n: ByteFlowLM(c, n, True),
+    }
+    # Meta construction does not allocate model weights or perturb the seed
+    # subsequently used for initialization. Frozen patcher counts are INCLUDED.
+    with torch.device("meta"):
+        target = nparams(OuroLM(c))
+        depth = c.body_layers + 2
+        if arch in builders:
+            base, one = nparams(builders[arch](0)), nparams(builders[arch](1))
+            per = one - base
+            depth = min(range(1, 33), key=lambda n: abs(base + n * per - target))
+    seed_all(seed)
+    if arch in ("ouro", "ouro_1pass"):
+        model = OuroLM(c)
+    elif arch == "entropy":
+        model = EntropyLM(c)
+        depth = c.entropy_layers
+    else:
+        model = builders[arch](depth)
+    meta = {"arch": arch, "total_params": nparams(model), "trainable_params": nparams(model, True),
+            "target_params": target, "depth": depth, "frozen_params": nparams(model)-nparams(model, True),
+            "param_delta_pct": 100 * (nparams(model)-target)/target,
+            "causal_ranking_eligible": arch != "byteflow_paper",
+            "fidelity": "paper-style diagnostic" if arch == "byteflow_paper" else
+                "explicit causal adaptation" if arch == "byteflow_causal" else "scaled architecture"}
+    return model, meta
+
+
+def forward_model(model, ids, arch, loops=None):
+    return model(ids, loops=1 if arch == "ouro_1pass" else loops)
+
+
+def targets_and_mask(sequence, offset=1):
+    # The caller forwards sequence[:, :-1]. Auxiliary heads lose their tail.
+    n = sequence.shape[1] - offset
+    target = sequence[:, offset:]
+    doc = documents(sequence)
+    valid = (target != BOS) & (target != PAD) & (doc[:, :n] == doc[:, offset:])
+    return target, valid
+
+
+def head_counts(batches, nheads):
+    return torch.tensor([sum(int(targets_and_mask(x, j+1)[1].sum()) for x in batches)
+                         for j in range(nheads)], dtype=torch.float64)
+
+
+def loss_and_stats(out, sequence, denominators=None, world=1):
+    heads = out.get("heads") or [out["logits"]]
+    loss = out["logits"].float().sum() * 0
+    for j, logits in enumerate(heads):
+        target, valid = targets_and_mask(sequence, j+1)
+        n = target.shape[1]
+        ce = F.cross_entropy(logits[:, :n].float().reshape(-1, VOCAB), target.reshape(-1), reduction="none").view_as(target)
+        denom = valid.sum().clamp_min(1) if denominators is None else denominators[j].clamp_min(1)
+        # DDP averages gradients; world/whole-update denominator makes this a
+        # true global token mean even with varying numbers of document markers.
+        loss = loss + (ce * valid).sum() * world / denom / len(heads)
+        if j == 0:
+            byte = valid & (target < 256)
+            correct = logits[:, :n].argmax(-1) == target
+            stats = torch.stack(((ce * valid).sum(), valid.sum(), (ce * byte).sum(),
+                                 byte.sum(), (correct & byte).sum())).detach().double()
+    return loss, stats
+
+

unified/ablations/unified-v1-blt-none-s42/source/checkpoints.py

@@ -0,0 +1,190 @@
+"""Atomic full-state checkpoints, verified newest-first local/Hub recovery."""
+import os,json,time,hashlib,shutil,uuid
+from pathlib import Path
+import torch
+from huggingface_hub import HfApi,hf_hub_download,CommitOperationAdd,CommitOperationDelete
+from huggingface_hub.errors import RepositoryNotFoundError
+from dataset_registry import retry
+from bet_model import BETConfig,EXPECTED_ARCH_SHAPE_SHA256
+
+FORMAT='cortex-sparkbet9m-full-v1'
+DEFAULT_HUB_CHECKPOINT_PREFIX='checkpoints/sparkbet9m'
+
+
+def digest(path):
+    h=hashlib.sha256()
+    with open(path,'rb') as f:
+        for b in iter(lambda:f.read(4*1024*1024),b''):h.update(b)
+    return h.hexdigest()
+
+
+def atomic_json(path,value):
+    path=Path(path);path.parent.mkdir(parents=True,exist_ok=True);tmp=path.with_name(path.name+'.tmp')
+    with open(tmp,'w') as f:json.dump(value,f,indent=2);f.flush();os.fsync(f.fileno())
+    os.replace(tmp,path)
+
+
+def pipeline_hash(root):
+    h=hashlib.sha256()
+    for p in sorted(Path(root).glob('*.py')):h.update(p.name.encode());h.update(p.read_bytes())
+    return h.hexdigest()
+
+
+def save(run,state,keep=3):
+    root=Path(run)/'checkpoints';root.mkdir(parents=True,exist_ok=True)
+    name=f'checkpoint-{state["step"]:012d}';dest=root/name
+    if (dest/'COMPLETE').exists():return dest
+    tmp=root/('.incomplete-'+uuid.uuid4().hex);tmp.mkdir()
+    try:
+        with open(tmp/'training.pt','wb') as f:torch.save(state,f);f.flush();os.fsync(f.fileno())
+        meta=dict(format=FORMAT,step=state['step'],lineage=state['lineage'],architecture=EXPECTED_ARCH_SHAPE_SHA256,
+                  pipeline=state['pipeline'],sha256=digest(tmp/'training.pt'),bytes=(tmp/'training.pt').stat().st_size,saved_at=time.time())
+        atomic_json(tmp/'metadata.json',meta);(tmp/'COMPLETE').write_text('complete\n')
+        if dest.exists():shutil.rmtree(dest)
+        os.replace(tmp,dest);atomic_json(Path(run)/'latest.json',dict(path=str(dest),**meta))
+        complete=sorted(p for p in root.glob('checkpoint-*') if (p/'COMPLETE').exists())
+        for p in complete[:-keep]:shutil.rmtree(p)
+        return dest
+    except BaseException:
+        if tmp.exists():shutil.rmtree(tmp)
+        raise
+
+
+def verified_load(path):
+    path=Path(path);meta=json.loads((path/'metadata.json').read_text())
+    if meta['format']!=FORMAT or meta['architecture']!=EXPECTED_ARCH_SHAPE_SHA256:raise ValueError('Incompatible checkpoint metadata')
+    file=path/'training.pt'
+    if file.stat().st_size!=meta['bytes'] or digest(file)!=meta['sha256']:raise ValueError('Checkpoint checksum mismatch')
+    state=torch.load(file,map_location='cpu',weights_only=True)
+    if state['format']!=FORMAT or state['step']!=meta['step'] or state['lineage']!=meta['lineage']:raise ValueError('Checkpoint state/metadata mismatch')
+    return state
+
+
+def candidates(roots,repo=None,cache=None,checkpoint_prefix=DEFAULT_HUB_CHECKPOINT_PREFIX):
+    items=[]
+    for root in roots:
+        root=Path(root)
+        if not root.exists():continue
+        for p in root.rglob('metadata.json'):
+            if not (p.parent/'COMPLETE').exists() or not (p.parent/'training.pt').exists():continue
+            try:
+                m=json.loads(p.read_text())
+                if m.get('format')==FORMAT and m.get('architecture')==EXPECTED_ARCH_SHAPE_SHA256:
+                    items.append(dict(meta=m,path=p.parent,remote=False))
+            except (ValueError,OSError):continue
+    if repo:
+        api=HfApi()
+        try:
+            info=retry(lambda:api.model_info(repo));files=[f.rfilename for f in info.siblings];revision=info.sha
+        except RepositoryNotFoundError:files=[];revision=None
+        prefix=checkpoint_prefix.strip('/')+'/'
+        for f in files:
+            if f.startswith(prefix) and f.endswith('/metadata.json') and f.rsplit('/',1)[0]+'/COMPLETE' in files:
+                local=retry(lambda:hf_hub_download(repo,f,revision=revision,cache_dir=cache));m=json.loads(Path(local).read_text())
+                if m.get('format')==FORMAT and m.get('architecture')==EXPECTED_ARCH_SHAPE_SHA256:
+                    items.append(dict(meta=m,prefix=f.rsplit('/',1)[0],remote=True,revision=revision))
+    return sorted(items,key=lambda x:(x['meta']['step'],x['meta']['saved_at']),reverse=True)
+
+
+def resume(roots,repo,run,expected_pipeline,checkpoint_prefix=DEFAULT_HUB_CHECKPOINT_PREFIX,
+           allowed_previous_pipelines=(),migration_schedule='progressive_data_stages_v2',
+           aux_stage_base_updates=128):
+    items=candidates(roots,repo,str(Path(run)/'hub_cache'),checkpoint_prefix)
+    lineages={i['meta']['lineage'] for i in items}
+    if len(lineages)>1:
+        raise RuntimeError('Multiple SparkBET run lineages found. Select the intended run instead of mixing checkpoint histories.')
+    for item in items:
+        try:
+            if item['remote']:
+                dest=Path(run)/'hub_resume'/Path(item['prefix']).name;dest.mkdir(parents=True,exist_ok=True)
+                for n in ['metadata.json','training.pt','COMPLETE']:
+                    retry(lambda n=n:hf_hub_download(repo,item['prefix']+'/'+n,revision=item['revision'],local_dir=Path(run)/'hub_download'))
+                    source=Path(run)/'hub_download'/item['prefix']/n;os.replace(source,dest/n)
+                path=dest
+            else:path=item['path']
+            state=verified_load(path)
+        except (OSError,ValueError,EOFError,RuntimeError) as exc:
+            print(f'Resume candidate step {item["meta"]["step"]} unusable: {type(exc).__name__}; trying older complete checkpoint.',flush=True);continue
+        if state['pipeline']!=expected_pipeline:
+            # Only an explicitly listed, exact previous pipeline fingerprint can
+            # cross the auxiliary-scheduling objective boundary. Never accept
+            # unrelated code/data changes or silently restart data cursors.
+            previous=state['pipeline']
+            if previous not in allowed_previous_pipelines:
+                raise RuntimeError('Training/data code differs from this SparkBET checkpoint. Exact data cursors are preserved by refusing an implicit migration.')
+            # Start the new long-form curriculum at L1 right here, not at
+            # an arbitrary stage computed from the historical global step.
+            config=state.setdefault('config',{})
+            config['aux_loop_schedule']=migration_schedule
+            config['aux_stage_base_updates']=int(aux_stage_base_updates)
+            config['aux_curriculum_origin_step']=int(state['step'])
+            state.setdefault('schedule_migrations',[]).append(dict(
+                step=state['step'],previous_pipeline=previous,
+                new_pipeline=expected_pipeline,schedule=migration_schedule,
+                curriculum_origin_step=int(state['step']),
+                aux_stage_base_updates=int(aux_stage_base_updates)))
+            state['pipeline']=expected_pipeline
+            print(f'Verified data-stage migration at update {state["step"]:,}; preserving model, optimizer, scaler and data cursors. New curriculum begins at L1.',flush=True)
+        print(f'Resumed complete state at update {state["step"]:,} from {"Hub" if item["remote"] else "local"}.',flush=True)
+        return state
+    if items:raise RuntimeError('Checkpoints exist but none passed verification; refusing a silent fresh run')
+    return None
+
+
+def legacy_import(roots,explicit=None):
+    # The previous BET-74M tensors/optimizer are shape-incompatible with SparkBET-9M.
+    # Do not silently splice two architectures. New SparkBET sessions can still
+    # recover complete SparkBET state from local/Hub checkpoints above.
+    if explicit:
+        raise ValueError('The legacy BET-74M checkpoint cannot be migrated into the 9.35M SparkBET architecture. Start fresh or use a SparkBET full-state checkpoint.')
+    return None
+
+
+class Publisher:
+    def __init__(self,repo,run,keep=2,checkpoint_prefix=DEFAULT_HUB_CHECKPOINT_PREFIX):
+        self.repo,self.run,self.keep=repo,Path(run),keep
+        self.checkpoint_prefix=checkpoint_prefix.strip('/')
+        self.api=HfApi()
+    def authenticate(self):
+        retry(lambda:self.api.whoami());retry(lambda:self.api.create_repo(self.repo,repo_type='model',exist_ok=True))
+    def logs(self):
+        folder=self.run/'publish_logs'
+        if folder.exists():
+            retry(lambda:self.api.upload_folder(repo_id=self.repo,folder_path=folder,path_in_repo='runs',commit_message='Cortex training metrics'))
+            events=sorted(folder.rglob('*tfevents*'),key=lambda p:p.stat().st_mtime)
+            for p in events[:-2]:p.unlink()
+    def checkpoint(self,path):
+        path=Path(path);prefix=self.checkpoint_prefix+'/'+path.name;files=retry(lambda:self.api.list_repo_files(self.repo))
+        base=self.checkpoint_prefix+'/'
+        old=sorted({f[len(base):].split('/')[0] for f in files if f.startswith(base+'checkpoint-') and f.endswith('/COMPLETE')})
+        keep=set(sorted(set(old+[path.name]))[-self.keep:])
+        operations=[CommitOperationAdd(path_in_repo=prefix+'/'+p.name,path_or_fileobj=str(p)) for p in path.iterdir() if p.is_file()]
+        for f in files:
+            if f.startswith(base+'checkpoint-'):
+                name=f[len(base):].split('/')[0]
+                if name not in keep:operations.append(CommitOperationDelete(path_in_repo=f))
+        retry(lambda:self.api.create_commit(repo_id=self.repo,operations=operations,commit_message=f'Full resumable SparkBET-9M state at {path.name}'))
+    def export(self,model,step,manifest,source_root):
+        from safetensors.torch import save_file
+        from dataclasses import asdict
+        dest=self.run/'export';dest.mkdir(exist_ok=True)
+        # HF wrapper owns the training model as `core`, so exported tensor names are
+        # prefixed accordingly and load strictly through AutoModelForCausalLM.
+        weights={f'core.{k}':v.detach().cpu().contiguous() for k,v in model.state_dict().items()}
+        save_file(weights,str(dest/'model.safetensors'),metadata={'format':'pt'})
+        c=asdict(model.c)
+        config=dict(
+            model_type='bet',architectures=['BETForCausalLM'],step=step,
+            vocab_size=c['vocab_size'],hidden_size=c['hidden_size'],intermediate_size=c['intermediate_size'],
+            prelude_layers=c['prelude_layers'],body_blocks=c['body_blocks'],coda_layers=c['coda_layers'],
+            num_attention_heads=c['num_heads'],num_key_value_heads=c['num_kv_heads'],head_dim=c['head_dim'],
+            lora_rank=c['lora_rank'],hyper_lanes=c['hyper_lanes'],max_position_embeddings=c['max_seq_len'],max_loops=c['max_loops'],
+            rope_theta=c['rope_theta'],rms_norm_eps=c['rms_eps'],ddl_beta_init=c['ddl_beta_init'],ddl_k_eps=c['ddl_k_eps'],
+            ddl_v_sigmoid_scale=c['ddl_v_sigmoid_scale'],refinement_cycles=8,use_cache=False,tie_word_embeddings=True,
+            pad_token_id=256,bos_token_id=257,eos_token_id=258,precision='fp16 autocast / fp32 master',
+            auto_map={'AutoConfig':'configuration_bet.BETConfig','AutoModelForCausalLM':'modeling_bet.BETForCausalLM','AutoTokenizer':['tokenization_bet.BETByteTokenizer',None]},
+        )
+        atomic_json(dest/'config.json',config);atomic_json(dest/'dataset_manifest.json',manifest)
+        for name in ['bet_model.py','inference.py','records.py','README.md','configuration_bet.py','modeling_bet.py','tokenization_bet.py','tokenizer_config.json']:
+            shutil.copy2(Path(source_root)/name,dest/name)
+        retry(lambda:self.api.upload_folder(repo_id=self.repo,folder_path=dest,commit_message=f'SparkBET-9M inference export at update {step}'))

unified/ablations/unified-v1-blt-none-s42/source/configuration_bet.py

@@ -0,0 +1,64 @@
+from transformers import PretrainedConfig
+
+
+class BETConfig(PretrainedConfig):
+    model_type = "bet"
+
+    def __init__(
+        self,
+        vocab_size=259,
+        hidden_size=324,
+        intermediate_size=864,
+        prelude_layers=1,
+        body_blocks=6,
+        coda_layers=1,
+        num_attention_heads=6,
+        num_key_value_heads=2,
+        head_dim=54,
+        lora_rank=16,
+        hyper_lanes=2,
+        max_position_embeddings=1024,
+        max_loops=8,
+        rope_theta=10_000.0,
+        rms_norm_eps=1e-6,
+        ddl_beta_init=1.0,
+        ddl_k_eps=1e-2,
+        ddl_v_sigmoid_scale=4.0,
+        refinement_cycles=8,
+        use_cache=False,
+        tie_word_embeddings=True,
+        pad_token_id=256,
+        bos_token_id=257,
+        eos_token_id=258,
+        **kwargs,
+    ):
+        super().__init__(
+            pad_token_id=pad_token_id,
+            bos_token_id=bos_token_id,
+            eos_token_id=eos_token_id,
+            tie_word_embeddings=tie_word_embeddings,
+            is_encoder_decoder=False,
+            **kwargs,
+        )
+        self.vocab_size=int(vocab_size)
+        self.hidden_size=int(hidden_size)
+        self.intermediate_size=int(intermediate_size)
+        self.prelude_layers=int(prelude_layers)
+        self.body_blocks=int(body_blocks)
+        self.coda_layers=int(coda_layers)
+        # Common HF tooling expects num_hidden_layers even though only the body loops.
+        self.num_hidden_layers=int(prelude_layers+body_blocks+coda_layers)
+        self.num_attention_heads=int(num_attention_heads)
+        self.num_key_value_heads=int(num_key_value_heads)
+        self.head_dim=int(head_dim)
+        self.lora_rank=int(lora_rank)
+        self.hyper_lanes=int(hyper_lanes)
+        self.max_position_embeddings=int(max_position_embeddings)
+        self.max_loops=int(max_loops)
+        self.rope_theta=float(rope_theta)
+        self.rms_norm_eps=float(rms_norm_eps)
+        self.ddl_beta_init=float(ddl_beta_init)
+        self.ddl_k_eps=float(ddl_k_eps)
+        self.ddl_v_sigmoid_scale=float(ddl_v_sigmoid_scale)
+        self.refinement_cycles=int(refinement_cycles)
+        self.use_cache=bool(use_cache)

unified/ablations/unified-v1-blt-none-s42/source/cortex_data.py

@@ -0,0 +1,132 @@
+"""Deterministic answer supervision and executable compositional train/test splits."""
+import hashlib
+import random
+import numpy as np
+
+REVISION = 'cortex-answer-execution-v1'
+HELD_PAIRS = {(1, 2), (2, 1)}  # rotate -> swap and swap -> rotate, by semantic operation
+
+
+def program_split(program):
+    return 'composition' if any(pair in HELD_PAIRS for pair in zip(program, program[1:])) else 'train'
+
+
+def sample_program(rng, split, depth):
+    for _ in range(10000):
+        program = tuple(rng.randrange(4) for _ in range(depth))
+        if program_split(program) == ('composition' if split == 'composition' else 'train'):
+            return program
+    raise RuntimeError('could not sample program split')
+
+
+def execute(values, program, permutation):
+    states = []
+    values = list(values)
+    for op in program:
+        if op == 0:
+            values = values[::-1]
+        elif op == 1:
+            values = values[1:] + values[:1]
+        elif op == 2:
+            values = values[:]
+            for i in range(0, len(values) - 1, 2):
+                values[i], values[i + 1] = values[i + 1], values[i]
+        elif op == 3:
+            values = [permutation[x] for x in values]
+        else:
+            raise ValueError(op)
+        states.append(values[:])
+    return states
+
+
+def composition_episode(c, seed, context, split='train', difficulty=1, trace=None):
+    """Reusable sequence operations with held-out *semantic* ordered pairs.
+
+    Byte roles, alphabets and substitution maps are independently randomized.
+    Training/query demonstrations NEVER include held-out pairs or depths > 4.
+    Trace mode emits intermediate states autoregressively; direct mode emits only
+    the result. The mode and separators are grounded by demonstrations in both.
+    """
+    rng = random.Random(int(seed))
+    surface = c.Surface(rng)
+    atoms = surface.atoms(6)
+    ops = surface.atoms(4)
+    begin, sep, end, chain, mode, state_sep = surface.take(6)
+    perm = list(range(6)); rng.shuffle(perm)
+    trace = bool(rng.randrange(2)) if trace is None else bool(trace)
+    depth = rng.randint(5, 8) if split == 'depth' else rng.randint(2 if split == 'composition' else 1, 2 if difficulty == 0 else 4)
+    length = rng.randint(13, 20) if split == 'length' else rng.randint(4, 8 if difficulty == 0 else 12)
+    program = sample_program(rng, split, depth)
+    b = c.Builder(25, 'compositional_execution')
+    def encode(xs): return [atoms[x] for x in xs]
+    def query(xs, prog): return [begin] + encode(xs) + [chain] + [ops[o] for o in prog] + [sep]
+    def answer(states, traced):
+        selected = states if traced else states[-1:]
+        result = []
+        for i, state in enumerate(selected):
+            if i: result.append(state_sep)
+            result.extend(encode(state))
+        return result
+    # Complete substitution table; unlike a few random demos, covers every query symbol.
+    for x in range(6):
+        b.emit(query([x], (3,)) + encode([perm[x]]) + [end])
+    # Distinct symbols and odd/even widths disambiguate positional primitives.
+    for op in range(3):
+        for width in (5, 6):
+            xs = rng.sample(range(6), width)
+            b.emit(query(xs, (op,)) + encode(execute(xs, (op,), perm)[-1]) + [end])
+    # The same local mode marker/grammar is used for demonstrations and query.
+    for demo_depth in (2, 3):
+        xs = [rng.randrange(6) for _ in range(5)]
+        prog = sample_program(rng, 'train', demo_depth)
+        b.emit([mode] + query(xs, prog) + answer(execute(xs, prog, perm), trace) + [end])
+    xs = [rng.randrange(6) for _ in range(length)]
+    b.emit([mode] + query(xs, program)); b.mark_prompt()
+    b.emit(answer(execute(xs, program, perm), trace), c.W_TARGET)
+    ep = b.finish(context)
+    ep.program = program; ep.split = split; ep.trace = trace
+    ep.latent_input = xs; ep.permutation = perm; ep.atoms = atoms; ep.state_sep = state_sep
+    ep.weights = [0] * ep.prompt_len + [c.W_TARGET] * (len(ep.tokens) - ep.prompt_len)
+    return ep
+
+
+def answer_episode(c, entry, difficulty, context, seed, candidates=1):
+    """One inference protocol for both training and evaluation, including EOS.
+
+    Remove ignored suffix terminators instead of feeding an unsupervised byte
+    during generation. Keep grid shape/format tokens and supervise the complete
+    continuation. Random demonstration outputs are context, never loss targets.
+    """
+    public = entry['name'] == 'gen_arc_static_public_train'
+    ep, _ = c._v12_generate_episode(entry, difficulty, context, seed,
+                                    'weighted' if public else 'answer', candidates)
+    tail=ep.tokens[ep.prompt_len:]
+    weights=ep.weights[ep.prompt_len:]
+    last=max((i for i,(t,w) in enumerate(zip(tail,weights)) if w>0 and t!=c.EOS),default=-1)
+    suffix=tail[:last+1]
+    if not suffix:
+        raise ValueError(f"empty answer: {entry['name']}")
+    ep.tokens = ep.tokens[:ep.prompt_len] + suffix + [c.EOS]
+    ep.weights = [0] * ep.prompt_len + [c.W_TARGET] * (len(suffix) + 1)
+    ep.validate(context)
+    return ep
+
+
+def padded_batch(c, episodes):
+    # Each row is an independent episode: no cross-example attention, no dropped
+    # overflow bins, no long-episode preference. Bucket only after sampling.
+    width = ((max(len(ep.tokens) - 1 for ep in episodes) + 31) // 32) * 32 + 1
+    tok = np.full((len(episodes), width), c.PAD, dtype=np.int64)
+    wt = np.zeros(tok.shape, dtype=np.float32)
+    seg = np.full(tok.shape, -1, dtype=np.int64)
+    for i, ep in enumerate(episodes):
+        n = len(ep.tokens)
+        tok[i, :n] = ep.tokens; wt[i, :n] = ep.weights; seg[i, :n] = i
+    return tok, wt, seg, len(episodes), float(np.mean(tok != c.PAD))
+
+
+def source_digest(*paths):
+    from pathlib import Path
+    h = hashlib.sha256()
+    for path in paths: h.update(Path(path).read_bytes())
+    return h.hexdigest()

unified/ablations/unified-v1-blt-none-s42/source/cortex_oracles.py

@@ -0,0 +1,250 @@
+"""Independent small-domain reference algorithms used only by tests/audits."""
+from itertools import permutations, product
+from functools import reduce
+
+def paths(edges,start,goal=None):
+    out=[]
+    def visit(path):
+        if goal is None or path[-1]==goal:out.append(path)
+        for a,b in edges:
+            if a==path[-1] and b not in path:visit(path+[b])
+    visit([start]);return out
+
+def program_reference(xs,program,mapping):
+    states=[];xs=list(xs)
+    for op in program:
+        n=len(xs)
+        if op=='reverse':indices=range(n-1,-1,-1)
+        elif op=='rotate':indices=[(i+1)%n for i in range(n)]
+        elif op=='swap':indices=[i+1 if i%2==0 and i+1<n else i-1 if i%2 else i for i in range(n)]
+        elif op=='map':xs=[dict(enumerate(mapping))[v] for v in xs];states.append(xs[:]);continue
+        elif op=='repeat':indices=[i for i in range(n) for _ in range(2)]
+        elif op=='evens':indices=range(0,n,2)
+        elif op=='dedup':indices=[i for i in range(n) if xs[i] not in xs[:i]]
+        else:raise KeyError(op)
+        xs=[xs[i] for i in indices];states.append(xs[:])
+    return states
+
+def oracle(name,x):
+    if name=='selection_sort':
+        a=x[:];states=[]
+        for i in range(len(a)-1):
+            value=sorted(a[i:])[0];j=a.index(value,i);a[j],a[i]=a[i],a[j];states.append(a[:])
+        return states
+    if name=='insertion_sort':return [sorted(x[:i+1])+x[i+1:] for i in range(1,len(x))]
+    if name=='minmax_scan':return [[sorted(x[:i])[0],sorted(x[:i])[-1]] for i in range(1,len(x)+1)]
+    if name=='linear_search':
+        a,t=x;return [a.index(t) if t in a else -1]
+    if name=='binary_search':
+        a,t=x
+        def visit(lo,hi):
+            if lo>=hi:return []
+            m=(lo+hi-1)//2
+            return [m]+([] if a[m]==t else visit(m+1,hi) if a[m]<t else visit(lo,m))
+        return [visit(0,len(a)),a.index(t) if t in a else -1]
+    if name=='string_match':
+        a,b=x;return [i for i in range(len(a)) if tuple(a[i:i+len(b)])==tuple(b)]
+    if name in ('bfs_order','dfs_order'):
+        edges,start=x
+        if name=='bfs_order':
+            # Minimal distance and lexicographically smallest discovery path.
+            pp=paths(edges,start);best={}
+            for p in pp:
+                k=(len(p),p)
+                if p[-1] not in best or k<best[p[-1]]:best[p[-1]]=k
+            return sorted(best,key=best.__getitem__)
+        out=[]
+        def visit(v):
+            if v in out:return
+            out.append(v)
+            for z in sorted(b for a,b in edges if a==v):visit(z)
+        visit(start);return out
+    if name=='topo_sort':
+        nodes,edges=x
+        return list(min(p for p in permutations(nodes) if all(p.index(a)<p.index(b) for a,b in edges)))
+    if name=='shortest_path':
+        pp=paths(*x);return min(pp,key=lambda p:(len(p),p)) if pp else []
+    if name=='union_find':
+        edges,a,b=x;components=[{v} for edge in edges for v in edge]+[{a},{b}]
+        for u,v in edges:
+            joined=set().union(*(s for s in components if u in s or v in s));components=[s for s in components if not(s&joined)]+[joined]
+        return any({a,b}<=s for s in components)
+    if name=='interval_overlap':
+        a,b,c,d=x;return bool(set(range(a,b+1))&set(range(c,d+1)))
+    if name in ('bracket_balance','bracket_depth'):
+        balances=[sum(1 if t=='open' else -1 for t in x[:i]) for i in range(1,len(x)+1)]
+        return (all(v>=0 for v in balances) and (balances[-1] if balances else 0)==0) if name=='bracket_balance' else max([0]+balances)
+    if name in ('queue','deque'):
+        q=[];removed=[]
+        for cmd,*args in x:
+            if cmd=='left':q=[args[0]]+q
+            elif cmd=='right':q=q+[args[0]]
+            elif not q:removed.append('empty')
+            elif cmd=='pop_left':removed.append(q[0]);q=q[1:]
+            elif cmd=='pop_right':removed.append(q[-1]);q=q[:-1]
+            else:raise KeyError(cmd)
+        return [removed,q]
+    if name in ('kv_lookup','role_filler','assoc_distractors','nested_lookup','mutable_dictionary','delayed_recall'):
+        records,(key,depth)=x
+        for _ in range(depth):key=next(v for k,v in reversed(records) if k==key)
+        return key
+    if name in ('register_swap','register_update','multi_register','program_trace'):
+        state,cmds=x;versions=[tuple(state)]
+        for op,a,b in cmds:
+            old=versions[-1]
+            versions.append(tuple(b if op=='set' and i==a else old[b] if op in ('swap','copy') and i==a else old[a] if op=='swap' and i==b else old[i] for i in range(len(old))))
+        return [list(v) for v in versions[1:]] if name=='program_trace' else list(versions[-1])
+    if name=='conditional_branch':return x[1:][0 if x[0] else 1]
+    if name=='loop_counter':return sorted(range(x+1),reverse=True)
+    if name in ('stack_expr','prefix_expr','postfix_expr'):
+        # Convert either notation to a syntax tree, then evaluate the tree.
+        tokens=[t for t in x if t!='push'];stack=[]
+        for t in (reversed(tokens) if name=='prefix_expr' else tokens):
+            if type(t) is int:stack.append(t)
+            else:
+                a,b=stack.pop(),stack.pop();stack.append((t,a,b))
+        def evaluate(t):
+            if type(t) is int:return t
+            op,a,b=t;a,b=evaluate(a),evaluate(b)
+            return sum([a,b]) if op=='add' else sum(a for _ in range(b))
+        return evaluate(stack[0])
+    if name=='relative_coords':
+        n,pos,moves=x;out=list(pos)
+        for move in moves:out=[sorted([0,v+d,n-1])[1] for v,d in zip(out,move)]
+        return out
+    if name=='toroidal':
+        n,p,moves=x;return (p+sum(moves))%n
+    if name=='collision':
+        a,va,b,vb,t=x;out=[]
+        # Equal masses in 1D = freely crossing ghost particles with sorted positions.
+        for k in range(1,t+1):
+            ghosts=sorted([(a+k*va,va),(b+k*vb,vb)],key=lambda z:(z[0],z[1]))
+            out.append([ghosts[0][0],ghosts[1][0],ghosts[0][1],ghosts[1][1]])
+        return out
+    if name=='falling':return list(map(list,zip(*(sorted(col) for col in zip(*x)))))
+    if name=='obstacle_motion':
+        n,p,blocked,moves=x;history=[p]
+        for m in moves:history.append(history[-1]+m if history[-1]+m in set(range(n))-set(blocked) else history[-1])
+        return history
+    if name=='line_of_sight':
+        a,b,blocked=x;return len(set(range(min(a,b)+1,max(a,b)))&set(blocked))==0
+    if name=='containment':
+        (a,b),(c,d)=x;return set(range(c,d+1))<=set(range(a,b+1))
+    if name in ('partial_map','dynamic_plan'):
+        if name=='partial_map':
+            observations,start,goal=x;edges=set()
+            for view in observations:edges.update(map(tuple,view))
+        else:
+            original,changes,start,goal=x
+            # For each edge, its last update determines membership.
+            candidates=set(map(tuple,original))|{(a,b) for _,a,b in changes}
+            edges={edge for edge in candidates if next((op=='add' for op,a,b in reversed(changes) if (a,b)==edge),edge in set(map(tuple,original)))}
+        choices=paths(sorted(edges),start,goal)
+        return min(choices,key=lambda p:(len(p),p)) if choices else []
+    if name in ('path_keys','door_key','sparse_nav','irreversible_action'):
+        edges,start,goal,keys,locks=x;front=[(start,start in keys,[])];seen=set()
+        while front:
+            node,key,p=front.pop(0)
+            if node==goal:return p
+            if (node,key) in seen:continue
+            seen.add((node,key))
+            for i,(a,b) in enumerate(edges):
+                if a==node and (key or i not in locks):front.append((b,key or b in keys,p+[i]))
+        return []
+    if name=='causal_abduction':
+        a,b,y=x;return next(u for u in (0,1) if ((u+a+b)%2)==y)
+    if name=='anomaly_cause':
+        u,a,b,vx,vy=x;return [key for key,actual,expected in [('x',vx,(u+a)%2),('y',vy,(vx+b)%2)] if actual!=expected]
+    if name in ('direct_cause','confounding','causal_intervention'):
+        u,a,b,(target,value)=x
+        vx=(u+a)%2
+        if name=='confounding':return [vx,(u+b)%2]
+        if target=='x':vx=value
+        return [vx,value if target=='y' else (vx+b)%2]
+    if name in ('competing_hypotheses','noisy_evidence','qual_bayes'):
+        priors,table,obs=x;score=[p*reduce(lambda a,b:a*b,(row[e] for e in obs),1) for p,row in zip(priors,table)]
+        return [i for i,s in enumerate(score) if all(s>=other for other in score)]
+    if name=='active_disambiguation':
+        columns=list(zip(*x));counts=[sum(a!=b for i,a in enumerate(c) for b in c[i+1:]) for c in columns]
+        return sorted(range(len(counts)),key=lambda i:(-counts[i],i))[0]
+    if name in ('rule_change','rule_switch'):
+        maps,events,start=x;out=[]
+        for k in range(1,len(events)+1):
+            mode=0;v=start
+            for op,arg in events[:k]:
+                if op=='switch':mode=arg
+                else:v=dict(enumerate(maps[mode]))[v]
+            out.append(v)
+        return out
+    if name=='goal_switch':
+        edges,start,goals=x;return [oracle('shortest_path',[edges,a,b]) for a,b in zip([start]+goals,goals)]
+    if name=='unknown_goal':return sorted(set(x[0])&set(x[1])-set(x[2]))
+    if name=='tool_affordance':return [i for i,tool in enumerate(x[0]) if set(x[1])<=set(tool)]
+    if name=='select_cell':return [list(divmod(i,len(x[0][0]))) for i,v in enumerate(sum(x[0],[])) if v==x[1]]
+    if name=='hidden_rule':
+        rules,demos,q=x;answers={(q+r)%7 for r in rules if all((b-a)%7==r for a,b in demos)}
+        assert len(answers)==1;return next(iter(answers))
+    if name=='explore_infer_exploit':
+        tables,obs,state,goal=x;accepted=[t for t in tables if [t[s][a] for s,a,_ in obs]==[z for _,_,z in obs]]
+        choices=[tuple(i for i,v in enumerate(t[state]) if v==goal) for t in accepted]
+        assert choices and len(set(choices))==1;return list(choices[0])
+    if name in ('sequence_composition','instruction_comp','modifier_comp','compositional_holdout','program_states'):
+        states=program_reference(*x);return states if name=='program_states' else states[-1]
+    if name in ('novel_primitive','rule_remap'):return dict(enumerate(x[0]))[x[1]]
+    if name=='representation_remap':return [dict(enumerate(x[0]))[v] for v in x[1]]
+    if name=='cross_domain':
+        phi,p,values=x;return [phi[p[phi.index(v)]] for v in values]
+    if name=='context_operator':return program_reference(x[0],['reverse' if len(x[1])%2 else 'rotate'],list(range(5)))[0]
+    if name=='meta_rule':return program_reference(x[0],[x[2][op] for op in x[1]],x[3])[-1]
+    if name=='mini_dsl':
+        pending=[(x,False)];values=[]
+        while pending:
+            node,visited=pending.pop();op,*args=node
+            if op=='literal':values.append(args[0])
+            elif not visited:
+                pending.append((node,True))
+                kids=[args[1]] if op=='repeat' else args
+                pending.extend((z,False) for z in reversed(kids))
+            elif op=='reverse':values.append(list(reversed(values.pop())))
+            elif op=='repeat':values.append(sum([values.pop()]*args[0],[]))
+            else:b,a=values.pop(),values.pop();values.append(a+b)
+        return values[0]
+    if name=='recursive_dsl':
+        rules,root,depth=x;word=[root]
+        for _ in range(depth):word=[v for t in word for v in rules.get(t,[t])]
+        return word
+    if name=='grammar_expansion':
+        rules,word,depth=x
+        def expand(t,n):return [t] if n==0 else sum([expand(z,n-1) for z in rules.get(t,[t])],[])
+        return sum([expand(t,depth) for t in word],[])
+    if name=='stable_partition':return sorted(x[0],key=lambda v:v not in x[1])
+    if name=='run_length_codec':
+        from itertools import groupby
+        return [[v,len(list(group))] for v,group in groupby(x)]
+    if name=='scope_shadowing':
+        merged={}
+        for frame in x[0]:merged.update(frame)
+        return merged.get(x[1],'missing')
+    if name=='relational_join':return [[a[0],a[1],b[1]] for a,b in product(*x) if a[0]==b[0]]
+    if name=='edit_script':
+        seq,edits=x;seq=seq[:]
+        for op,i,v in edits:
+            if op=='insert':seq=seq[:i]+[v]+seq[i:]
+            elif op=='delete':seq=seq[:i]+seq[i+1:]
+            else:seq=seq[:i]+[v]+seq[i+1:]
+        return seq
+    if name=='conditional_rewrite':
+        xs,a,b,guard=x;return [b if (prev,value)==(guard,a) else value for prev,value in zip([None]+xs,xs)]
+    if name=='span_reordering':
+        xs,a,b=x;return [xs[a+b-i-1] if a<=i<b else xs[i] for i in range(len(xs))]
+    if name=='nested_substitution':
+        tree,mapping=x
+        import copy
+        out=copy.deepcopy(tree);pending=[out]
+        while pending:
+            node=pending.pop()
+            for i,v in enumerate(node):
+                if isinstance(v,list):pending.append(v)
+                else:node[i]=mapping.get(v,v)
+        return out
+    raise KeyError(f'No independent oracle for {name}')

unified/ablations/unified-v1-blt-none-s42/source/cortex_source.py

@@ -0,0 +1,83 @@
+"""Paired symbolic/nonce tasks with bounded rejection and preserved answer grammar."""
+import random,hashlib,json,string
+import cortex_curriculum_v16 as kernel
+from cortex_data import answer_episode,composition_episode
+from records import BOS,EOS,ids,validate,Oversize
+CATALOG=kernel._v11_family_catalog(max(kernel.STAGES),False)
+FAMILIES=[e['name'] for e in CATALOG]+['compositional_execution']
+
+def seed_for(namespace,index):return int.from_bytes(hashlib.blake2b(f'{namespace}:{index}'.encode(),digest_size=8).digest(),'big')
+
+class WordSurface:
+    def __init__(self,seed):
+        rng=random.Random(seed);self.codes=rng.sample([a+b for a in string.ascii_lowercase for b in string.ascii_lowercase],256)
+        self.inverse={v:i for i,v in enumerate(self.codes)}
+    def encode(self,raw):return ' '.join(self.codes[b] for b in raw)
+    def decode(self,text):return [self.inverse[s] for s in text.split()]
+
+def render(ep,family,seed,wordlike,limit):
+    p=ep.tokens[1:ep.prompt_len];a=ep.tokens[ep.prompt_len:-1]
+    if not a or any(not 0<=t<256 for t in p+a):raise ValueError('Invalid kernel payload')
+    header='Cortex: infer the demonstrated rule and complete the query.\n'
+    if hasattr(ep,'audit'):
+        au=ep.audit
+        # Previously string operations (e.g. left/right/pop) had ungrounded random
+        # labels and sometimes appeared for the first time in the query. Ground
+        # their interpretation explicitly without printing any query solution.
+        bindings={f'{v:02x}':key[1] for key,v in au['symbols'].items() if key[0]!='int'}
+        start,close,sep,end,unit,neg=au['markers']
+        header+=f"Mechanism: {au['mechanism']}. Indices are zero-based; graph ties use increasing numeric order.\n"
+        header+='Byte bindings (hex): '+json.dumps(bindings,ensure_ascii=True,separators=(',',':'))+'\n'
+        header+=f'Containers {start:02x}/{close:02x}; input/output {sep:02x}; demo end {end:02x}. Numeric values are grounded by unary examples.\n'
+        # Every referenced binding byte is present in the binary context too.
+        p=list(bindings_byte for bindings_byte in au['symbols'].values() if bindings_byte not in p)+p
+    if wordlike:
+        surface=WordSurface(seed)
+        # Map only legend-referenced roles; the rest are learned by matching demos.
+        if hasattr(ep,'audit'):
+            used=ep.audit['symbols'].values();header+='Hex-to-nonce: '+' '.join(f'{b:02x}={surface.codes[b]}' for b in used)+'\n'
+            header+='Container codes: '+' '.join(f'{b:02x}={surface.codes[b]}' for b in ep.audit['markers'])+'\n'
+        prefix=ids(header+'Nonce examples:\n'+surface.encode(p)+'\nAnswer:\n')
+        answer=ids(surface.encode(a))
+    else:
+        prefix=ids(header+'Byte examples:\n')+p;answer=a
+    r=dict(ids=[BOS]+prefix+answer+[EOS],weights=[0]*(len(prefix)+1)+[1]*(len(answer)+1),
+           source='cortex',prompt_len=len(prefix)+1,meta={'family':family,'surface':'nonce' if wordlike else 'symbolic'})
+    if len(r['ids'])>limit+1:raise Oversize(f'{family}: expanded {len(r["ids"])}')
+    return validate(r,limit)
+
+def episode_pair(index,namespace='cortex74/train/v2',family=None,split='train',limit=1024):
+    pair=index//2;cycle,offset=divmod(pair,len(FAMILIES));order=list(range(len(FAMILIES)))
+    random.Random(seed_for(namespace+'/order',cycle)).shuffle(order)
+    selected=order[offset] if family is None else FAMILIES.index(family)
+    name=FAMILIES[selected]
+    for attempt in range(96):
+        seed=seed_for(namespace,(pair,attempt));difficulty=seed%3
+        try:
+            if name=='compositional_execution':ep=composition_episode(kernel,seed,2048,split,difficulty)
+            else:ep=answer_episode(kernel,CATALOG[selected],difficulty,2048,seed)
+            # Preserve the alternating nonce surface whenever it fits. At the 1K
+            # model context, a few nonce expansions are intrinsically >1K even when
+            # the underlying symbolic task fits; those records fall back to the
+            # complete symbolic view rather than truncating or dropping the family.
+            want_wordlike=bool(index%2)
+            try:
+                result=render(ep,name,seed,want_wordlike,limit)
+            except Oversize:
+                if not want_wordlike:raise
+                result=render(ep,name,seed,False,limit)
+                result['meta']['surface_fallback']='symbolic_due_to_context'
+            result['meta']['split']=split
+            return result
+        except Oversize:continue
+    raise RuntimeError(f'{name}: no complete paired example fits {limit} bytes')
+
+class CortexSource:
+    def __init__(self,state=None,limit=1024):self.s=dict(state or {'index':0});self.limit=limit
+    def record(self):
+        r=episode_pair(self.s['index'],limit=self.limit);self.s['index']+=1;return r
+    def state_dict(self):return dict(self.s)
+    def validation(self):
+        return [episode_pair(2*i+j,'cortex74/validation/v2',f,split,self.limit)
+                for i,(f,split) in enumerate([(FAMILIES[k],'train') for k in range(0,len(FAMILIES)-1,17)]+
+                [('compositional_execution',s) for s in ['train','composition','depth','length']]) for j in (0,1)]

unified/ablations/unified-v1-blt-none-s42/source/cortex_tasks.py

@@ -0,0 +1,521 @@
+"""Explicit executable tasks. Shared encoding never substitutes one task for another."""
+from collections import deque, Counter
+from dataclasses import dataclass
+from itertools import product
+import random
+
+REVISION = 'cortex-explicit-mechanisms-v1'
+
+# Operations deliberately have separate semantics; unknown operations are errors.
+def sequence_execute(xs, program, mapping):
+    xs=list(xs);trace=[]
+    for op in program:
+        if op=='reverse': xs=list(reversed(xs))
+        elif op=='rotate': xs=xs[1:]+xs[:1]
+        elif op=='swap':
+            for i in range(0,len(xs)-1,2):xs[i],xs[i+1]=xs[i+1],xs[i]
+        elif op=='map':xs=[mapping[x] for x in xs]
+        elif op=='dedup':xs=list(dict.fromkeys(xs))
+        elif op=='repeat':xs=[x for x in xs for _ in range(2)]
+        elif op=='evens':xs=xs[::2]
+        else:raise ValueError(op)
+        trace.append(xs[:])
+    return trace
+
+
+def traverse(edges,start,depth_first=False):
+    adj={}
+    for a,b in edges:adj.setdefault(a,[]).append(b)
+    seen=set();out=[];frontier=[start]
+    while frontier:
+        x=frontier.pop() if depth_first else frontier.pop(0)
+        if x in seen:continue
+        seen.add(x);out.append(x)
+        children=sorted(adj.get(x,[]))
+        frontier.extend(reversed(children) if depth_first else children)
+    return out
+
+
+def shortest(edges,start,goal):
+    q=deque([(start,[start])]);seen={start}
+    while q:
+        x,path=q.popleft()
+        if x==goal:return path
+        for a,b in sorted(edges):
+            if a==x and b not in seen:seen.add(b);q.append((b,path+[b]))
+    return []
+
+
+def selection_trace(xs):
+    xs=xs[:];out=[]
+    for i in range(len(xs)-1):
+        j=min(range(i,len(xs)),key=xs.__getitem__);xs[i],xs[j]=xs[j],xs[i];out.append(xs[:])
+    return out
+
+
+def insertion_trace(xs):
+    xs=xs[:];out=[]
+    for i in range(1,len(xs)):
+        x=xs[i];j=i
+        while j and xs[j-1]>x:xs[j]=xs[j-1];j-=1
+        xs[j]=x;out.append(xs[:])
+    return out
+
+
+def solve(name,x):
+    """Production semantics on structured latent inputs; no family-name fallback."""
+    if name=='selection_sort':return selection_trace(x)
+    if name=='insertion_sort':return insertion_trace(x)
+    if name=='minmax_scan':return [[min(x[:i]),max(x[:i])] for i in range(1,len(x)+1)]
+    if name=='linear_search':
+        arr,target=x;return [i for i,v in enumerate(arr) if v==target][:1] or [-1]
+    if name=='binary_search':
+        arr,target=x;lo,hi=0,len(arr)-1;visited=[]
+        while lo<=hi:
+            m=(lo+hi)//2;visited.append(m)
+            if arr[m]==target:break
+            if arr[m]<target:lo=m+1
+            else:hi=m-1
+        return [visited, m if lo<=hi else -1]
+    if name=='string_match':
+        text,pat=x;return [i for i in range(len(text)-len(pat)+1) if text[i:i+len(pat)]==pat]
+    if name in ('bfs_order','dfs_order'):return traverse(x[0],x[1],name=='dfs_order')
+    if name=='topo_sort':
+        nodes,edges=x;out=[]
+        while len(out)<len(nodes):
+            ready=[n for n in nodes if n not in out and all(a in out for a,b in edges if b==n)]
+            if not ready:raise ValueError('Cycle in topological task')
+            out.append(min(ready))
+        return out
+    if name=='shortest_path':return shortest(*x)
+    if name=='union_find':
+        edges,a,b=x;return b in traverse(edges+[[v,u] for u,v in edges],a)
+    if name=='interval_overlap':
+        a,b,c,d=x;return max(a,c)<=min(b,d)
+    if name=='bracket_balance':
+        depth=0
+        for op in x:
+            depth+=1 if op=='open' else -1
+            if depth<0:return False
+        return depth==0
+    if name=='bracket_depth':
+        d=best=0
+        for op in x:d+=1 if op=='open' else -1;best=max(best,d)
+        return best
+    if name in ('queue','deque'):
+        q=deque();out=[]
+        for op,*args in x:
+            if op=='right':q.append(args[0])
+            elif op=='left':q.appendleft(args[0])
+            elif op=='pop_left':out.append(q.popleft() if q else 'empty')
+            elif op=='pop_right':out.append(q.pop() if q else 'empty')
+            else:raise ValueError(op)
+        return [out,list(q)]
+    if name in ('kv_lookup','role_filler','assoc_distractors','nested_lookup','mutable_dictionary','delayed_recall'):
+        records,query=x;table={}
+        for k,v in records:table[k]=v
+        value=query[0]
+        for _ in range(query[1]):value=table[value]
+        return value
+    if name in ('register_swap','register_update','multi_register','program_trace'):
+        state,program=x;state=state[:];trace=[]
+        for op,a,b in program:
+            if op=='set':state[a]=b
+            elif op=='swap':state[a],state[b]=state[b],state[a]
+            elif op=='copy':state[a]=state[b]
+            else:raise ValueError(op)
+            trace.append(state[:])
+        return trace if name=='program_trace' else state
+    if name=='conditional_branch':
+        pred,a,b=x;return a if pred else b
+    if name=='loop_counter':return list(range(x,-1,-1))
+    if name in ('stack_expr','prefix_expr','postfix_expr'):
+        if name=='prefix_expr':
+            it=iter(x)
+            def parse():
+                t=next(it)
+                if isinstance(t,int):return t
+                a,b=parse(),parse()
+                return a+b if t=='add' else a*b
+            return parse()
+        stack=[]
+        for t in x:
+            if isinstance(t,int):stack.append(t)
+            elif t=='push':continue
+            else:
+                b,a=stack.pop(),stack.pop();stack.append(a+b if t=='add' else a*b)
+        if len(stack)!=1:raise ValueError('Bad expression')
+        return stack[0]
+    if name=='relative_coords':
+        n,(a,b),moves=x
+        for dx,dy in moves:a=max(0,min(n-1,a+dx));b=max(0,min(n-1,b+dy))
+        return [a,b]
+    if name=='toroidal':
+        n,p,moves=x
+        for delta in moves:p=(p+delta)%n
+        return p
+    if name=='collision':
+        a,va,b,vb,steps=x;trace=[]
+        for _ in range(steps):
+            na,nb=a+va,b+vb
+            a,b=min(na,nb),max(na,nb)
+            if na>=nb:va,vb=vb,va
+            trace.append([a,b,va,vb])
+        return trace
+    if name=='falling':
+        # Downward simultaneous settling: each column conserves occupied cells.
+        h=len(x);return [[1 if r>=h-sum(row[c] for row in x) else 0 for c in range(len(x[0]))] for r in range(h)]
+    if name=='obstacle_motion':
+        n,p,blocked,moves=x;visited=[p]
+        for delta in moves:
+            q=p+delta
+            if 0<=q<n and q not in blocked:p=q
+            visited.append(p)
+        return visited
+    if name=='line_of_sight':
+        a,b,blocked=x;return not any(min(a,b)<v<max(a,b) for v in blocked)
+    if name=='containment':
+        outer,inner=x;return outer[0]<=inner[0] and inner[1]<=outer[1]
+    if name=='partial_map':
+        observations,start,goal=x
+        edges=sorted({tuple(edge) for observation in observations for edge in observation})
+        return shortest(edges,start,goal)
+    if name=='dynamic_plan':
+        edges,changes,start,goal=x; edges=[e[:] for e in edges]
+        for op,a,b in changes:
+            if op=='remove':edges=[e for e in edges if e!=[a,b]]
+            elif op=='add' and [a,b] not in edges:edges.append([a,b])
+        return shortest(edges,start,goal)
+    if name in ('path_keys','door_key','sparse_nav','irreversible_action'):
+        # Real search in a directed graph; keys change which edges are traversable.
+        edges,start,goal,key_nodes,locks=x;q=deque([(start,0,[])]);seen={(start,0)}
+        while q:
+            node,key,path=q.popleft();key=key or int(node in key_nodes)
+            if node==goal:return path
+            for action,(a,b) in enumerate(edges):
+                if a!=node or (action in locks and not key):continue
+                state=(b,key)
+                if state not in seen:seen.add(state);q.append((b,key,path+[action]))
+        return []
+    if name=='causal_abduction':
+        a,b,y=x;return a^b^y
+    if name=='anomaly_cause':
+        u,a,b,observed_x,observed_y=x
+        return (['x'] if observed_x!=(u^a) else [])+(['y'] if observed_y!=(observed_x^b) else [])
+    if name in ('direct_cause','causal_intervention','confounding'):
+        # X=U xor A; Y=X xor B, except explicit interventions override equations.
+        u,a,b,intervention=x
+        xv=u^a;yv=xv^b
+        if intervention[0]=='x':xv=intervention[1];yv=xv^b
+        elif intervention[0]=='y':yv=intervention[1]
+        if name=='confounding':return [u^a,u^b] # fork U -> X and U -> Y
+        return [xv,yv]
+    if name=='abduce_noise':
+        a,b,observed_y=x;return observed_y^b^a
+    if name in ('competing_hypotheses','noisy_evidence','qual_bayes'):
+        # Integer likelihoods: exact posterior ranking, with priors and repeated evidence.
+        priors,likelihood,observations=x;score=priors[:]
+        for obs in observations:score=[s*row[obs] for s,row in zip(score,likelihood)]
+        best=max(score);return [i for i,v in enumerate(score) if v==best]
+    if name=='active_disambiguation':
+        # Pick the smallest test separating the largest number of hypothesis pairs.
+        table=x;counts=[sum(row[i]!=other[i] for j,row in enumerate(table) for other in table[j+1:]) for i in range(len(table[0]))]
+        return counts.index(max(counts))
+    if name in ('rule_change','rule_switch'):
+        maps,events,start=x;regime=0;state=start;out=[]
+        for op,arg in events:
+            if op=='switch':regime=arg
+            elif op=='step':state=maps[regime][state]
+            else:raise ValueError(op)
+            out.append(state)
+        return out
+    if name=='goal_switch':
+        edges,start,goals=x;out=[]
+        for goal in goals:
+            path=shortest(edges,start,goal);out.append(path);start=goal
+        return out
+    if name=='unknown_goal':
+        candidates,rewarded,rejected=x;return [n for n in candidates if n in rewarded and n not in rejected]
+    if name=='tool_affordance':
+        tools,requirements=x;return [i for i,t in enumerate(tools) if all(v in t for v in requirements)]
+    if name=='select_cell':
+        grid,color=x;return [[r,c] for r,row in enumerate(grid) for c,v in enumerate(row) if v==color]
+    if name=='hidden_rule':
+        rules,demos,query=x;valid=[r for r in rules if all((a+r)%7==b for a,b in demos)]
+        out={(query+r)%7 for r in valid}
+        if len(out)!=1:raise ValueError('Unidentifiable hidden rule')
+        return out.pop()
+    if name=='explore_infer_exploit':
+        # Hidden environment version; observations identify it, then pick rewarding action.
+        tables,observations,state,goal=x
+        valid=[t for t in tables if all(t[s][a]==z for s,a,z in observations)]
+        answers={tuple(a for a,z in enumerate(t[state]) if z==goal) for t in valid}
+        if len(answers)!=1:raise ValueError('Unidentifiable action')
+        return list(answers.pop())
+    if name in ('sequence_composition','instruction_comp','modifier_comp','compositional_holdout'):
+        xs,program,mapping=x;return sequence_execute(xs,program,mapping)[-1]
+    if name=='program_states':
+        xs,program,mapping=x;return sequence_execute(xs,program,mapping)
+    if name in ('novel_primitive','rule_remap'):
+        table,xv=x;return table[xv]
+    if name=='representation_remap':
+        mapping,values=x;return [mapping[v] for v in values]
+    if name=='cross_domain':
+        phi,perm,values=x;inv={v:i for i,v in enumerate(phi)};return [phi[perm[inv[v]]] for v in values]
+    if name=='context_operator':
+        xs,context=x;return list(reversed(xs)) if len(context)%2 else xs[1:]+xs[:1]
+    if name=='meta_rule':
+        xs,program,remap,perm=x;return sequence_execute(xs,[remap[o] for o in program],perm)[-1]
+    if name=='mini_dsl':
+        def run(node):
+            op,*args=node
+            if op=='literal':return args[0]
+            if op=='reverse':return run(args[0])[::-1]
+            if op=='concat':return run(args[0])+run(args[1])
+            if op=='repeat':return run(args[1])*args[0]
+            raise ValueError(op)
+        return run(x)
+    if name=='grammar_expansion':
+        rules,word,steps=x
+        for _ in range(steps):word=sum((rules.get(t,[t]) for t in word),[])
+        return word
+    if name=='recursive_dsl':
+        rules,root,depth=x
+        def expand(t,d):return [t] if not d or t not in rules else sum((expand(z,d-1) for z in rules[t]),[])
+        return expand(root,depth)
+    if name=='stable_partition':
+        xs,selected=x;return [v for v in xs if v in selected]+[v for v in xs if v not in selected]
+    if name=='run_length_codec':
+        out=[]
+        for v in x:
+            if out and out[-1][0]==v:out[-1][1]+=1
+            else:out.append([v,1])
+        return out
+    if name=='scope_shadowing':
+        frames,key=x
+        for frame in reversed(frames):
+            if key in frame:return frame[key]
+        return 'missing'
+    if name=='relational_join':
+        left,right=x;return [[k,a,b] for k,a in left for j,b in right if k==j]
+    if name=='edit_script':
+        xs,edits=x;xs=xs[:]
+        for op,i,v in edits:
+            if op=='insert':xs.insert(i,v)
+            elif op=='replace':xs[i]=v
+            elif op=='delete':xs.pop(i)
+            else:raise ValueError(op)
+        return xs
+    if name=='conditional_rewrite':
+        xs,a,b,guard=x;return [b if v==a and i and xs[i-1]==guard else v for i,v in enumerate(xs)]
+    if name=='span_reordering':
+        xs,start,end=x;return xs[:start]+list(reversed(xs[start:end]))+xs[end:]
+    if name=='nested_substitution':
+        tree,mapping=x
+        def walk(v):return [walk(z) for z in v] if isinstance(v,list) else mapping.get(v,v)
+        return walk(tree)
+    raise KeyError(f'No semantics for {name}')
+
+
+ALGORITHM_NAMES = 'selection_sort insertion_sort minmax_scan linear_search binary_search string_match bfs_order dfs_order topo_sort union_find shortest_path interval_overlap'.split()
+MEMORY_NAMES = 'kv_lookup role_filler assoc_distractors nested_lookup mutable_dictionary queue deque delayed_recall register_swap register_update conditional_branch loop_counter multi_register program_trace stack_expr prefix_expr postfix_expr bracket_balance bracket_depth'.split()
+WORLD_NAMES = 'relative_coords line_of_sight containment collision toroidal falling obstacle_motion path_keys dynamic_plan direct_cause confounding competing_hypotheses noisy_evidence active_disambiguation qual_bayes causal_intervention causal_abduction rule_change anomaly_cause'.split()
+META_NAMES = 'novel_primitive modifier_comp mini_dsl cross_domain context_operator meta_rule representation_remap compositional_holdout recursive_dsl instruction_comp rule_remap'.split()
+INTERACTIVE_NAMES = 'hidden_rule unknown_goal door_key tool_affordance sparse_nav partial_map irreversible_action explore_infer_exploit rule_switch goal_switch select_cell'.split()
+NEW_NAMES = 'sequence_composition program_states grammar_expansion stable_partition run_length_codec scope_shadowing relational_join edit_script conditional_rewrite span_reordering nested_substitution'.split()
+EXPLICIT_NAMES = ALGORITHM_NAMES+MEMORY_NAMES+WORLD_NAMES+META_NAMES+INTERACTIVE_NAMES+NEW_NAMES
+
+
+def sample_input(name,rng,difficulty=1,split='train'):
+    n=rng.randint(3,5) if split!='length' else rng.randint(8,10)
+    arr=lambda:[rng.randrange(5) for _ in range(n)]
+    order=rng.sample(range(6),6)
+    edges=[[order[i],order[i+1]] for i in range(5)]
+    edges += [[order[i],order[j]] for i in range(4) for j in range(i+2,6) if rng.random()<.35]
+    if name in ('selection_sort','insertion_sort','minmax_scan','run_length_codec'):return arr()
+    if name in ('linear_search','binary_search'):
+        xs=sorted(set(arr())) if name=='binary_search' else arr();return [xs,rng.choice(xs+[7])]
+    if name=='string_match':
+        xs=arr()+arr();i=rng.randrange(len(xs)-2);return [xs,xs[i:i+2]]
+    if name in ('bfs_order','dfs_order'):return [edges,rng.choice(order[:3])]
+    if name=='topo_sort':return [list(range(6)),[e for e in edges if rng.random()<.8]]
+    if name=='shortest_path':return [edges,order[0],rng.choice(order[2:])]
+    if name=='union_find':return [[[0,1],[1,2],[3,4]],rng.randrange(5),rng.randrange(5)]
+    if name=='interval_overlap':
+        return sorted(rng.sample(range(7),2))+sorted(rng.sample(range(7),2))
+    if name in ('bracket_balance','bracket_depth'):
+        depth=rng.randint(1,4 if split!='depth' else 7);seq=['open']*depth+['close']*depth
+        if name=='bracket_balance' and rng.random()<.5:seq=rng.choice([seq[:-1],['close']+seq])
+        return seq
+    if name in ('queue','deque'):
+        ops=[['right',rng.randrange(5)] for _ in range(n)]
+        if name=='deque':ops.insert(1,['left',rng.randrange(5)]);ops.append(['pop_right'])
+        return ops+[['pop_left'] for _ in range(rng.randint(1,n+1))]
+    if name in ('kv_lookup','role_filler','assoc_distractors','nested_lookup','mutable_dictionary','delayed_recall'):
+        keys=['a','b','c'];records=[[k,rng.randrange(5)] for k in keys];key=rng.choice(keys)
+        if name=='role_filler':records=[['agent',records[0][1]],['object',records[1][1]],['location',records[2][1]]];key=rng.choice(['agent','object','location'])
+        if name=='mutable_dictionary':records.append([key,rng.randrange(5)])
+        if name in ('assoc_distractors','delayed_recall'):records.extend([[f'noise{i}',rng.randrange(5)] for i in range(n)])
+        if name=='nested_lookup':
+            records=[['a','b'],['b','c'],['c',rng.randrange(5)]];return [records,['a',3]]
+        return [records,[key,1]]
+    if name in ('register_swap','register_update','multi_register','program_trace'):
+        ops=['swap'] if name=='register_swap' else ['set'] if name=='register_update' else ['set','copy','swap']
+        program=[[rng.choice(ops),rng.randrange(3),rng.randrange(3)] for _ in range(n)]
+        return [arr()[:3],program]
+    if name=='conditional_branch':return [bool(rng.randrange(2)),arr(),arr()]
+    if name=='loop_counter':return rng.randint(2,7)
+    if name in ('stack_expr','prefix_expr','postfix_expr'):
+        a,b,c=[rng.randint(0,2) for _ in range(3)];op=rng.choice(['add','mul'])
+        if name=='prefix_expr':return [op,'add',a,b,c]
+        if name=='postfix_expr':return [a,b,'add',c,op]
+        return ['push',a,'push',b,'add','push',c,op]
+    if name=='relative_coords':return [7,[rng.randrange(7),rng.randrange(7)],[rng.choice([[1,0],[-1,0],[0,1],[0,-1]]) for _ in range(n)]]
+    if name=='toroidal':return [5,rng.randrange(5),[rng.choice([-1,1]) for _ in range(n)]]
+    if name=='collision':return [0,1,rng.choice([2,3,4]),-1,rng.randint(2,3)]
+    if name=='falling':return [[rng.randrange(2) for _ in range(3)] for _ in range(4)]
+    if name=='obstacle_motion':return [7,2,[4],[rng.choice([-1,1]) for _ in range(n)]]
+    if name=='line_of_sight':return [0,6,rng.sample(range(8),rng.randrange(4))]
+    if name=='containment':return [sorted(rng.sample(range(8),2)),sorted(rng.sample(range(8),2))]
+    if name=='partial_map':
+        shuffled=edges[:];rng.shuffle(shuffled)
+        observations=[shuffled[::2],shuffled[1::2]]
+        # Neither observation alone is guaranteed to expose the full route.
+        return [observations,order[0],order[-1]]
+    if name=='dynamic_plan':
+        # A formerly shortest direct route disappears; another edge may appear.
+        start,goal=order[0],order[-1]
+        base=edges+[[start,goal]]
+        changes=[['remove',start,goal]]
+        if rng.random()<.5:changes.append(['add',order[1],goal])
+        return [base,changes,start,goal]
+    if name in ('path_keys','door_key'):
+        nodes=rng.sample(range(8),8);start,key,trap=nodes[:3]
+        path=[start]+nodes[3:3+rng.randint(2,4)]
+        road=[[a,b] for a,b in zip(path,path[1:])]
+        gate=road[rng.randrange(len(road))]
+        graph=road+[[start,key],[key,start],[start,trap]]
+        rng.shuffle(graph)
+        return [graph,start,path[-1],[key],[graph.index(gate)]]
+    if name=='irreversible_action':
+        nodes=rng.sample(range(7),7);start,trap=nodes[:2]
+        route=[start]+nodes[2:rng.randint(5,7)]
+        graph=[[a,b] for a,b in zip(route,route[1:])]+[[start,trap],[trap,trap]]
+        rng.shuffle(graph)
+        return [graph,start,route[-1],[],[]]
+    if name=='sparse_nav':
+        rng.shuffle(edges)
+        return [edges,order[0],rng.choice(order[2:]),[],[]]
+    if name=='causal_abduction':return [rng.randrange(2),rng.randrange(2),rng.randrange(2)]
+    if name=='anomaly_cause':
+        u,a,b=[rng.randrange(2) for _ in range(3)];vx=u^a;vy=vx^b
+        if rng.random()<.5:vx^=1;vy=vx^b
+        else:vy^=1
+        return [u,a,b,vx,vy]
+    if name in ('direct_cause','causal_intervention','confounding'):
+        iv=['none',0] if name in ('direct_cause','confounding') else [rng.choice(['x','y']),rng.randrange(2)]
+        return [rng.randrange(2),rng.randrange(2),rng.randrange(2),iv]
+    if name in ('competing_hypotheses','noisy_evidence','qual_bayes'):
+        prior=[1,1] if name=='competing_hypotheses' else [rng.randint(1,3),rng.randint(1,3)]
+        return [prior,[[3,1],[1,3]],[rng.randrange(2) for _ in range(n if name!='qual_bayes' else 1)]]
+    if name=='active_disambiguation':return [[rng.randrange(2) for _ in range(4)] for _ in range(3)]
+    if name in ('rule_change','rule_switch'):
+        maps=[rng.sample(range(5),5),rng.sample(range(5),5)]
+        return [maps,[['step',0],['switch',1],['step',0],['step',0]],rng.randrange(5)]
+    if name=='goal_switch':return [edges+[[5,0]],0,[rng.choice([3,4]),5,0]]
+    if name=='unknown_goal':
+        goal=rng.randrange(5);return [list(range(5)),[goal],[v for v in range(5) if v!=goal]]
+    if name=='tool_affordance':return [[rng.sample(range(5),rng.randint(1,5)) for _ in range(3)],rng.sample(range(5),2)]
+    if name=='select_cell':return [[[rng.randrange(3) for _ in range(3)] for _ in range(3)],rng.randrange(3)]
+    if name=='hidden_rule':
+        delta=rng.randrange(7);a=rng.randrange(7);return [list(range(7)),[[a,(a+delta)%7]],rng.randrange(7)]
+    if name=='explore_infer_exploit':
+        tables=[[[0,1],[1,0]],[[1,0],[0,1]]];world=rng.randrange(2);return [tables,[[0,0,tables[world][0][0]]],rng.randrange(2),1]
+    if name in ('sequence_composition','program_states','instruction_comp','modifier_comp','compositional_holdout','meta_rule'):
+        mapping=rng.sample(range(5),5);ops=['reverse','rotate','swap','map'];depth=rng.randint(1,3) if split!='depth' else rng.randint(4,6)
+        if split=='composition':program=['rotate','swap']
+        else:
+            while True:
+                program=[rng.choice(ops) for _ in range(depth)]
+                if all((a,b) not in [('rotate','swap'),('swap','rotate')] for a,b in zip(program,program[1:])):break
+        if name=='modifier_comp':program=['reverse']+program[:1]+['reverse']
+        if name=='meta_rule':return [arr(),program,dict(zip(ops,rng.sample(ops,len(ops)))),mapping]
+        return [arr(),program,mapping]
+    if name in ('novel_primitive','rule_remap'):return [rng.sample(range(7),7),rng.randrange(7)]
+    if name=='representation_remap':return [rng.sample(range(5),5),arr()]
+    if name=='cross_domain':return [rng.sample(range(5),5),rng.sample(range(5),5),arr()]
+    if name=='context_operator':return [arr(),arr()[:rng.randint(1,n)]]
+    if name=='mini_dsl':return ['concat',['reverse',['literal',arr()]],['repeat',2,['literal',arr()[:2]]]]
+    if name=='grammar_expansion':return [{0:[1,2],1:[0],2:[3]},arr()[:2],rng.randint(1,3)]
+    if name=='recursive_dsl':
+        depth=rng.randint(1,3) if split!='depth' else 4
+        return [{'a':['b','c'],'b':['a'],'c':['d']},rng.choice(['a','b']),depth]
+    if name=='stable_partition':return [arr(),rng.sample(range(5),2)]
+    if name=='scope_shadowing':return [[{'a':1,'b':2},{'a':rng.randrange(5)},{'b':rng.randrange(5)}],rng.choice(['a','b'])]
+    if name=='relational_join':return [[[rng.randrange(3),rng.randrange(5)] for _ in range(3)],[[rng.randrange(3),rng.randrange(5)] for _ in range(3)]]
+    if name=='edit_script':return [arr(),[['insert',1,rng.randrange(5)],['replace',0,rng.randrange(5)],['delete',2,0]]]
+    if name=='conditional_rewrite':return [arr()+[1,2],2,3,1]
+    if name=='span_reordering':return [arr(),1,n]
+    if name=='nested_substitution':return [[arr()[:2],[arr()[:2],rng.randrange(5)]],{0:1,1:2,2:3,3:4,4:0}]
+    raise KeyError(f'No sampler for {name}')
+
+
+def answer(name,x):return solve(name,x)
+
+
+def build_episode(c,name,rng,context,stage,split='train',difficulty=1):
+    cases=[]
+    if name in ('queue','deque'):
+        v=[['pop_left']];cases.append((v,answer(name,v)))
+    if name in ('sequence_composition','program_states','instruction_comp','modifier_comp','compositional_holdout'):
+        for op in ('reverse','rotate','swap','map'):
+            v=[[0,1,2,3,4],[op],[1,2,3,4,0]]
+            cases.append((v,answer(name,v)))
+    for _ in range(3):
+        x=sample_input(name,rng,difficulty,'train');cases.append((x,answer(name,x)))
+    x=sample_input(name,rng,difficulty,split)
+    # Avoid exact query replay from the demonstrations when the support permits it.
+    for _ in range(30):
+        if all(x!=a for a,_ in cases):break
+        x=sample_input(name,rng,difficulty,split)
+    query=(x,answer(name,x))
+    s=c.Surface(rng);start,close,sep,end,unit,neg=s.take(6);symbols={}
+    def symbol(v):
+        key=(type(v).__name__,v)
+        if key not in symbols:symbols[key]=s.one()
+        return symbols[key]
+    def encode(v):
+        if isinstance(v,dict):return [start]+sum((encode([k,z]) for k,z in v.items()),[])+[close]
+        if isinstance(v,(list,tuple)):return [start]+sum((encode(z) for z in v),[])+[close]
+        return [symbol(v)]
+    # Ground numeric order and magnitude once, not English number words.
+    def integers(v):
+        if type(v) is int:yield v
+        elif isinstance(v,dict):
+            for a,b in v.items():yield from integers(a);yield from integers(b)
+        elif isinstance(v,(list,tuple)):
+            for z in v:yield from integers(z)
+    values=set(integers([cases,query]));b=c.Builder(stage,name)
+    if not values.issubset(set(range(-8,17))):raise ValueError(f'Numeric domain overflow: {name}')
+    for n in range(-8,17):b.emit(([neg] if n<0 else [])+[unit]*abs(n)+[sep,symbol(n),end])
+    b.emit([symbol(False),sep,symbol(0),end,symbol(True),sep,symbol(1),end])
+    for a,y in cases:b.emit(encode(a)+[sep]+encode(y)+[end])
+    b.emit(encode(query[0])+[sep]);b.mark_prompt();b.emit(encode(query[1]),c.W_TARGET)
+    ep=b.finish(context);ep.audit={'mechanism':name,'cases':cases,'query':query,'symbols':symbols,'markers':[start,close,sep,end,unit,neg],'split':split}
+    return ep
+
+
+def install(namespace):
+    # Replace each misleading v16 alias with an explicitly implemented mechanism.
+    for stage,names in [(23,MEMORY_NAMES),(25,WORLD_NAMES),(26,ALGORITHM_NAMES),(27,META_NAMES),(28,INTERACTIVE_NAMES),(29,NEW_NAMES)]:
+        funcs=[]
+        for name in names:
+            def generator(rng,context,st=stage,_name=name):
+                class Kernel:pass
+                c=Kernel()
+                for key in ['Surface','Builder','W_TARGET']:setattr(c,key,namespace[key])
+                return build_episode(c,_name,rng,context,st,difficulty=namespace['_g16_diff']())
+            generator.__name__='gen_'+name;namespace[generator.__name__]=generator;funcs.append(generator)
+        namespace['STAGES'][stage]=funcs
+        namespace['_V10_CONTEXT_TABLE'][stage]=(1536,1536,1536)
+    namespace['STAGE_NAMES'][29]='compositional_structures_and_pseudoword_transfer'

unified/ablations/unified-v1-blt-none-s42/source/data_stream.py

@@ -0,0 +1,193 @@
+"""Pinned, repeatable shard streams. A single coordinator owns all cursors."""
+import io,json,gzip,copy,hashlib,random,math
+from pathlib import Path
+from collections import Counter
+import numpy as np
+import httpx,requests,time
+from huggingface_hub import HfFileSystem,get_token
+from parquet_stream import ParquetStream
+from records import record,plain_chunks,validate,Oversize,InvalidRecord,PAD
+from media_data import visual_records,audio_records
+from cortex_source import CortexSource
+from dataset_registry import SOURCES,CORTEX_WEIGHT,retry
+
+HOLDOUT=2
+
+def row_key(spec,row):
+    # Same source media bytes map to the same key across repositories and captions.
+    h=hashlib.sha256()
+    media=row.get('images', [row.get('image')]) if spec['kind']=='image' else [row.get('audio')] if spec['kind']=='audio' else []
+    for v in media:
+        if isinstance(v,dict) and v.get('bytes') is not None:h.update(v['bytes'])
+        else:h.update(repr(v).encode())
+    if not media:h.update(json.dumps(row,sort_keys=True,ensure_ascii=False,default=str).encode())
+    return h.hexdigest()
+
+class JSONRows:
+    """Compressed JSONL resume replays only the current pinned shard, never the corpus."""
+    def __init__(self,spec,filename,state=None,opener=None):
+        self.spec,self.filename=spec,filename;self.row=int((state or {}).get('row',0));self.opener=opener
+    def state_dict(self):return {'format':'jsonl-row-v1','row':self.row}
+    def __iter__(self):
+        handle=self.opener() if self.opener else HfFileSystem(token=get_token()).open(f'datasets/{self.spec["repo"]}@{self.spec["revision"]}/{self.filename}','rb',block_size=256*1024,cache_type='readahead')
+        with handle:
+            reader=handle
+            if self.filename.endswith('.zst'):
+                import zstandard
+                reader=zstandard.ZstdDecompressor().stream_reader(handle)
+            elif self.filename.endswith('.gz'):reader=gzip.GzipFile(fileobj=handle)
+            with io.TextIOWrapper(reader,encoding='utf-8') as lines:
+                for i,line in enumerate(lines):
+                    if i<self.row:continue
+                    if not line.strip():self.row=i+1;continue
+                    data=json.loads(line);self.row=i+1;yield data
+
+class Source:
+    def __init__(self,name,spec,state=None,limit=1024,loader=None):
+        self.name,self.spec,self.limit,self.loader=name,spec,limit,loader
+        self.s=copy.deepcopy(state) if state else dict(shard=0,epoch=0,cursor=None,rows=0,pass_rows=0,pending=[],held=[],held_keys=[],initialized=False,rejected=0,accepted=0)
+        self.reader=self.it=None
+    def close(self):
+        if self.it is not None and hasattr(self.it,'close'):self.it.close()
+        self.it=self.reader=None
+    def open(self):
+        self.close();f=self.spec['files'][self.s['shard']]
+        if self.loader:self.reader=self.loader(f,self.s['cursor'])
+        elif f.endswith('.parquet'):
+            self.reader=ParquetStream(self.spec['repo'],f,self.spec['revision'],list(self.spec['columns']),batch_size=1 if self.spec['kind'] in ('image','audio') else 64)
+            if self.s['cursor']:self.reader.load_state_dict(self.s['cursor'])
+        else:self.reader=JSONRows(self.spec,f,self.s['cursor'])
+        self.it=iter(self.reader)
+    def raw(self):
+        attempts=0
+        while True:
+            if self.it is None:retry(self.open)
+            try:
+                row=next(self.it);self.s['cursor']=copy.deepcopy(self.reader.state_dict());self.s['rows']+=1;self.s['pass_rows']+=1;return row
+            except StopIteration:
+                self.close();self.s['cursor']=None;self.s['shard']+=1
+                if self.s['shard']==len(self.spec['files']):
+                    if not self.s['pass_rows']:raise RuntimeError(self.name+': empty dataset')
+                    self.s['shard']=0;self.s['epoch']+=1;self.s['pass_rows']=0
+            except (OSError,TimeoutError,ConnectionError,httpx.HTTPError,requests.RequestException) as e:
+                # Reopen at the last committed row, never move past a failed fetch.
+                attempts+=1
+                if attempts>=5:raise RuntimeError(self.name+': stream retry limit reached; resume from the durable checkpoint') from e
+                print(self.name+': retrying current shard at the saved row cursor',flush=True)
+                time.sleep(min(16,2**attempts));self.close()
+    def transform(self,row,key):
+        kind=self.spec['kind']
+        if kind=='image':return visual_records(self.name,row,self.limit)
+        if kind=='audio':return audio_records(self.name,row,self.limit,self.s['epoch'],key)
+        if kind in ('rewrite','openhermes'):
+            instruction=row.get('instruction');text=row.get('text') if kind=='rewrite' else row.get('input')
+            if kind=='rewrite' or (isinstance(text,str) and text.strip()):
+                output=row.get('output')
+                if not all(isinstance(v,str) for v in [instruction,text,output]):raise InvalidRecord(self.name+': expected instruction/text/output strings')
+                if kind=='rewrite':
+                    # rewrite-plus format: no labels, exactly one blank line between fields.
+                    # Train causally on every byte so rewrite6 behaves like corpus text.
+                    return plain_chunks(f'{instruction}\n\n{text}\n\n{output}',self.name,self.limit)
+                return [record(f'Instruction:\n{instruction}\n\nText:\n{text}\n\nRewrite:\n',output,self.name,self.limit,supervise_all=True)]
+        text=row.get(self.spec['column'])
+        if not isinstance(text,str):raise InvalidRecord(self.name+': missing text column '+self.spec['column'])
+        return plain_chunks(text,self.name,self.limit)
+    def initialize(self):
+        if self.s['initialized']:return
+        for _ in range(128):
+            row=self.raw();key=row_key(self.spec,row)
+            try:records=self.transform(row,key)
+            except (Oversize,InvalidRecord,ValueError):self.s['rejected']+=1;continue
+            self.s['held_keys'].append(key)
+            # One complete record per held-out source row; fixed across restarts.
+            self.s['held'].append(records[0])
+            if self.spec['kind']=='audio':
+                self.s['held'].extend(r for r in records[1:] if r['meta'].get('task')=='generation')
+            if len(self.s['held_keys'])==HOLDOUT:break
+        if len(self.s['held_keys'])<HOLDOUT:raise RuntimeError(self.name+': unable to construct validation records')
+        self.s['initialized']=True
+    def next_record(self,excluded):
+        if self.s['pending']:return self.s['pending'].pop(0)
+        for _ in range(256):
+            row=self.raw();key=row_key(self.spec,row)
+            if key in excluded:continue
+            try:rs=self.transform(row,key)
+            except Oversize:self.s['rejected']+=1;continue
+            except (InvalidRecord,ValueError) as e:
+                self.s['rejected']+=1
+                if self.s['rejected']<=3:print(f'{self.name}: rejected {type(e).__name__}: {e}',flush=True)
+                continue
+            self.s['accepted']+=1;self.s['pending']=rs[1:];return validate(rs[0],self.limit)
+        raise RuntimeError(self.name+': 256 consecutive unusable/held-out rows; fix the schema or byte budget')
+    def state_dict(self):return copy.deepcopy(self.s)
+
+class Mixer:
+    """Source weights choose blocks, not rows. Complete structured records never split.
+
+    Text chunks carry the boundary byte. Packing is ordinary causal document packing
+    with BOS/EOS boundaries; there is no quadratic block-diagonal attention mask.
+    """
+    def __init__(self,manifest,limit=1024,state=None,loaders=None,cortex=True,mix_weights=None):
+        state=state or {};self.limit=limit;self.manifest=manifest
+        self.sources={n:Source(n,spec,state.get('sources',{}).get(n),limit,(loaders or {}).get(n)) for n,spec in manifest.items()}
+        self.cortex=CortexSource(state.get('cortex'),limit) if cortex else None
+        self.rng=random.Random(20260911)
+        if 'rng' in state:self.rng.setstate(state['rng'])
+        defaults={n:int(spec['weight']) for n,spec in manifest.items()}
+        if self.cortex:defaults['cortex']=int(CORTEX_WEIGHT)
+        requested=dict(state.get('mix_weights',{}));requested.update(mix_weights or {})
+        self.mix_weights={n:int(requested.get(n,defaults[n])) for n in defaults}
+        if any(v<=0 for v in self.mix_weights.values()):raise ValueError('All active mix weights must be positive integers')
+        self.mix_queue=list(state.get('mix_queue',[]))
+        if any(n not in self.mix_weights for n in self.mix_queue):self.mix_queue=[]
+        self.pending=copy.deepcopy(state.get('pending',{}));self.stats=Counter(state.get('stats',{}))
+        self.excluded=set(state.get('excluded',[]))
+    def initialize(self):
+        for s in self.sources.values():
+            s.initialize();self.excluded.update(s.s['held_keys'])
+        # Validate one real training record per source before GPU allocation.
+        for name,s in self.sources.items():
+            if name not in self.pending:self.pending[name]=s.next_record(self.excluded)
+        return self
+    def next_for(self,name):
+        if name in self.pending:return self.pending.pop(name)
+        return self.cortex.record() if name=='cortex' else self.sources[name].next_record(self.excluded)
+    def _refill_mix_queue(self):
+        # Low-discrepancy weighted window: exact long-run quotas, randomized phase,
+        # and roughly even spacing. This keeps small rewrite6/Cortex shares present
+        # across the entire run without random multi-thousand-block droughts.
+        slots=[]
+        for name,weight in self.mix_weights.items():
+            phase=self.rng.random()
+            for k in range(weight):
+                position=(k+phase)/weight
+                jitter=(self.rng.random()-.5)*(.30/weight)
+                slots.append((position+jitter,self.rng.random(),name))
+        slots.sort();self.mix_queue=[name for _,__,name in slots]
+        self.stats['mix/windows']+=1
+    def block(self):
+        if not self.mix_queue:self._refill_mix_queue()
+        name=self.mix_queue.pop(0);tokens=[];mask=[]
+        while True:
+            r=self.next_for(name);validate(r,self.limit)
+            if tokens and len(tokens)+len(r['ids'])>self.limit+1:self.pending[name]=r;break
+            tokens.extend(r['ids']);mask.extend(r['weights'])
+            if len(tokens)==self.limit+1 or len(tokens)>=int(self.limit*.90):break
+            # Large language chunks are already context-sized; small rows pack.
+        used=len(tokens);n=self.limit+1-used
+        tokens.extend([PAD]*n);mask.extend([0]*n)
+        self.stats[name+'/blocks']+=1;self.stats[name+'/targets']+=sum(mask[1:]);self.stats[name+'/tokens']+=used-1
+        return tokens,mask,name,used
+    def batch(self,count):
+        blocks=[self.block() for _ in range(count)]
+        # All ranks receive the same trimmed width, bounded by the full 1024 limit.
+        width=min(self.limit+1,1+math.ceil(max(b[3]-1 for b in blocks)/128)*128)
+        return {'ids':np.array([b[0][:width] for b in blocks],dtype=np.int64),'weights':np.array([b[1][:width] for b in blocks],dtype=np.float32),'sources':[b[2] for b in blocks]}
+    def validation(self):
+        return [r for s in self.sources.values() for r in s.s['held']]+(self.cortex.validation() if self.cortex else [])
+    def state_dict(self):
+        return dict(sources={n:s.state_dict() for n,s in self.sources.items()},cortex=self.cortex.state_dict() if self.cortex else None,
+                    rng=self.rng.getstate(),mix_weights=dict(self.mix_weights),mix_queue=list(self.mix_queue),
+                    pending=copy.deepcopy(self.pending),stats=dict(self.stats),excluded=sorted(self.excluded))
+    def close(self):
+        for s in self.sources.values():s.close()

unified/ablations/unified-v1-blt-none-s42/source/dataset_registry.py

@@ -0,0 +1,46 @@
+"""Enabled language-training sources for the current SparkBET run."""
+from dataclasses import dataclass,asdict
+import fnmatch,time
+from huggingface_hub import HfApi
+
+@dataclass(frozen=True)
+class Spec:
+    repo:str
+    patterns:tuple
+    columns:tuple
+    weight:float
+    kind:str='text'
+    column:str='text'
+
+# Keep the full project data framework while the current run uses language-only
+# repositories. Non-language dataset entries are intentionally inactive.
+SOURCES={
+ 'climbmix':Spec('karpathy/climbmix-400b-shuffle',('*.parquet','**/*.parquet'),('text',),50,column='text'),
+ 'rewrite6':Spec('appvoid/rewrite6',('**/rewrite.jsonl','rewrite.jsonl','**/*.jsonl'),('instruction','text','output'),10,'rewrite'),
+ 'ultra_style':Spec('openbmb/Ultra-FineWeb-L3',('data/ultrafineweb_en_l3/multi_style/*.parquet',),('content',),20,column='content'),
+ 'ultra_qa':Spec('openbmb/Ultra-FineWeb-L3',('data/ultrafineweb_en_l3/qa/*.parquet',),('content',),20,column='content'),
+}
+CORTEX_WEIGHT=15
+
+
+def retry(fn,attempts=5):
+    for i in range(attempts):
+        try:return fn()
+        except Exception as e:
+            status=getattr(getattr(e,'response',None),'status_code',None)
+            if status in (401,403,404):raise
+            if i==attempts-1:raise
+            print(f'Network retry {i+1}/{attempts}: {type(e).__name__}',flush=True);time.sleep(min(16,2**i))
+
+
+def make_manifest(overrides=None,disabled=()):
+    api=HfApi();cache={};manifest={};overrides=overrides or {}
+    for name,spec in SOURCES.items():
+        if name in disabled:continue
+        if spec.repo not in cache:cache[spec.repo]=retry(lambda:api.dataset_info(spec.repo))
+        info=cache[spec.repo];patterns=overrides.get(name,spec.patterns)
+        files=sorted({f.rfilename for f in info.siblings if any(fnmatch.fnmatch(f.rfilename,p) for p in patterns)})
+        if not files:raise RuntimeError(f'{name}: no training files match {patterns} in {spec.repo}. Set DATA_FILE_OVERRIDES after inspecting the repository.')
+        manifest[name]={**asdict(spec),'revision':info.sha,'files':files}
+        print(f'{name}: pinned {len(files)} shards at {info.sha[:12]}',flush=True)
+    return manifest

unified/ablations/unified-v1-blt-none-s42/source/inference.py

@@ -0,0 +1,43 @@
+"""Small text-generation helper for an exported SparkBET repository."""
+from pathlib import Path
+import torch
+from safetensors.torch import load_file
+from bet_model import SparkBET,BETConfig,uniform_steps
+
+
+class Cortex:
+    def __init__(self,model,device=None):
+        self.model=model
+        self.device=torch.device(device or ("cuda" if torch.cuda.is_available() else "cpu"))
+        self.model.to(self.device).eval()
+
+    @classmethod
+    def from_export(cls,folder,device=None):
+        folder=Path(folder);model=SparkBET(BETConfig())
+        state=load_file(str(folder/"model.safetensors"),device="cpu")
+        if state and all(k.startswith("core.") for k in state):state={k[5:]:v for k,v in state.items()}
+        model.load_state_dict(state,strict=True)
+        return cls(model,device)
+
+    def generate_ids(self,ids,max_new_tokens=128,loops=8,temperature=0.0,top_k=None):
+        out=list(map(int,ids))
+        for _ in range(int(max_new_tokens)):
+            current=out[-self.model.c.max_seq_len:]
+            x=torch.tensor([current],device=self.device,dtype=torch.long)
+            with torch.inference_mode(),torch.autocast(self.device.type,dtype=torch.float16,enabled=self.device.type=="cuda"):
+                logits=self.model(x,uniform_steps(loops))[0,-1].float()
+            if temperature and temperature>0:
+                logits=logits/float(temperature)
+                if top_k:
+                    values,_=torch.topk(logits,min(int(top_k),logits.numel()));logits[logits<values[-1]]=-float("inf")
+                nxt=int(torch.multinomial(torch.softmax(logits,-1),1))
+            else:nxt=int(logits.argmax())
+            out.append(nxt)
+            if nxt==258:break
+        return out
+
+    def generate(self,text,max_new_tokens=128,loops=8,temperature=0.0,top_k=None):
+        ids=[257]+list(text.encode("utf-8"))
+        out=self.generate_ids(ids,max_new_tokens,loops,temperature,top_k)
+        body=bytes(i for i in out[1:] if 0<=i<=255)
+        return body.decode("utf-8",errors="replace")

unified/ablations/unified-v1-blt-none-s42/source/media_data.py

@@ -0,0 +1,88 @@
+"""Complete caption/chat and audio records with strict byte budgets."""
+import io,json,hashlib,math
+from PIL import Image,ImageOps
+from records import record,Oversize,InvalidRecord
+from braille_encoder import encode_image,fit_box_to_square,local_contrast,floyd_steinberg,binary_to_braille
+from audio_codec import canonical_audio,overview,encode_tile,CHUNK,RATE
+
+def load_image(value):
+    if isinstance(value,Image.Image):im=value.copy()
+    elif isinstance(value,dict):
+        src=io.BytesIO(value['bytes']) if value.get('bytes') is not None else value.get('path')
+        if src is None:raise InvalidRecord('Image has no embedded data or local path')
+        with Image.open(src) as source:im=source.copy()
+    else:
+        with Image.open(value) as source:im=source.copy()
+    im=ImageOps.exif_transpose(im).convert('RGBA');im.thumbnail((1024,1024))
+    return im
+
+def image_text(value,grid=80):
+    if grid>80 or grid<8 or grid%4:raise ValueError('Grid must be a multiple of four, at most 80')
+    im=load_image(value)
+    # Full-frame fit uses the supplied contrast/dither/dot layout. Saliency crops
+    # can discard the object asked about, so dataset examples preserve the frame.
+    small=fit_box_to_square(im,(0,0,*im.size),grid)
+    gray,alpha=local_contrast(small)
+    return '<image>\n'+binary_to_braille(floyd_steinberg(gray,alpha),alpha)+'\n</image>'
+
+def content_text(content):
+    if isinstance(content,str):return content.replace('<image>','').strip()
+    if not isinstance(content,list):raise InvalidRecord('Unknown chat content')
+    return '\n'.join(p['text'] for p in content if p.get('type')=='text' and p.get('text')).strip()
+
+def visual_records(name,row,limit=8192):
+    pictures=row['images'] if name=='llava' else [row['image']]
+    if not pictures:raise InvalidRecord('No image')
+    # Prefer full 80x80 resolution; reduce only to fit multiple complete images.
+    for grid in [80,64,48,32,16,8]:
+        image='\n'.join(image_text(v,grid) for v in pictures)
+        if len(image.encode())<limit-1024:break
+    else:raise Oversize('Too many images')
+    out=[]
+    def add(prefix,answer):
+        if not isinstance(answer,str) or not answer.strip():raise InvalidRecord('Empty visual answer')
+        try:out.append(record(prefix,answer,name,limit,meta={'grid':grid}))
+        except Oversize:pass  # Whole target rejected; no fabricated answer fragments.
+    if name=='flickr8k':
+        for i in range(5):add(image+'\n\nCaption:\n',row[f'caption_{i}'])
+    elif name=='coco_captions':
+        values=row['caption'] if isinstance(row['caption'],list) else [row['caption']]
+        for caption in values:add(image+'\n\nCaption:\n',caption)
+    elif name=='chartqa':
+        values=row['label'] if isinstance(row['label'],list) else [row['label']]
+        for answer in values:add(image+'\n\nUser:\n'+row['query']+'\n\nAssistant:\n',answer)
+    else:
+        messages=row['messages'] if name=='llava' else [{'role':{'human':'user','gpt':'assistant'}[m['from']],'content':m['value']} for m in row['conversations']]
+        history=image;last=None
+        for m in messages:
+            role=m['role'];text=content_text(m['content'])
+            if role not in ('user','assistant','system'):raise InvalidRecord('Unknown role')
+            if role=='assistant':
+                if last!='user':raise InvalidRecord('Assistant without user context')
+                prefix=history+'\n\nAssistant:\n';add(prefix,text)
+                # Preserve full history for follow-ups; never reset to question only.
+            history+='\n\n'+role.title()+':\n'+text;last=role
+    if not out:raise Oversize('No complete visual turn fits the context')
+    return out
+
+def audio_records(name,row,limit=8192,epoch=0,identity=''):
+    wave=canonical_audio(row['audio']);caption=row['text'] if name=='librispeech' else row['caption']
+    if not isinstance(caption,str) or not caption.strip():raise InvalidRecord('Missing audio text')
+    if len(wave)>RATE*30:raise Oversize('Whole-clip overview restricted to 30 seconds; no partial audio/full transcript pairs')
+    length=len(wave)/RATE;out=[]
+    kind='Transcript' if name=='librispeech' else 'Caption'
+    prefix=f'<audio_overview sr={RATE} seconds={length:.6f} freq_low=bottom time=right>\n'+overview(wave)+'\n</audio_overview>\n\n'+kind+':\n'
+    try:out.append(record(prefix,caption,name,limit,meta={'task':'understanding','seconds':length}))
+    except Oversize:pass
+    total=math.ceil(len(wave)/CHUNK)
+    # Each source-row pass advances its selected tile. Every tile is eventually
+    # visited without storing thousands of expanded samples in checkpoint state.
+    offset=int.from_bytes(hashlib.blake2b(identity.encode(),digest_size=8).digest(),'big')
+    tile=(offset+epoch)%total
+    target=encode_tile(wave[tile*CHUNK:(tile+1)*CHUNK])
+    previous=encode_tile(wave[(tile-1)*CHUNK:tile*CHUNK]) if tile else '<audio_start>'
+    prefix=f'{kind}:\n{caption}\n\nGenerate audio tile {tile+1}/{total}; start_sample={tile*CHUNK}; sample_rate={RATE}.\nPrevious:\n{previous}\nNext:\n'
+    try:out.append(record(prefix,target,name,limit,meta={'task':'generation','tile':tile,'tiles':total}))
+    except Oversize:pass
+    if not out:raise Oversize('No complete audio record fits')
+    return out

unified/ablations/unified-v1-blt-none-s42/source/modeling_bet.py

@@ -0,0 +1,76 @@
+import os
+import torch
+import torch.nn.functional as F
+from transformers import PreTrainedModel, GenerationMixin
+from transformers.modeling_outputs import CausalLMOutput
+
+from .configuration_bet import BETConfig
+from .bet_model import BETConfig as CoreConfig, SparkBET, uniform_steps
+
+
+class BETPreTrainedModel(PreTrainedModel):
+    config_class=BETConfig
+    base_model_prefix="core"
+    supports_gradient_checkpointing=False
+    _no_split_modules=["PlainBlock","LoopedBlock"]
+
+
+class BETForCausalLM(BETPreTrainedModel,GenerationMixin):
+    def __init__(self,config):
+        super().__init__(config)
+        core_cfg=CoreConfig(
+            vocab_size=config.vocab_size,
+            hidden_size=config.hidden_size,
+            intermediate_size=config.intermediate_size,
+            prelude_layers=config.prelude_layers,
+            body_blocks=config.body_blocks,
+            coda_layers=config.coda_layers,
+            num_heads=config.num_attention_heads,
+            num_kv_heads=config.num_key_value_heads,
+            head_dim=config.head_dim,
+            lora_rank=config.lora_rank,
+            hyper_lanes=config.hyper_lanes,
+            max_seq_len=config.max_position_embeddings,
+            max_loops=config.max_loops,
+            rope_theta=config.rope_theta,
+            rms_eps=config.rms_norm_eps,
+            ddl_beta_init=config.ddl_beta_init,
+            ddl_k_eps=config.ddl_k_eps,
+            ddl_v_sigmoid_scale=config.ddl_v_sigmoid_scale,
+        )
+        self.core=SparkBET(core_cfg)
+
+    def get_input_embeddings(self):return self.core.embed
+    def set_input_embeddings(self,value):self.core.embed=value
+    def get_output_embeddings(self):return None
+    def set_output_embeddings(self,value):
+        if value is not None:raise ValueError("SparkBET uses tied input/output embeddings")
+
+    def _cycles(self,cycles=None):
+        if cycles is None:
+            cycles=int(os.environ.get("BET_EVAL_CYCLES",self.config.refinement_cycles))
+        cycles=int(cycles)
+        if not 1<=cycles<=self.config.max_loops:
+            raise ValueError(f"refinement cycles must be in [1,{self.config.max_loops}]")
+        return cycles
+
+    def forward(
+        self,input_ids=None,attention_mask=None,labels=None,cycles=None,
+        past_key_values=None,use_cache=None,return_dict=True,**kwargs,
+    ):
+        if input_ids is None:raise ValueError("input_ids is required")
+        if past_key_values is not None:raise ValueError("SparkBET does not implement a KV cache")
+        logits=self.core(input_ids,uniform_steps(self._cycles(cycles)),attention_mask=attention_mask)
+        loss=None
+        if labels is not None:
+            shift_logits=logits[:,:-1].contiguous().float();shift_labels=labels[:,1:].contiguous()
+            loss=F.cross_entropy(shift_logits.view(-1,shift_logits.size(-1)),shift_labels.view(-1),ignore_index=-100)
+        if not return_dict:return tuple(v for v in (loss,logits) if v is not None)
+        return CausalLMOutput(loss=loss,logits=logits)
+
+    def prepare_inputs_for_generation(self,input_ids,attention_mask=None,**kwargs):
+        max_len=self.config.max_position_embeddings
+        if input_ids.shape[1]>max_len:
+            input_ids=input_ids[:,-max_len:]
+            if attention_mask is not None:attention_mask=attention_mask[:,-max_len:]
+        return {"input_ids":input_ids,"attention_mask":attention_mask,"use_cache":False}

unified/ablations/unified-v1-blt-none-s42/source/parquet_stream.py

@@ -0,0 +1,37 @@
+"""Synchronous Parquet row-group streaming with an explicit resumable row cursor.
+
+Avoids background Arrow scanner callbacks into Python file objects during shutdown.
+Only selected columns and the current batch are materialized; no local corpus cache.
+"""
+import copy
+import pyarrow.parquet as pq
+from huggingface_hub import HfFileSystem,get_token
+class ParquetStream:
+    def __init__(self,repo,filename,revision,columns,batch_size=128,opener=None):
+        self.columns,self.batch_size=columns,batch_size
+        self.cursor={'format':'parquet-row-v1','row_group':0,'row':0}
+        self.opener=opener or (lambda:HfFileSystem(token=get_token()).open(
+            f'datasets/{repo}@{revision}/{filename}','rb',block_size=65536,cache_type='bytes'))
+    def state_dict(self):return copy.deepcopy(self.cursor)
+    def load_state_dict(self,state):
+        if state.get('format')!='parquet-row-v1':
+            raise ValueError('Unsupported Parquet cursor; preserve the previous trainer for older runs')
+        self.cursor=copy.deepcopy(state)
+    def __iter__(self):
+        with self.opener() as handle:
+            reader=pq.ParquetFile(handle,pre_buffer=False,buffer_size=65536)
+            try:
+                for group in range(self.cursor['row_group'],reader.num_row_groups):
+                    skip=self.cursor['row'] if group==self.cursor['row_group'] else 0
+                    position=0
+                    for batch in reader.iter_batches(batch_size=self.batch_size,row_groups=[group],
+                                                     columns=self.columns,use_threads=False):
+                        if position+batch.num_rows<=skip:
+                            position+=batch.num_rows;continue
+                        for row in batch.to_pylist():
+                            position+=1
+                            if position<=skip:continue
+                            self.cursor={'format':'parquet-row-v1','row_group':group,'row':position}
+                            yield row
+                    self.cursor={'format':'parquet-row-v1','row_group':group+1,'row':0}
+            finally:reader.close()

unified/ablations/unified-v1-blt-none-s42/source/prepared_data.py

@@ -0,0 +1,445 @@
+"""rewrite7: immutable prepared records, epoch reuse and bounded disk prefetch.
+
+Preparation and observation are distinct. Only a successful optimizer update may
+advance the consumer cursor or enter the observation ledger. Dataset uploads
+precede consumption; the model checkpoint is the durable recovery authority.
+"""
+from collections import OrderedDict
+import bisect,copy,hashlib,json,math,os,queue,threading,time,traceback
+from pathlib import Path
+import numpy as np
+import pyarrow as pa
+import pyarrow.parquet as pq
+import torch
+from huggingface_hub import HfApi,hf_hub_download,CommitOperationAdd
+from checkpoints import atomic_json,digest
+from dataset_registry import retry
+from unified_data import CoverageSource,CoverageMixer,CodecCortex
+from cortex_source import CortexSource
+from records import PAD,validate
+
+SCHEMA=1
+
+def sha(value):return hashlib.sha256(json.dumps(value,sort_keys=True,default=str).encode()).hexdigest()
+
+def recipe_hash():
+    root=Path(__file__).parent;h=hashlib.sha256()
+    for name in ('prepared_data.py','unified_data.py','records.py','cortex_source.py','cortex_data.py','cortex_tasks.py','cortex_curriculum_v16.py','canvas_flow.py'):
+        h.update(name.encode());h.update((root/name).read_bytes())
+    return h.hexdigest()
+
+def cursor_key(cursor):
+    if cursor is None:return (-1,-1)
+    if 'row_group' in cursor:return (int(cursor['row_group']),int(cursor['row']))
+    return (0,int(cursor.get('row',cursor.get('i',0))))
+
+class EndFile(Exception):pass
+
+class OneFile(CoverageSource):
+    def raw(self):
+        if self.it is None:retry(self.open)
+        try:row=next(self.it)
+        except StopIteration:self.close();raise EndFile
+        self.s['cursor']=copy.deepcopy(self.reader.state_dict());self.s['rows']+=1;self.s['pass_rows']+=1
+        return row
+
+def _worker_init():
+    torch.set_num_threads(1)
+
+def _cortex_record(args):
+    kind,index,limit,codec=args
+    if kind=='kernel':return CortexSource({'index':index},limit).record()
+    return CodecCortex({'visual_index':index,'until_visual':0},limit,codec).record()
+
+class Generated:
+    def __init__(self,kind,start,count,limit,codec,state=None):
+        self.kind,self.start,self.count,self.limit,self.codec=kind,start,count,limit,codec
+        self.s=copy.deepcopy(state) if state else dict(index=0,cursor=None,rows=0,pending_document=None)
+    def initialize(self):pass
+    def next_record(self):
+        i=self.s['index']
+        if i>=self.count:raise EndFile
+        if self.kind=='kernel':r=CortexSource({'index':self.start+i},self.limit).record()
+        else:r=CodecCortex({'visual_index':self.start+i,'until_visual':0},self.limit,self.codec).record()
+        self.s.update(index=i+1,rows=i+1,cursor={'row':i+1})
+        return r
+    def records(self,count,pool=None):
+        n=min(count,self.count-self.s['index'])
+        if pool is None:
+            for _ in range(n):yield self.next_record()
+        else:
+            start=self.s['index'];args=[(self.kind,self.start+i,self.limit,self.codec) for i in range(start,start+n)]
+            for r in pool.map(_cortex_record,args,chunksize=8):
+                i=self.s['index']+1;self.s.update(index=i,rows=i,cursor={'row':i});yield r
+    def state_dict(self):return copy.deepcopy(self.s)
+    def close(self):pass
+
+class Archive:
+    def __init__(self,cfg,api=None):
+        self.cfg=cfg;self.repo=cfg.get('prepared_repo','appvoid/rewrite7')
+        self.root=Path(cfg['work_dir'])/'rewrite7_cache';self.root.mkdir(parents=True,exist_ok=True)
+        self.online=bool(cfg.get('upload',False));self.read_remote=bool(cfg.get('hub_resume',False))
+        self.api=api or HfApi();self.head=None;self.remote=set();self.lock=threading.RLock();self.partitions={};self.tables=OrderedDict();self.pool=None
+        self.recipe=recipe_hash();self.prepared_records=0
+        if self.online:retry(lambda:self.api.create_repo(self.repo,repo_type='dataset',private=True,exist_ok=True))
+        if self.online or self.read_remote:
+            info=retry(lambda:self.api.dataset_info(self.repo));self.head=info.sha;self.remote={s.rfilename for s in info.siblings}
+        card=self.root/'README.md'
+        card.write_text('''---
+pretty_name: rewrite7 prepared Cortex mixture
+configs:
+- config_name: default
+  data_files:
+  - split: train
+    path: data/**/*.parquet
+---
+# rewrite7
+
+Canonical prepared UTF-8 byte records from ClimbMix, Ultra-FineWeb-L3 **English QA/multi-style only**, rewrite6, and the deterministic Cortex generator. Original repository revisions, source filenames, row cursors, processing recipe and sample IDs are preserved. Upstream licenses and provenance remain applicable; this does not assign a new license to upstream content.
+
+`data/<source>/<partition>/part-*.parquet` stores each source position once per processing recipe, **not once per epoch or training run**. Index manifests store the independent preparation cursor. `observed/<run>/<lineage>/...json` logs successful sample presentations and identifies the corresponding model checkpoint SHA. Repeated epochs reuse IDs and files. Prepared rows include a lookahead buffer; they must not all be counted as observed. Model checkpoints remain authoritative for durable recovery because the model and dataset repositories cannot commit atomically together.
+
+Record fields: sample_id, source, ids (uint16), weights (uint8), prompt_len, metadata JSON, original cursor JSON, row ordinal, and last_chunk. The model's committed source/shard/record/epoch cursors provide the exact next training position. Canvas layout/image tensors are prepared in local dispatch shards; their deterministic recipe/seed/update are recorded in the observation ledger rather than duplicating corpus records for every view or epoch.
+''')
+        if self.online and 'README.md' not in self.remote:self.publish({'README.md':card},'Initialize rewrite7 prepared-data schema')
+    def fetch(self,relative):
+        p=self.root/relative
+        if p.exists():os.utime(p,None);return p
+        if relative not in self.remote:raise FileNotFoundError(relative)
+        # local_dir avoids a second full copy in the global Hugging Face cache.
+        retry(lambda:hf_hub_download(self.repo,relative,repo_type='dataset',revision=self.head,local_dir=str(self.root)))
+        return p
+    def publish(self,files,message):
+        if not self.online:return
+        with self.lock:
+            operations=[CommitOperationAdd(path_in_repo=n,path_or_fileobj=str(p)) for n,p in files.items()]
+            result=self.api.create_commit(repo_id=self.repo,repo_type='dataset',operations=operations,
+                parent_commit=self.head,commit_message=message)
+            self.head=result.oid;self.remote.update(files)
+    def cortex_pool(self):
+        workers=self.cfg.get('prepared_workers',1)
+        if workers<=1:return None
+        if self.pool is None:
+            import multiprocessing as mp
+            from concurrent.futures import ProcessPoolExecutor
+            self.pool=ProcessPoolExecutor(max_workers=workers,mp_context=mp.get_context('spawn'),initializer=_worker_init)
+        return self.pool
+    def touch_table(self,part):
+        self.tables.pop(part.prefix,None);self.tables[part.prefix]=part
+        while len(self.tables)>self.cfg.get('prepared_memory_shards',6):
+            _,old=self.tables.popitem(last=False);old.table=None;old.table_number=None
+    def evict_disk(self):
+        if not self.online:return
+        budget=int(self.cfg.get('prepared_cache_gb',40)*1024**3)
+        files=list(self.root.glob('data/**/*.parquet'))+list(self.root.glob('data/**/cursor-*.pt'))
+        total=sum(p.stat().st_size for p in files)
+        for p in sorted(files,key=lambda p:p.stat().st_mtime):
+            if total<=budget:break
+            if p.relative_to(self.root).as_posix() in self.remote:
+                total-=p.stat().st_size;p.unlink()
+    def partition(self,name,spec,filename,limit,loader=None):
+        descriptor=dict(schema=SCHEMA,recipe=self.recipe,source=name,repo=spec['repo'],revision=spec['revision'],file=filename,context=limit)
+        key=sha(descriptor)
+        if key not in self.partitions:
+            single={**spec,'files':[filename]}
+            factory=lambda state:OneFile(name,single,state,limit,loader)
+            self.partitions[key]=Partition(self,'data/'+name+'/'+key,descriptor,factory,self.cfg.get('prepared_records_per_shard',32768))
+        return self.partitions[key]
+    def generated(self,kind,index,limit,codec):
+        count=65536;start=index//count*count  # Canonical span is independent of physical shard tuning.
+        descriptor=dict(schema=SCHEMA,recipe=self.recipe,source='cortex',kind=kind,start=start,count=count,context=limit,codec=codec if kind=='visual' else 'none')
+        key=sha(descriptor)
+        if key not in self.partitions:
+            factory=lambda state:Generated(kind,start,count,limit,codec,state)
+            self.partitions[key]=Partition(self,'data/cortex/'+key,descriptor,factory,self.cfg.get('cortex_records_per_shard',512))
+        return self.partitions[key].get(index-start)
+    def observe(self,run_id,lineage,step,path,events):
+        if not events:return
+        # Immutable, checkpoint-identified presentation log. The same update's
+        # retry/resume reuses the same ledger path and never appends corpus rows.
+        relative=f'observed/{run_id}/{lineage}/step-{step:012d}.json';dest=self.root/relative
+        value=dict(schema=SCHEMA,run_id=run_id,lineage=lineage,through_step=step,
+                   model_checkpoint_sha256=digest(Path(path) if Path(path).is_file() else Path(path)/'training.pt'),recipe=self.recipe,
+                   prepared_revision=self.head,events=events,
+                   presentations=sum(len(e['sample_ids']) for e in events),
+                   unique_ids_in_segment=len({s for e in events for s in e['sample_ids']}))
+        atomic_json(dest,value);self.publish({relative:dest},f'{run_id}: observed sample references through update {step}')
+    def close(self):
+        for p in self.partitions.values():p.close()
+        if self.pool is not None:self.pool.shutdown(wait=True,cancel_futures=True);self.pool=None
+
+class Partition:
+    def __init__(self,archive,prefix,descriptor,factory,chunk):
+        self.a,self.prefix,self.descriptor,self.factory,self.chunk=archive,prefix,descriptor,factory,chunk
+        self.root=archive.root/prefix;self.root.mkdir(parents=True,exist_ok=True)
+        self.index=self.root/'index.json';self.builder=None;self.table=None;self.table_number=None
+        rel=prefix+'/index.json'
+        if rel in archive.remote:
+            # A fresh run reads the remote catalog; existing local progress may
+            # be newer after an upload failure and is republished before use.
+            remote_path=hf_hub_download(archive.repo,rel,repo_type='dataset',revision=archive.head,cache_dir=str(archive.root/'index_cache'))
+            remote=json.loads(Path(remote_path).read_text())
+            local=json.loads(self.index.read_text()) if self.index.exists() else None
+            if local is None or remote['count']>local['count']:atomic_json(self.index,remote)
+        self.meta=json.loads(self.index.read_text()) if self.index.exists() else dict(descriptor=descriptor,count=0,complete=False,shards=[])
+        if self.meta['descriptor']!=descriptor:raise RuntimeError('Prepared-data recipe mismatch')
+    def _builder(self):
+        if self.builder is None:
+            state=None
+            if self.meta['shards']:
+                tail=self.meta['shards'][-1];p=self.a.fetch(self.prefix+'/'+tail['state'])
+                if digest(p)!=tail['state_sha']:raise ValueError('Prepared source cursor checksum mismatch')
+                state=torch.load(p,map_location='cpu',weights_only=True)
+            self.builder=self.factory(state);self.builder.initialize()
+        return self.builder
+    def prepare(self):
+        if self.meta['complete']:return
+        builder=self._builder();rows=[];finished=False
+        # Read a large bounded block once. Cortex CPU work can run in spawned
+        # workers without forking an initialized CUDA runtime.
+        generated=iter(builder.records(self.chunk,self.a.cortex_pool())) if isinstance(builder,Generated) else None
+        for _ in range(self.chunk):
+            try:r=next(generated) if generated is not None else builder.next_record()
+            except (EndFile,StopIteration):finished=True;break
+            cursor=copy.deepcopy(builder.s['cursor']);offset=r.get('meta',{}).get('chunk_offset',0)
+            identity=sha(dict(partition=self.descriptor,cursor=cursor,offset=offset))
+            meta={**r.get('meta',{}),'sample_id':identity,
+                  'prepared_ref':self.prefix+'#'+str(self.meta['count']+len(rows)),
+                  'origin':self.descriptor}
+            rows.append(dict(sample_id=identity,source=r['source'],ids=r['ids'],weights=r['weights'],prompt_len=r['prompt_len'],
+                metadata=json.dumps(meta,sort_keys=True),cursor=json.dumps(cursor,sort_keys=True),row_ordinal=builder.s['rows'],last_chunk=builder.s.get('pending_document') is None))
+        if isinstance(builder,Generated) and builder.s['index']>=builder.count:finished=True
+        number=len(self.meta['shards']);files={}
+        if rows:
+            name=f'part-{number:06d}.parquet';dest=self.root/name
+            schema=pa.schema([('sample_id',pa.string()),('source',pa.string()),('ids',pa.list_(pa.uint16())),('weights',pa.list_(pa.uint8())),
+                ('prompt_len',pa.int32()),('metadata',pa.string()),('cursor',pa.string()),('row_ordinal',pa.int64()),('last_chunk',pa.bool_())])
+            temp=dest.with_suffix('.tmp');pq.write_table(pa.Table.from_pylist(rows,schema=schema),temp,compression='zstd',row_group_size=1024);os.replace(temp,dest)
+            state_name=f'cursor-{number:06d}.pt';state_path=self.root/state_name
+            temp=state_path.with_suffix('.tmp');torch.save(builder.state_dict(),temp);os.replace(temp,state_path)
+            self.meta['shards'].append(dict(file=name,start=self.meta['count'],count=len(rows),sha256=digest(dest),
+                state=state_name,state_sha=digest(state_path),first_cursor=json.loads(rows[0]['cursor']),last_cursor=json.loads(rows[-1]['cursor'])))
+            self.meta['count']+=len(rows);self.a.prepared_records+=len(rows)
+            files.update({self.prefix+'/'+name:dest,self.prefix+'/'+state_name:state_path})
+        self.meta['complete']=finished;atomic_json(self.index,self.meta);files[self.prefix+'/index.json']=self.index
+        self.a.publish(files,f'Prepare {self.descriptor["source"]}: {self.meta["count"]} unique records')
+        self.a.evict_disk()
+    def _load(self,number):
+        shard=self.meta['shards'][number]
+        if self.table_number!=number:
+            p=self.a.fetch(self.prefix+'/'+shard['file'])
+            if digest(p)!=shard['sha256']:raise ValueError('Prepared record shard checksum mismatch')
+            self.table=pq.read_table(p);self.table_number=number
+        # Recover a locally completed but not yet remotely committed shard.
+        if self.a.online and self.prefix+'/'+shard['file'] not in self.a.remote:
+            names=[shard['file'],shard['state'],'index.json']
+            self.a.publish({self.prefix+'/'+n:self.root/n for n in names},'Recover prepared shard upload')
+        self.a.touch_table(self)
+        return self.table
+    def get(self,index):
+        while index>=self.meta['count'] and not self.meta['complete']:self.prepare()
+        if index>=self.meta['count']:raise EndFile
+        number=bisect.bisect_right([s['start'] for s in self.meta['shards']],index)-1
+        table=self._load(number);row=table.slice(index-self.meta['shards'][number]['start'],1).to_pylist()[0]
+        return dict(ids=row['ids'],weights=row['weights'],source=row['source'],prompt_len=row['prompt_len'],meta={**json.loads(row['metadata']),'prepared_ref':self.prefix+'#'+str(index)},
+                    prepared_cursor=json.loads(row['cursor']),prepared_last=row['last_chunk'])
+    def after_cursor(self,cursor):
+        if cursor is None:return 0
+        i=0
+        while True:
+            try:r=self.get(i)
+            except EndFile:return self.meta['count']
+            if cursor_key(r['prepared_cursor'])>cursor_key(cursor):return i
+            i+=1
+    def row_records(self,cursor):
+        out=[];i=0;target=cursor_key(cursor)
+        while True:
+            try:r=self.get(i)
+            except EndFile:break
+            current=cursor_key(r['prepared_cursor'])
+            if current>target:break
+            if current==target:out.append(r)
+            i+=1
+        if not out:raise ValueError('Legacy cursor not found in its pinned prepared shard')
+        return out
+    def close(self):
+        if self.builder:self.builder.close()
+        self.builder=None;self.table=None;self.table_number=None
+
+class PreparedSource(CoverageSource):
+    def __init__(self,*args,archive,**kwargs):
+        super().__init__(*args,**kwargs);self.archive=archive
+    def initialize(self):
+        super().initialize();self.s.setdefault('prepared_position',None)
+    def partition(self):return self.archive.partition(self.name,self.spec,self.spec['files'][self.s['shard']],self.limit,self.loader)
+    def identify_borrowed(self,r):
+        if r.get('meta',{}).get('sample_id'):return r
+        rows=self.partition().row_records(self.s['cursor'])
+        doc=self.s.get('pending_document')
+        idx=(doc['offset']//self.limit-1) if doc else len(rows)-len(self.s['pending'])-1
+        candidate=rows[idx]
+        if candidate['ids']!=r['ids'] or candidate['weights']!=r['weights']:raise ValueError('Legacy buffered record differs from canonical prepared record')
+        return candidate
+    def next_record(self,excluded=()):
+        if self.s['pending']:
+            rows=self.partition().row_records(self.s['cursor']);r=rows[-len(self.s['pending'])];old=self.s['pending'].pop(0)
+            if r['ids']!=old['ids']:raise ValueError('Legacy pending chunk mismatch')
+            return r
+        if self.s.get('pending_document'):
+            rows=self.partition().row_records(self.s['cursor']);r=rows[self.s['pending_document']['offset']//self.limit]
+            old=super()._chunk()
+            if old['ids']!=r['ids']:raise ValueError('Legacy partial-document mismatch')
+            return r
+        visited=0
+        while True:
+            part=self.partition()
+            if self.s['prepared_position'] is None:self.s['prepared_position']=part.after_cursor(self.s['cursor'])
+            try:r=part.get(self.s['prepared_position'])
+            except EndFile:
+                visited+=1
+                if visited>len(self.spec['files']):raise RuntimeError(self.name+': empty prepared corpus')
+                self.s['shard']+=1;self.s['cursor']=None;self.s['prepared_position']=0
+                if self.s['shard']==len(self.spec['files']):self.s['shard']=0;self.s['epoch']+=1;self.s['pass_rows']=0
+                continue
+            if r['prepared_cursor']!=self.s['cursor']:
+                self.s['rows']+=1;self.s['pass_rows']+=1;self.s['accepted']+=1
+            self.s['cursor']=r['prepared_cursor'];self.s['prepared_position']+=1
+            self.s['documents_emitted']+=int(r['prepared_last'])
+            return r
+
+class PreparedCortex(CodecCortex):
+    def __init__(self,*args,archive,**kwargs):super().__init__(*args,**kwargs);self.archive=archive
+    def record(self):
+        if self.codec=='none' or self.until_visual:
+            r=self.archive.generated('kernel',self.base.s['index'],self.limit,'none')
+            self.base.s['index']+=1;self.until_visual=max(0,self.until_visual-1);return r
+        r=self.archive.generated('visual',self.visual_index,self.limit,self.codec)
+        self.visual_index+=1;self.until_visual=3;return r
+
+class PreparedMixer(CoverageMixer):
+    def __init__(self,manifest,limit=1024,state=None,loaders=None,group_weights=None,codec='none',archive=None):
+        super().__init__(manifest,limit,state,loaders,group_weights,codec);state=state or {};self.archive=archive
+        self.sources={n:PreparedSource(n,s,state.get('sources',{}).get(n),limit,(loaders or {}).get(n),archive=archive) for n,s in manifest.items()}
+        self.cortex=PreparedCortex(state.get('cortex'),limit,codec,archive=archive);self.batch_refs=[];self.batch_positions=[]
+    def initialize(self):
+        super().initialize()
+        for name,r in list(self.pending.items()):
+            if name in self.sources:self.pending[name]=self.sources[name].identify_borrowed(r)
+            elif name=='cortex' and not r.get('meta',{}).get('sample_id'):
+                # The already borrowed legacy Cortex record is the immediately
+                # preceding generator item. Match against both possible kinds.
+                candidates=[]
+                if self.cortex.base.s['index']>0:candidates.append(self.archive.generated('kernel',self.cortex.base.s['index']-1,self.limit,'none'))
+                if self.cortex.codec!='none' and self.cortex.visual_index>0:candidates.append(self.archive.generated('visual',self.cortex.visual_index-1,self.limit,self.cortex.codec))
+                matches=[x for x in candidates if x['ids']==r['ids'] and x['weights']==r['weights']]
+                if not matches:raise ValueError('Legacy Cortex buffer could not be identified')
+                self.pending[name]=matches[0]
+        return self
+    def block(self):
+        if not self.mix_queue:self._refill_mix_queue()
+        name=self.mix_queue.pop(0);tokens=[];weights=[];refs=[];positions=[]
+        while True:
+            r=self.next_for(name);validate(r,self.limit)
+            if tokens and len(tokens)+len(r['ids'])>self.limit+1:self.pending[name]=r;break
+            tokens.extend(r['ids']);weights.extend(r['weights']);refs.append(r['meta']['sample_id']);positions.append(r['meta']['prepared_ref'])
+            if len(tokens)==self.limit+1 or len(tokens)>=int(self.limit*.90):break
+        used=len(tokens);padding=self.limit+1-used;tokens.extend([PAD]*padding);weights.extend([0]*padding)
+        self.stats[name+'/blocks']+=1;self.stats[name+'/targets']+=sum(weights[1:]);self.stats[name+'/tokens']+=used-1
+        self.batch_refs.extend(refs);self.batch_positions.extend(positions)
+        return tokens,weights,name,used
+    def batch(self,count):
+        self.batch_refs=[];self.batch_positions=[];data=super().batch(count);data['sample_ids']=list(self.batch_refs);data['sample_refs']=list(self.batch_positions);return data
+
+class ShardPrefetcher:
+    """One CPU producer, bounded disk dispatch shards, exact consumer snapshots."""
+    def __init__(self,mixer,cfg,run_id,start_step,remaining=None,variant='sparkbet',codec='none',mapper=None,observations=None):
+        self.mixer,self.cfg,self.archive=mixer,cfg,mixer.archive;self.manifest=mixer.manifest
+        self.delivered=mixer.state_dict();self.initial_step=start_step;self.variant=variant;self.codec=codec;self.mapper=mapper
+        self.root=Path(cfg['work_dir'])/run_id/'dispatch';self.root.mkdir(parents=True,exist_ok=True)
+        self.depth=cfg.get('prefetch_shards',2);self.per=cfg.get('dispatch_batches_per_shard',16)
+        self.queue=queue.Queue(maxsize=self.depth);self.stop_event=threading.Event();self.ready=threading.Condition()
+        self.made=0;self.error=None;self.done=False;self.current=[];self.cursor=0;self.current_path=None;self.pending_event=None;self.events=[]
+        self.remaining=remaining;self.panel=mixer.validation()
+        self.observations=copy.deepcopy(observations or dict(presentations=0,unique_samples=0,ranges={}))
+        for old in self.root.glob('dispatch-*.pt'):old.unlink()
+        self.worker=threading.Thread(target=self._produce,name='rewrite7-shard-producer',daemon=True);self.worker.start()
+    def _produce(self):
+        produced=0;number=0
+        try:
+            while not self.stop_event.is_set() and (self.remaining is None or produced<self.remaining):
+                # Backpressure BEFORE construction bounds the extra in-flight shard.
+                while self.queue.full() and not self.stop_event.wait(.1):pass
+                if self.stop_event.is_set():break
+                entries=[]
+                for _ in range(self.per):
+                    if self.stop_event.is_set() or (self.remaining is not None and produced>=self.remaining):break
+                    data=self.mixer.batch(self.cfg['global_batch']);step=self.initial_step+produced
+                    if self.mapper:data['canvas']=self.mapper(data,step)
+                    # Torch serialization contains tensors/primitive state only.
+                    data['ids']=torch.from_numpy(data['ids']);data['weights']=torch.from_numpy(data['weights'])
+                    entries.append(dict(data=data,after=self.mixer.state_dict(),step=step));produced+=1
+                if not entries:break
+                path=self.root/f'dispatch-{self.initial_step+produced-len(entries):012d}.pt';tmp=path.with_suffix('.tmp')
+                torch.save(entries,tmp);os.replace(tmp,path)
+                while not self.stop_event.is_set():
+                    try:self.queue.put(path,timeout=.1);break
+                    except queue.Full:pass
+                number+=1
+                with self.ready:self.made=number;self.ready.notify_all()
+            with self.ready:self.done=True;self.ready.notify_all()
+        except BaseException as e:
+            self.error=(e,traceback.format_exc())
+            with self.ready:self.done=True;self.ready.notify_all()
+        finally:self.mixer.close()
+    def prewarm(self):
+        target=self.depth if self.remaining is None else min(self.depth,math.ceil(self.remaining/self.per))
+        began=time.monotonic();last=began
+        with self.ready:
+            while self.made<target and not self.done:
+                self.ready.wait(timeout=5)
+                if time.monotonic()-last>=30:print(f'Preparing rewrite7 startup buffer: {self.made}/{target} dispatch shards ready',flush=True);last=time.monotonic()
+        self._raise();self.preparation_seconds=time.monotonic()-began
+        print(f'rewrite7 startup buffer ready: {self.made} shards, {self.preparation_seconds:.1f}s',flush=True)
+    def _raise(self):
+        if self.error:raise RuntimeError('Background shard preparation failed; committed model cursor is unchanged.\n'+self.error[1]) from self.error[0]
+    def batch(self,count):
+        if count!=self.cfg['global_batch']:raise ValueError('Prefetch logical batch changed')
+        if self.pending_event is not None:raise RuntimeError('Commit the previous successful batch before advancing')
+        if self.cursor==len(self.current):
+            if self.current_path:self.current_path.unlink(missing_ok=True)
+            while True:
+                self._raise()
+                try:self.current_path=self.queue.get(timeout=.5);break
+                except queue.Empty:
+                    if self.done:raise StopIteration('Prepared finite run exhausted')
+            self.current=torch.load(self.current_path,map_location='cpu',weights_only=True);self.cursor=0
+        entry=self.current[self.cursor];self.cursor+=1;data=entry['data']
+        data={**data,'ids':data['ids'].numpy(),'weights':data['weights'].numpy()}
+        self.pending_after=entry['after'];self.pending_event=dict(update=entry['step']+1,sample_ids=data['sample_ids'],sample_refs=data['sample_refs'],
+            canvas_seed=self.cfg.get('seed'),canvas_step=entry['step'] if self.mapper else None,variant=self.variant,codec=self.codec)
+        return data
+    def commit_batch(self):
+        if self.pending_event is None:raise RuntimeError('No prepared batch to commit')
+        self.delivered=self.pending_after
+        for ref in self.pending_event['sample_refs']:
+            prefix,position=ref.rsplit('#',1);position=int(position)
+            ranges=self.observations['ranges'].setdefault(prefix,[])
+            if not any(a<=position<b for a,b in ranges):
+                ranges.append([position,position+1]);ranges.sort();merged=[]
+                for a,b in ranges:
+                    if merged and a<=merged[-1][1]:merged[-1][1]=max(merged[-1][1],b)
+                    else:merged.append([a,b])
+                self.observations['ranges'][prefix]=merged;self.observations['unique_samples']+=1
+            self.observations['presentations']+=1
+        self.events.append(self.pending_event);self.pending_event=None
+    def state_dict(self):return copy.deepcopy(self.delivered)
+    def validation(self):return self.panel
+    def coverage(self):
+        return {n:dict(epoch=s['epoch'],shard=s['shard'],shards=len(self.manifest[n]['files']),rows_read=s['rows'],
+            prepared_position=s.get('prepared_position'),documents_emitted=s.get('documents_emitted',0)) for n,s in self.delivered['sources'].items()}
+    def flush_observed(self,run_id,lineage,step,path):
+        self.archive.observe(run_id,lineage,step,path,self.events);self.events=[]
+    def close(self):
+        self.stop_event.set();self.worker.join(timeout=30)
+        if self.worker.is_alive():raise RuntimeError('Background preparation has not stopped; retry after its current network operation ends')
+        self.archive.close()

unified/ablations/unified-v1-blt-none-s42/source/records.py

@@ -0,0 +1,37 @@
+"""Direct UTF-8 bytes. Special IDs match BET, not the old Cortex tokenizer."""
+from dataclasses import dataclass
+PAD,BOS,EOS=256,257,258
+
+class Oversize(ValueError):pass
+class InvalidRecord(ValueError):pass
+
+def ids(text):return list(text.encode('utf-8'))
+def decode(tokens):return bytes(t for t in tokens if 0<=t<256).decode('utf-8',errors='replace')
+
+def record(prefix,answer,source,limit=1024,supervise_all=False,meta=None):
+    p,a=ids(prefix),ids(answer)
+    if not a:raise InvalidRecord('Empty target: '+source)
+    tokens=[BOS]+p+a+[EOS]
+    if len(tokens)>limit+1:raise Oversize(f'{source}: {len(tokens)} IDs exceeds {limit+1}; no truncation')
+    weights=[0]+([1]*len(p) if supervise_all else [0]*len(p))+[1]*(len(a)+1)
+    return dict(ids=tokens,weights=weights,source=source,prompt_len=1+len(p),meta=meta or {})
+
+def plain_chunks(text,source,limit=1024):
+    # Lossless bytes, including split UTF-8 sequences: decoder assembles the byte stream.
+    # No false EOS at chunk boundaries. One-token overlap predicts each byte once.
+    if not isinstance(text,str) or not text.strip():raise InvalidRecord('Empty/non-string text: '+source)
+    raw_bytes=text.encode('utf-8')
+    if len(raw_bytes)>1024*1024:raise Oversize('Document exceeds the 1 MiB bounded-buffer limit; rejected intact')
+    raw=[BOS]+list(raw_bytes)+[EOS];out=[]
+    for offset in range(0,len(raw)-1,limit):
+        chunk=raw[offset:offset+limit+1]
+        out.append(dict(ids=chunk,weights=[0]+[1]*(len(chunk)-1),source=source,prompt_len=1,meta={}))
+    return out
+
+def validate(r,limit=1024):
+    assert 2<=len(r['ids'])<=limit+1
+    assert len(r['ids'])==len(r['weights'])
+    assert all(type(x)==int and 0<=x<259 for x in r['ids'])
+    assert all(x in (0,1) for x in r['weights']) and sum(r['weights'][1:])>0
+    assert r['weights'][0]==0
+    return r

unified/ablations/unified-v1-blt-none-s42/source/tokenization_bet.py

@@ -0,0 +1,137 @@
+import json
+import os
+from transformers import PreTrainedTokenizer
+
+
+BYTE_PREFIX = "<0x"
+PAD_TOKEN = "<pad>"
+BOS_TOKEN = "<bos>"
+EOS_TOKEN = "<eos>"
+
+
+class BETByteTokenizer(PreTrainedTokenizer):
+    """Lossless UTF-8 byte tokenizer used by BET.
+
+    IDs:
+      0..255 -> raw byte values
+      256    -> PAD
+      257    -> BOS
+      258    -> EOS
+
+    No UNK token is required because every UTF-8 string is representable as bytes.
+    """
+
+    vocab_files_names = {"vocab_file": "byte_vocab.json"}
+    model_input_names = ["input_ids", "attention_mask"]
+
+    def __init__(
+        self,
+        vocab_file=None,
+        pad_token=PAD_TOKEN,
+        bos_token=BOS_TOKEN,
+        eos_token=EOS_TOKEN,
+        unk_token=None,
+        model_max_length=1024,
+        padding_side="left",
+        clean_up_tokenization_spaces=False,
+        **kwargs,
+    ):
+        # Transformers v5 loads values from tokenizer_config.json into this
+        # constructor. Make every value that we also forward to PythonBackend
+        # an explicit argument so it is consumed exactly once instead of being
+        # duplicated inside **kwargs.
+        self.vocab_file = vocab_file
+        kwargs.setdefault("split_special_tokens",True)
+        super().__init__(
+            pad_token=pad_token,
+            bos_token=bos_token,
+            eos_token=eos_token,
+            unk_token=unk_token,
+            model_max_length=model_max_length,
+            padding_side=padding_side,
+            clean_up_tokenization_spaces=clean_up_tokenization_spaces,
+            **kwargs,
+        )
+
+    @property
+    def vocab_size(self):
+        return 259
+
+    def get_vocab(self):
+        vocab = {f"<0x{i:02X}>": i for i in range(256)}
+        vocab[PAD_TOKEN] = 256
+        vocab[BOS_TOKEN] = 257
+        vocab[EOS_TOKEN] = 258
+        return vocab
+
+    def _tokenize(self, text, **kwargs):
+        return [f"<0x{b:02X}>" for b in text.encode("utf-8", errors="replace")]
+
+    def _convert_token_to_id(self, token):
+        if token == PAD_TOKEN:
+            return 256
+        if token == BOS_TOKEN:
+            return 257
+        if token == EOS_TOKEN:
+            return 258
+        if isinstance(token, str) and token.startswith(BYTE_PREFIX) and token.endswith(">"):
+            try:
+                value = int(token[3:-1], 16)
+                if 0 <= value <= 255:
+                    return value
+            except ValueError:
+                pass
+        # This branch should be unreachable for text encoded by this tokenizer.
+        return 0
+
+    def _convert_id_to_token(self, index):
+        index = int(index)
+        if 0 <= index <= 255:
+            return f"<0x{index:02X}>"
+        if index == 256:
+            return PAD_TOKEN
+        if index == 257:
+            return BOS_TOKEN
+        if index == 258:
+            return EOS_TOKEN
+        return "<0x00>"
+
+    def convert_tokens_to_string(self, tokens):
+        out = []
+        buf = bytearray()
+
+        def flush():
+            nonlocal buf
+            if buf:
+                out.append(bytes(buf).decode("utf-8", errors="replace"))
+                buf = bytearray()
+
+        for token in tokens:
+            idx = self._convert_token_to_id(token)
+            if isinstance(token, str) and 0 <= idx <= 255 and token.startswith(BYTE_PREFIX):
+                buf.append(idx)
+            else:
+                flush()
+                out.append(str(token))
+        flush()
+        return "".join(out)
+
+    def build_inputs_with_special_tokens(self, token_ids_0, token_ids_1=None):
+        # BET pretraining did not automatically insert BOS/EOS around ordinary text.
+        if token_ids_1 is None:
+            return list(token_ids_0)
+        return list(token_ids_0) + list(token_ids_1)
+
+    def create_token_type_ids_from_sequences(self, token_ids_0, token_ids_1=None):
+        n = len(token_ids_0) + (len(token_ids_1) if token_ids_1 is not None else 0)
+        return [0] * n
+
+    def save_vocabulary(self, save_directory, filename_prefix=None):
+        os.makedirs(save_directory, exist_ok=True)
+        name = "byte_vocab.json" if filename_prefix is None else f"{filename_prefix}-byte_vocab.json"
+        path = os.path.join(save_directory, name)
+        vocab = {f"<0x{i:02X}>": i for i in range(256)}
+        vocab.update({PAD_TOKEN: 256, BOS_TOKEN: 257, EOS_TOKEN: 258})
+        with open(path, "w", encoding="utf-8") as f:
+            json.dump(vocab, f, indent=2, sort_keys=True)
+        return (path,)

unified/ablations/unified-v1-blt-none-s42/source/trainer.py

@@ -0,0 +1,481 @@
+"""Continuous SparkBET training on one GPU or torchrun DDP, always FP16 autocast."""
+import os,json,time,math,random,signal,gc,uuid,shutil,fcntl,subprocess,sys,traceback
+from pathlib import Path
+from datetime import timedelta
+from contextlib import nullcontext
+import numpy as np
+import torch
+import torch.distributed as dist
+from torch.nn.parallel import DistributedDataParallel as DDP
+from bet_model import (
+    BETFog,BETConfig,uniform_steps,verify_architecture,
+    set_gradient_checkpointing,set_state_noise_sigma,
+)
+from records import decode
+from data_stream import Mixer
+from checkpoints import FORMAT,save,resume,legacy_import,pipeline_hash,atomic_json,Publisher
+from dataset_registry import make_manifest
+
+ROOT=Path(__file__).resolve().parent
+
+
+def lr_at(step,cfg):
+    """Warm up once, then keep learning indefinitely; no terminal update exists."""
+    warm=cfg['warmup_steps']
+    return cfg['lr']*min(1.0,(step+1)/max(1,warm))
+
+
+def loss_sum(logits,target,mask):
+    # FP32 cross entropy even when matmuls use FP16.
+    losses=torch.nn.functional.cross_entropy(
+        logits.float().flatten(0,1),target.reshape(-1),reduction='none'
+    ).view_as(mask)
+    return (losses*mask).sum()
+
+
+def scaler_new(enabled=True):
+    # FP16 + GradScaler is deliberately the one CUDA precision contract on T4,
+    # A10/A10G and RTX PRO 6000 so hardware changes do not change numerics by policy.
+    return torch.amp.GradScaler('cuda',enabled=enabled and torch.cuda.is_available(),init_scale=1024,growth_interval=2000)
+
+
+def optimizer_new(model,cfg):
+    return torch.optim.AdamW(model.parameters(),lr=cfg['lr'],betas=(.9,.95),eps=1e-8,weight_decay=.1)
+
+
+def broadcast_object(value,rank,world):
+    if world==1:return value
+    box=[value if rank==0 else None];dist.broadcast_object_list(box,src=0);return box[0]
+
+
+def coordinator_call(fn,rank,world):
+    """Broadcast rank-zero errors before peers enter the next collective."""
+    result=None;error=None
+    if rank==0:
+        try:result=fn()
+        except Exception as exc:error=f'{type(exc).__name__}: {exc}'
+    error=broadcast_object(error,rank,world)
+    if error:raise RuntimeError('Coordinator operation failed: '+error)
+    return result
+
+
+def all_true(flag,device,world):
+    value=torch.tensor(int(flag),device=device,dtype=torch.int32)
+    if world>1:dist.all_reduce(value,op=dist.ReduceOp.MIN)
+    return bool(value.item())
+
+
+def amp(device):
+    return torch.autocast(device.type,dtype=torch.float16,enabled=device.type=='cuda')
+
+
+def rng_state():
+    return dict(
+        python=random.getstate(),torch=torch.get_rng_state(),
+        cuda=torch.cuda.get_rng_state().cpu() if torch.cuda.is_available() else None,
+    )
+
+
+def restore_rng(s):
+    random.setstate(s['python']);torch.set_rng_state(s['torch'])
+    if s['cuda'] is not None and torch.cuda.is_available():torch.cuda.set_rng_state(s['cuda'])
+
+
+def auxiliary_stage(update_step,cfg):
+    """Sustained fresh-data stages: L8+Lr for base_updates*r optimizer updates.
+
+    One optimizer update fetches exactly one *new batch draw* from the mixer.
+    Batches may revisit finite data across epochs, but do not repeat a cached
+    mini-batch merely to inflate the number of stage steps. The selected depth
+    is independent of microbatch, DDP partitions, and FP16 overflow retries.
+    The origin anchors the new curriculum at an explicitly migrated checkpoint.
+    """
+    lo=int(cfg.get('aux_loop_min',1));hi=int(cfg.get('aux_loop_max',7))
+    if not (1<=lo<=hi<8):raise ValueError('auxiliary loop range must stay inside L1..L7')
+    if cfg.get('aux_loop_schedule','progressive_data_stages_v2')!='progressive_data_stages_v2':
+        raise ValueError('Unsupported auxiliary schedule: expected progressive_data_stages_v2')
+    base=int(cfg.get('aux_stage_base_updates',128))
+    if base<1:raise ValueError('aux_stage_base_updates must be positive')
+    step=int(update_step);origin=int(cfg.get('aux_curriculum_origin_step',0))
+    if step<0 or origin<0 or step<origin:
+        raise ValueError('Curriculum step must be >= its nonnegative origin')
+    cycle_updates=base*sum(range(lo,hi+1))
+    cycle,phase=divmod(step-origin,cycle_updates)
+    for depth in range(lo,hi+1):
+        stage_updates=base*depth
+        if phase<stage_updates:
+            return dict(depth=depth,cycle=cycle,cycle_updates=cycle_updates,
+                        stage_updates=stage_updates,stage_update=phase,
+                        stage_remaining=stage_updates-phase-1)
+        phase-=stage_updates
+    raise AssertionError('Invalid progressive curriculum phase')
+
+
+def auxiliary_loop(update_step,micro_index,cfg):
+    """One exact auxiliary budget per committed update; micro_index unused."""
+    return auxiliary_stage(update_step,cfg)['depth']
+
+
+def prepare(cfg, keep_datasets=False):
+    """Runs once in the coordinator before CUDA/DDP is launched."""
+    run=Path(cfg['run_dir']);run.mkdir(parents=True,exist_ok=True)
+    lock=open(run/'trainer.lock','w')
+    try:fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
+    except BlockingIOError:raise RuntimeError('A trainer is already running in this output directory')
+    publisher=Publisher(cfg['repo'],run,checkpoint_prefix=cfg.get('hub_checkpoint_prefix','checkpoints/sparkbet9m'))
+    if cfg['upload']:publisher.authenticate()
+    sha=pipeline_hash(ROOT)
+    if cfg.get('aux_loop_schedule')!='progressive_data_stages_v2':
+        raise ValueError('This notebook requires progressive_data_stages_v2 auxiliary scheduling')
+    if int(cfg.get('aux_stage_base_updates',0))<1:
+        raise ValueError('aux_stage_base_updates must be a positive integer')
+    # Allow only two precisely verified prior code fingerprints; fail closed otherwise.
+    previous=("273f8513fadd50aeb20907fce71d2352fbb12a965519e7810950caedf36bd207", "a306e734261104d381edf90097254fcb59b298deec64314f48fe85764e5159df", "78638dd200d2834060ffb746ee680feb15b95586b432d37e25606a2575b131aa") if cfg.get('allow_verified_schedule_migration',True) else ()
+    state=resume(
+        cfg['resume_roots'],cfg['repo'] if cfg['hub_resume'] else None,run,sha,
+        checkpoint_prefix=cfg.get('hub_checkpoint_prefix','checkpoints/sparkbet9m'),
+        allowed_previous_pipelines=previous,
+        migration_schedule='progressive_data_stages_v2',
+        aux_stage_base_updates=int(cfg['aux_stage_base_updates']),
+    )
+    old=None
+    if state is None:old=legacy_import(cfg['resume_roots'],cfg.get('legacy_checkpoint'))
+    if state:
+        # Optimization semantics follow the checkpoint. Hardware-only choices
+        # (microbatch and checkpointing) are re-probed on every launch.
+        for k in ['global_batch','context','lr','warmup_steps','primary_loops','aux_loop_weight','aux_loop_min','aux_loop_max','aux_loop_schedule','aux_stage_base_updates','aux_curriculum_origin_step','mix_weights']:
+            if k in state['config'] and cfg.get(k)!=state['config'][k]:
+                print(f'Resuming saved {k}={state["config"][k]}',flush=True);cfg[k]=state['config'][k]
+        # When crossing into the rewrite-plus live mix, keep the learned model,
+        # optimizer, scaler and Cortex generator position but intentionally reset
+        # language source cursors. The source set itself changed, so reusing the old
+        # manifest/cursors would silently keep training on the previous corpus.
+        migration=state.get('schedule_migrations',[])
+        just_migrated=bool(migration and migration[-1].get('new_pipeline')==sha and migration[-1].get('step')==state['step'])
+        if just_migrated and cfg.get('migrate_to_rewrite_plus_mix',True):
+            manifest=make_manifest(cfg.get('data_file_overrides'),cfg.get('disabled_sources',[]))
+            old_mix=state.get('mixer') or {}
+            mix_state=dict(cortex=old_mix.get('cortex'),rng=old_mix.get('rng'),stats=old_mix.get('stats',{}))
+            mixer=Mixer(manifest,cfg['context'],mix_state,mix_weights=cfg.get('mix_weights')).initialize()
+            state.setdefault('data_migrations',[]).append(dict(step=state['step'],kind='rewrite_plus_live_mix',language_cursors_reset=True))
+            print('Migrated to live rewrite-plus mix: model/optimizer/scaler/Cortex preserved; language streams restart under the new fair interleaver.',flush=True)
+        else:
+            manifest=state['manifest'];mixer=Mixer(manifest,cfg['context'],state['mixer'],mix_weights=cfg.get('mix_weights')).initialize()
+    else:
+        manifest=make_manifest(cfg.get('data_file_overrides'),cfg.get('disabled_sources',[]));mixer=Mixer(manifest,cfg['context'],mix_weights=cfg.get('mix_weights')).initialize()
+        state=dict(
+            format=FORMAT,step=int(old['completed_steps']) if old else 0,lineage=uuid.uuid4().hex,
+            pipeline=sha,config=cfg,manifest=manifest,mixer=mixer.state_dict(),rng_by_rank=None,
+            tokens=0,targets=0,model=old['model'] if old else None,optimizer=old['optimizer'] if old else None,
+            scaler=old['scaler'] if old else None,migration=old.get('migration') if old else None,
+        )
+    # Keep the checkpoint's stage origin and budget; never restart a stage
+    # accidentally when the notebook reruns after a checkpoint recovery.
+    auxiliary_stage(state['step'],cfg)
+    panel=mixer.validation();state['manifest']=manifest;state['mixer']=mixer.state_dict();state['config']=cfg
+    staging=run/'launch_state.pt';tmp=run/'launch_state.tmp.pt';torch.save(state,tmp);os.replace(tmp,staging)
+    torch.save(panel,run/'validation.pt');atomic_json(run/'config.json',cfg);atomic_json(run/'dataset_manifest.json',manifest)
+    print(f'Preflight: {len(manifest)} language sources + Cortex; {len(panel)} fixed validation records. Target {cfg["repo"]}.',flush=True)
+    print('CPU data validation complete. GPU memory/gradient preflight runs in the training process.',flush=True)
+    if keep_datasets:return lock,mixer
+    mixer.close();return lock
+
+
+def _preflight_candidate(model,opt,sc,device,cfg,mb,use_checkpointing):
+    set_gradient_checkpointing(use_checkpointing)
+    torch.cuda.empty_cache();torch.cuda.reset_peak_memory_stats();opt.zero_grad(set_to_none=True)
+    # Use full 1024 input length. Training slices a 1025-token record to 1024 inputs.
+    x=torch.randint(0,256,(mb,cfg['context']),device=device)
+    y=torch.roll(x,-1,1);mask=torch.ones_like(x,dtype=torch.float32)
+    paths=[(int(cfg.get('primary_loops',8)),1.0),(int(cfg.get('aux_loop_max',7)),float(cfg.get('aux_loop_weight',.20)))]
+    for loops,weight in paths:
+        with amp(device):
+            logits=model(x,uniform_steps(loops));loss=loss_sum(logits,y,mask)*weight/mask.sum()
+        sc.scale(loss).backward();del logits,loss
+    sc.unscale_(opt)
+    finite=all(p.grad is None or bool(torch.isfinite(p.grad).all()) for p in model.parameters())
+    if not finite:raise RuntimeError('Non-finite FP16 preflight gradients')
+    torch.nn.utils.clip_grad_norm_(model.parameters(),1.)
+    sc.step(opt);sc.update();torch.cuda.synchronize()
+    used=torch.cuda.max_memory_allocated()
+    del x,y,mask;opt.zero_grad(set_to_none=True)
+    return used
+
+
+def memory_preflight(device,cfg):
+    """Full-context L8+L7 probe. Prefer physical batch before checkpointing."""
+    saved=rng_state();torch.manual_seed(42)
+    model=BETFog(BETConfig()).to(device);verify_architecture(model,1)
+    memory=torch.cuda.get_device_properties(device).total_memory
+    local_batch=cfg['global_batch']//cfg['world_size']
+    requested=[local_batch,32,16,12,8,6,4,3,2,1]+list(cfg.get('microbatch_candidates') or [])
+    candidates=[]
+    for v in requested:
+        v=int(v)
+        if v>=1 and v<=local_batch and local_batch%v==0 and v not in candidates:candidates.append(v)
+    if 1 not in candidates:candidates.append(1)
+    candidates=sorted(candidates,reverse=True)
+    report=[];selected=None;selected_checkpointing=None
+    # First spend available memory on real batch. Checkpoint only if no uncheckpointed
+    # candidate has enough safety headroom.
+    for use_checkpointing in (False,True):
+        for mb in candidates:
+            opt=optimizer_new(model,cfg);sc=scaler_new()
+            try:
+                used=_preflight_candidate(model,opt,sc,device,cfg,mb,use_checkpointing)
+                headroom=used<=memory*float(cfg.get('preflight_memory_fraction',.86))
+                report.append(dict(microbatch=mb,gradient_checkpointing=use_checkpointing,peak_gib=used/2**30,headroom=headroom))
+                if headroom:
+                    selected=mb;selected_checkpointing=use_checkpointing
+                    del opt,sc;break
+            except torch.cuda.OutOfMemoryError:
+                report.append(dict(microbatch=mb,gradient_checkpointing=use_checkpointing,oom=True))
+                torch.cuda.empty_cache()
+            finally:
+                if 'opt' in locals():
+                    try:opt.zero_grad(set_to_none=True)
+                    except Exception:pass
+                    del opt
+                if 'sc' in locals():del sc
+                gc.collect();torch.cuda.empty_cache()
+        if selected is not None:break
+    del model;gc.collect();torch.cuda.empty_cache();restore_rng(saved)
+    if selected is None:
+        raise RuntimeError('No full-context L8+L7 microbatch fits with memory headroom. Context/architecture were not silently reduced.')
+    set_gradient_checkpointing(selected_checkpointing)
+    print('GPU preflight:',json.dumps(report),'selected',selected,'checkpointing',selected_checkpointing,flush=True)
+    return selected,selected_checkpointing,report
+
+
+def train_update(engine,model,opt,sc,batch,cfg,device,rank=0,world=1,update_step=0):
+    """L8 + sustained fresh-data stage auxiliary; overflow retries are schedule-stable."""
+    all_ids,all_w=batch;denom=all_w[:,1:].sum()
+    if denom<=0:raise RuntimeError('No supervised targets')
+    ids=all_ids.chunk(world,0)[rank];weights=all_w.chunk(world,0)[rank]
+    mb=cfg['microbatch'];total=ids.shape[0];aux_weight=float(cfg.get('aux_loop_weight',.20))
+    # One exact auxiliary budget per optimizer update, independent of physical
+    # microbatch partitioning. Every successful update consumes another global
+    # batch; longer Lr stages therefore truly receive more data and updates.
+    aux_loops_for_update=auxiliary_loop(update_step,0,cfg)
+    for attempt in range(9):
+        opt.zero_grad(set_to_none=True);losses=torch.zeros(2,device=device);aux_stats=torch.zeros(2,device=device)
+        for micro_index,offset in enumerate(range(0,total,mb)):
+            x=ids[offset:offset+mb,:-1];y=ids[offset:offset+mb,1:];m=weights[offset:offset+mb,1:]
+            aux_loops=aux_loops_for_update
+            for j,(loops,weight) in enumerate([(int(cfg.get('primary_loops',8)),1.0),(aux_loops,aux_weight)]):
+                sync=world>1 and offset+mb>=total and j==1
+                ctx=engine.no_sync() if world>1 and not sync else nullcontext()
+                with ctx:
+                    with amp(device):
+                        logits=engine(x,uniform_steps(loops));raw=loss_sum(logits,y,m);loss=raw*(world*weight)/denom
+                    if not all_true(torch.isfinite(raw).item(),device,world):
+                        raise RuntimeError('Non-finite forward loss; checkpoint remains at last successful update')
+                    sc.scale(loss).backward();losses[j]+=raw.detach();del logits,raw,loss
+            aux_stats[0]+=float(aux_loops);aux_stats[1]+=1
+        sc.unscale_(opt)
+        finite=all(p.grad is None or bool(torch.isfinite(p.grad).all()) for p in model.parameters());finite=all_true(finite,device,world)
+        if finite:
+            norm=torch.nn.utils.clip_grad_norm_(model.parameters(),1.);finite=all_true(torch.isfinite(norm).item(),device,world)
+        if finite:
+            sc.step(opt);sc.update()
+            if world>1:dist.all_reduce(losses);dist.all_reduce(aux_stats)
+            return (losses/denom).tolist(),float(norm),attempt,int(denom.item()),float(aux_stats[0]/aux_stats[1])
+        sc.update(new_scale=sc.get_scale()*.5)
+        if rank==0:print(f'FP16 overflow: retrying the same update at scale {sc.get_scale()}',flush=True)
+    raise RuntimeError('FP16 overflow persists after eight retries; no data/step advancement')
+
+
+@torch.inference_mode()
+def evaluate(model,panel,step,cfg,device):
+    model.eval();metrics={}
+    full=step%cfg['full_eval_every']==0
+    rows=panel if full else [panel[(step//cfg['eval_every']*8+i)%len(panel)] for i in range(min(8,len(panel)))]
+    groups={};budgets=range(1,9) if full else (3,6,8)
+    for loops in budgets:
+        for r in rows:
+            x=torch.tensor([r['ids'][:-1]],device=device);y=torch.tensor([r['ids'][1:]],device=device);w=torch.tensor([r['weights'][1:]],device=device,dtype=torch.float32)
+            with amp(device):logits=model(x,uniform_steps(loops));loss=loss_sum(logits,y,w)
+            name=r['source'];tag=f'eval/{name}/L{loops}'
+            if name=='cortex':tag+=f'/{r["meta"]["surface"]}/{r["meta"].get("split","train")}'
+            acc=((logits.argmax(-1)==y)*w).sum();bucket=groups.setdefault(tag,[0.,0.,0.]);bucket[0]+=float(loss);bucket[1]+=float(acc);bucket[2]+=float(w.sum())
+        chosen=[r for r in panel if r['source']=='cortex' and len(r['ids'])-r['prompt_len']<=128][:4]
+        correct=0
+        for r in chosen:
+            context=list(r['ids'][:r['prompt_len']]);expected=r['ids'][r['prompt_len']:];ok=True
+            for target in expected:
+                # Keep only the last context window on generated sequences.
+                current=context[-cfg['context']:]
+                with amp(device):next_id=int(model(torch.tensor([current],device=device),uniform_steps(loops))[0,-1].argmax())
+                if next_id!=target:ok=False;break
+                context.append(next_id)
+            correct+=ok
+        if chosen:metrics[f'eval/cortex/greedy_exact_L{loops}']=correct/len(chosen)
+    for tag,(loss,correct,total) in groups.items():
+        metrics[tag+'/loss']=loss/total;metrics[tag+'/byte_accuracy']=correct/total
+    model.train();return metrics
+
+
+def main(cfg):
+    rank=int(os.environ.get('RANK','0'));local=int(os.environ.get('LOCAL_RANK','0'));world=int(os.environ.get('WORLD_SIZE','1'))
+    if not torch.cuda.is_available():raise RuntimeError('Training requires a CUDA GPU. CPU mode is only used by packaged tests.')
+    torch.cuda.set_device(local);device=torch.device('cuda',local)
+    if world>1:dist.init_process_group('nccl',timeout=timedelta(minutes=90))
+    cfg['world_size']=world;run=Path(cfg['run_dir']);lock=None
+    def acquire_lock():
+        nonlocal lock
+        lock=open(run/'trainer.lock','w')
+        try:fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
+        except BlockingIOError:raise RuntimeError('Another trainer is already running here')
+    coordinator_call(acquire_lock,rank,world)
+    state=torch.load(run/'launch_state.pt',map_location='cpu',weights_only=True)
+    random.seed(42);np.random.seed(42);torch.manual_seed(42);torch.cuda.manual_seed(42)
+    torch.backends.cuda.matmul.allow_tf32=False;torch.backends.cudnn.allow_tf32=False
+    # Keep the mathematically equivalent memory-efficient kernels available. Precision
+    # remains FP16 regardless of accelerator family.
+    print(f'Rank {rank}: measuring full {cfg["context"]}-byte FP16 L8+L7 backward/Adam memory on {torch.cuda.get_device_name(device)}...',flush=True)
+    mb,use_checkpointing,report=memory_preflight(device,cfg)
+    m=torch.tensor(mb,device=device,dtype=torch.int32);cp=torch.tensor(int(use_checkpointing),device=device,dtype=torch.int32)
+    if world>1:
+        dist.all_reduce(m,op=dist.ReduceOp.MIN);dist.all_reduce(cp,op=dist.ReduceOp.MAX)
+    cfg['microbatch']=int(m.item());cfg['gradient_checkpointing']=bool(cp.item());set_gradient_checkpointing(cfg['gradient_checkpointing'])
+    set_state_noise_sigma(float(cfg.get('state_noise_sigma',0.0)))
+    model=BETFog(BETConfig()).to(device);verify_architecture(model,rank)
+    opt=optimizer_new(model,cfg);sc=scaler_new()
+    if state['model'] is not None:model.load_state_dict(state['model'],strict=True)
+    if state['optimizer'] is not None:opt.load_state_dict(state['optimizer'])
+    if state['scaler'] is not None:sc.load_state_dict(state['scaler'])
+    for values in opt.state.values():
+        for k,v in values.items():
+            if torch.is_tensor(v) and k!='step':values[k]=v.to(device)
+    if state.get('rng_by_rank'):restore_rng(state['rng_by_rank'][rank%len(state['rng_by_rank'])])
+    step=state['step'];token_count=state['tokens'];target_count=state['targets'];lineage=state['lineage'];manifest=state['manifest'];committed_mixer=state['mixer'];sha=state['pipeline'];migration=state.get('migration');schedule_migrations=state.get('schedule_migrations',[])
+    del state;gc.collect()
+    if cfg.get('compile')=='auto':do_compile=torch.cuda.get_device_capability(device)[0]>=8 and not cfg['gradient_checkpointing']
+    else:do_compile=bool(cfg.get('compile'))
+    if do_compile:
+        torch._dynamo.config.suppress_errors=True
+        try:
+            modules=[*model.prelude,*model.body,*model.coda]
+            for layer in modules:layer.compile(dynamic=True)
+            print(f'Rank {rank}: physical-block compilation enabled (FP16 unchanged).',flush=True)
+        except Exception as e:print('Compile unavailable; eager blocks retained:',type(e).__name__,flush=True)
+    engine=DDP(model,device_ids=[local],broadcast_buffers=False,gradient_as_bucket_view=True) if world>1 else model
+    mixer=Mixer(manifest,cfg['context'],committed_mixer,mix_weights=cfg.get('mix_weights')).initialize() if rank==0 else None
+    panel=torch.load(run/'validation.pt',weights_only=True) if rank==0 else None
+    from torch.utils.tensorboard import SummaryWriter
+    publisher=Publisher(cfg['repo'],run,checkpoint_prefix=cfg.get('hub_checkpoint_prefix','checkpoints/sparkbet9m'));writer=None;segment=None
+    def new_writer():
+        nonlocal writer,segment
+        segment=run/'publish_logs'/lineage;segment.mkdir(parents=True,exist_ok=True)
+        writer=SummaryWriter(str(segment),purge_step=step+1,max_queue=10,flush_secs=30,filename_suffix=f'.step-{step:012d}')
+    if rank==0:
+        new_writer();atomic_json(run/'gpu_preflight.json',dict(report=report,world=world,microbatch=cfg['microbatch'],gradient_checkpointing=cfg['gradient_checkpointing'],precision='float16'))
+    stop=False
+    def stop_handler(*_):
+        nonlocal stop;stop=True
+    signal.signal(signal.SIGINT,stop_handler);signal.signal(signal.SIGTERM,stop_handler)
+    start=time.monotonic();last_save=start;last_upload=start;last_logs=start;last_export=start;saved_step=-1
+    def checkpoint_now(upload=False):
+        nonlocal saved_step,last_save,last_upload
+        rng=rng_state();rngs=[None]*world if rank==0 else None
+        if world>1:dist.gather_object(rng,rngs,dst=0)
+        else:rngs=[rng]
+        def do_save():
+            nonlocal saved_step,last_save,last_upload
+            payload=dict(format=FORMAT,step=step,lineage=lineage,pipeline=sha,config=cfg,manifest=manifest,mixer=committed_mixer,
+                rng_by_rank=rngs,tokens=token_count,targets=target_count,model=model.state_dict(),optimizer=opt.state_dict(),scaler=sc.state_dict(),migration=migration,schedule_migrations=schedule_migrations)
+            path=save(run,payload,cfg['keep_local']);saved_step=step;last_save=time.monotonic();print(f'Saved full checkpoint: update {step:,}',flush=True)
+            if cfg.get('modal_volume_name'):
+                try:
+                    import modal;modal.Volume.from_name(cfg['modal_volume_name']).commit()
+                except Exception as e:print('Volume commit failed; Hub upload still attempted:',type(e).__name__,flush=True)
+            if upload and cfg['upload']:
+                try:publisher.checkpoint(path);last_upload=time.monotonic();print('Full-state Hub upload complete.',flush=True)
+                except Exception as e:print('Hub upload failed; local checkpoint is intact:',type(e).__name__,str(e)[:200],flush=True)
+        coordinator_call(do_save,rank,world)
+    try:
+        while True:
+            stop_flag=stop or (run/'STOP').exists();should_stop=not all_true(not stop_flag,device,world)
+            if should_stop:break
+            stage=auxiliary_stage(step,cfg)
+            if rank==0 and stage['stage_update']==0:
+                examples=int(cfg['global_batch'])*stage['stage_updates']
+                print(f'Curriculum cycle {stage["cycle"]}: L8 + L{stage["depth"]} | '
+                      f'{stage["stage_updates"]:,} fresh batch draws, '
+                      f'{examples:,} example presentations allocated.',flush=True)
+            began=time.monotonic();data=coordinator_call(lambda:mixer.batch(cfg['global_batch']),rank,world)
+            shape=broadcast_object(data['ids'].shape if rank==0 else None,rank,world)
+            x=torch.as_tensor(data['ids'],device=device) if rank==0 else torch.empty(shape,dtype=torch.long,device=device)
+            w=torch.as_tensor(data['weights'],device=device) if rank==0 else torch.empty(shape,dtype=torch.float32,device=device)
+            if world>1:dist.broadcast(x,0);dist.broadcast(w,0)
+            current_lr=lr_at(step,cfg)
+            for pg in opt.param_groups:pg['lr']=current_lr
+            losses,norm,overflows,targets,aux_mean=train_update(engine,model,opt,sc,(x,w),cfg,device,rank,world,step)
+            step+=1;token_count+=int((x[:,1:]!=256).sum());target_count+=targets
+            if rank==0:committed_mixer=mixer.state_dict()
+            del x,w,data
+            if rank==0:
+                elapsed=time.monotonic()-began;aux_weight=float(cfg.get('aux_loop_weight',.20))
+                values={'train/loss_L8':losses[0],'train/loss_aux':losses[1],'train/aux_loop_mean':aux_mean,
+                        'train/objective':losses[0]+aux_weight*losses[1],'train/lr':current_lr,'train/grad_norm':norm,
+                        'train/scaler':sc.get_scale(),'train/overflow_retries':overflows,'train/supervised_tokens':target_count,
+                        'train/nonpad_tokens':token_count,'train/update_seconds':elapsed,'train/target_bytes_per_second':targets/elapsed,
+                        'train/aux_stage_depth':stage['depth'], 'train/aux_stage_cycle':stage['cycle'],
+                        'train/aux_stage_updates_completed':stage['stage_update']+1,
+                        'train/aux_stage_updates_budget':stage['stage_updates'],
+                        'train/aux_stage_examples_presented':(stage['stage_update']+1)*int(cfg['global_batch']),
+                        'train/aux_stage_updates_remaining':stage['stage_remaining']}
+                for k,v in values.items():writer.add_scalar(k,v,step)
+                if step%cfg['log_every']==0:
+                    print(f'update {step:,} | L8 {losses[0]:.4f} | aux {losses[1]:.4f} @ L{stage["depth"]} '
+                          f'({stage["stage_update"]+1}/{stage["stage_updates"]} stage updates) | '
+                          f'lr {current_lr:.2e} | {elapsed:.1f}s | targets {target_count:,}',flush=True)
+                    for n,s in mixer.sources.items():writer.add_scalar('data/'+n+'/epochs',s.s['epoch'],step);writer.add_scalar('data/'+n+'/rejected_rows',s.s['rejected'],step)
+                    atomic_json(run/'metrics_latest.json',dict(step=step,**values))
+            if step%cfg['eval_every']==0:
+                def evaluate_and_log():
+                    metrics=evaluate(model,panel,step,cfg,device)
+                    for k,v in metrics.items():writer.add_scalar(k,v,step)
+                    atomic_json(run/'eval_latest.json',dict(step=step,**metrics));print('Validation complete:',json.dumps(metrics),flush=True)
+                coordinator_call(evaluate_and_log,rank,world)
+            actions=None
+            if rank==0:
+                now=time.monotonic();actions=dict(
+                    save=step%cfg['save_every']==0 or now-last_save>=cfg['save_seconds'],
+                    upload=step%cfg['hub_checkpoint_steps']==0,
+                    logs=now-last_logs>=cfg['hub_log_seconds'],export=now-last_export>=cfg['hub_export_seconds'])
+            actions=broadcast_object(actions,rank,world)
+            if actions['save'] or actions['upload']:checkpoint_now(actions['upload'])
+            if rank==0 and actions['logs']:
+                writer.flush();writer.close()
+                if cfg['upload']:
+                    try:publisher.logs()
+                    except Exception as e:print('Metric upload pending; retry next interval:',type(e).__name__,flush=True)
+                new_writer();last_logs=time.monotonic()
+            if rank==0 and actions['export'] and cfg['upload']:
+                try:publisher.export(model,step,manifest,ROOT);last_export=time.monotonic()
+                except Exception as e:print('Inference export pending:',type(e).__name__,str(e)[:200],flush=True)
+    except BaseException:
+        if rank==0:
+            print('Training failed. Recover from the last completed checkpoint; no partial update is published.',flush=True)
+            if writer:writer.flush();writer.close()
+            if mixer:mixer.close()
+        raise
+    else:
+        if rank==0:print('Final checkpoint and metric flush...',flush=True)
+        checkpoint_now(upload=True)
+        if rank==0:
+            writer.flush();writer.close()
+            if cfg['upload']:
+                try:publisher.logs();publisher.export(model,step,manifest,ROOT)
+                except Exception as e:print('Final publication incomplete; local state preserved:',type(e).__name__,str(e)[:200],flush=True)
+            mixer.close()
+        if world>1:dist.destroy_process_group()
+
+
+if __name__=='__main__':
+    import argparse
+    parser=argparse.ArgumentParser();parser.add_argument('--config',required=True);parser.add_argument('--prepare',action='store_true');args=parser.parse_args()
+    cfg=json.loads(Path(args.config).read_text())
+    if args.prepare:prepare(cfg)
+    else:main(cfg)

unified/ablations/unified-v1-blt-none-s42/source/unified_data.py

@@ -0,0 +1,157 @@
+"""Coverage-preserving four-family data stream. No permanent train exclusions."""
+import copy,fnmatch,hashlib,json,math
+from collections import Counter
+from functools import lru_cache
+import numpy as np
+from dataset_registry import SOURCES,retry
+from data_stream import Source,Mixer,row_key
+from cortex_source import CortexSource,episode_pair,FAMILIES
+from records import BOS,EOS,PAD,validate,record,plain_chunks,InvalidRecord
+
+GROUPS=('climbmix','ultrafine','cortex','rewrite6')
+DEFAULT_WEIGHTS=dict(climbmix=50,ultrafine=30,cortex=15,rewrite6=5)
+
+def group_of(name): return 'ultrafine' if name.startswith('ultra_') else name
+
+def complete_manifest(old=None,api=None):
+    """Retain old pinned revisions/cursors; add missing partitions at that revision."""
+    from huggingface_hub import HfApi
+    from dataclasses import asdict
+    api=api or HfApi();old={k:v for k,v in (old or {}).items() if k in SOURCES};revisions={s['repo']:s['revision'] for s in old.values()}
+    info={};result={}
+    for name,spec in SOURCES.items():
+        if spec.repo not in info:
+            info[spec.repo]=retry(lambda:api.dataset_info(spec.repo,revision=revisions.get(spec.repo)))
+        entry=info[spec.repo]
+        files=sorted(f.rfilename for f in entry.siblings if any(fnmatch.fnmatch(f.rfilename,p) for p in spec.patterns))
+        if not files:raise RuntimeError(f'{name}: no matching shards at pinned revision {entry.sha}')
+        if name in old:
+            if set(old[name]['files'])-set(files):raise RuntimeError(f'{name}: saved shards missing at pinned revision')
+            result[name]=copy.deepcopy(old[name])
+            # Preserve the old shard-index meaning. Newly discovered shards append.
+            result[name]['files']=list(old[name]['files'])+[f for f in files if f not in old[name]['files']]
+        else: result[name]={**asdict(spec),'revision':entry.sha,'files':files}
+    return result
+
+@lru_cache(maxsize=64)
+def quota_tuple(count,weights):
+    """Closest integer allocation with strict priority and every family present."""
+    if count<10:raise ValueError('global_batch must be >=10 for strict climbmix > ultrafine > cortex > rewrite6 quotas')
+    target=np.array(weights,float)*count/sum(weights);best=None;score=float('inf')
+    for rewrite in range(1,count//4+1):
+        for cortex in range(rewrite+1,count//3+1):
+            for ultra in range(cortex+1,count//2+1):
+                climb=count-rewrite-cortex-ultra
+                if climb<=ultra:continue
+                q=(climb,ultra,cortex,rewrite);loss=float(((np.array(q)-target)**2).sum())
+                if loss<score:score=loss;best=q
+    if best is None:raise ValueError('Cannot satisfy per-batch source priority')
+    return best
+
+class CoverageSource(Source):
+    def initialize(self):
+        # Legacy held-out rows rejoin the stream when their shard is visited again.
+        # Never reset a partially consumed shard merely to change the mixture.
+        self.s['initialized']=True
+        self.s.setdefault('pending_document',None)
+        self.s.setdefault('documents_emitted',0)
+    def _chunk(self):
+        doc=self.s['pending_document'];raw=doc['bytes'];offset=doc['offset'];total=len(raw)+2
+        stop=min(total,offset+self.limit+1)
+        ids=([BOS] if offset==0 else [])+list(raw[max(0,offset-1):min(len(raw),stop-1)])+([EOS] if stop==total else [])
+        r=dict(ids=ids,weights=[0]+[1]*(len(ids)-1),source=self.name,prompt_len=1,
+               meta={'row_key':doc['key'],'chunk_offset':offset,'coverage':'lossless'})
+        doc['offset']+=self.limit
+        if doc['offset']>=total-1:
+            self.s['pending_document']=None;self.s['documents_emitted']+=1
+        return validate(r,self.limit)
+    def next_record(self,excluded=()):
+        if self.s['pending']:return self.s['pending'].pop(0) # Preserve legacy buffered records.
+        if self.s.get('pending_document') is not None:return self._chunk()
+        row=self.raw();key=row_key(self.spec,row)
+        try:
+            if self.spec['kind']=='rewrite':
+                values=[row.get(k) for k in ('instruction','text','output')]
+                if not all(isinstance(v,str) for v in values):raise InvalidRecord('instruction/text/output must be strings')
+                text='\n\n'.join(values)
+            else:
+                text=row.get(self.spec['column'])
+                if not isinstance(text,str):raise InvalidRecord('missing string column '+self.spec['column'])
+            raw=text.encode('utf-8')  # Empty/whitespace rows still contribute their EOS.
+        except (ValueError,TypeError,UnicodeError) as e:
+            raise RuntimeError(f'{self.name}: invalid row at shard={self.s["shard"]}, cursor={self.s["cursor"]}; no sample was silently skipped. Fix the schema, then resume the last complete checkpoint.') from e
+        self.s['accepted']+=1
+        self.s['pending_document']={'bytes':raw,'offset':0,'key':key}
+        return self._chunk()
+
+class CodecCortex:
+    """Add deterministic graphics within Cortex's share; never skip a task index."""
+    def __init__(self,state=None,limit=1024,codec='none'):
+        state=state or {};self.codec=codec;self.limit=limit
+        self.base=CortexSource(state.get('kernel',state if 'index' in state else None),limit)
+        self.visual_index=state.get('visual_index',0);self.until_visual=state.get('until_visual',3)
+    def record(self):
+        if self.codec=='none' or self.until_visual:
+            self.until_visual=max(0,self.until_visual-1);return self.base.record()
+        from canvas_flow import draw_picture,FAMILIES as GRAPHICS,pack_braille
+        rng=np.random.default_rng(92000+self.visual_index);family=GRAPHICS[self.visual_index%len(GRAPHICS)]
+        bitmap=draw_picture(rng,family,48)
+        if self.codec=='braille':
+            cells=pack_braille(bitmap);art='\n'.join(''.join(chr(0x2800+int(v)) for v in row) for row in cells)
+        elif self.codec=='ascii':
+            density=bitmap.reshape(12,4,24,2).mean((1,3));ramp=' .:-=+*#%@'
+            art='\n'.join(''.join(ramp[int(round(v*9))] for v in row) for row in density)
+        else:raise ValueError(self.codec)
+        r=record(f'Image encoding={self.codec}; name the drawing.\n{art}\nAnswer: ',family,'cortex',self.limit,
+                 meta={'family':'visual_classification','codec':self.codec,'bitmap_seed':self.visual_index})
+        self.visual_index+=1;self.until_visual=3;return r
+    def state_dict(self):return dict(kernel=self.base.state_dict(),visual_index=self.visual_index,until_visual=self.until_visual)
+    def validation(self):
+        rows=self.base.validation()
+        if self.codec!='none':
+            other=CodecCortex({'visual_index':100000,'until_visual':0},self.limit,self.codec)
+            for _ in range(10):other.until_visual=0;rows.append(other.record())
+        return rows
+
+class CoverageMixer(Mixer):
+    def __init__(self,manifest,limit=1024,state=None,loaders=None,group_weights=None,codec='none'):
+        super().__init__(manifest,limit,state,loaders,cortex=False)
+        state=state or {}
+        self.sources={n:CoverageSource(n,s,state.get('sources',{}).get(n),limit,(loaders or {}).get(n)) for n,s in manifest.items()}
+        self.cortex=CodecCortex(state.get('cortex'),limit,codec)
+        self.group_weights=dict(group_weights or state.get('group_weights') or DEFAULT_WEIGHTS)
+        if set(self.group_weights)!=set(GROUPS) or not all(v>0 for v in self.group_weights.values()):raise ValueError('Exactly four positive source weights are required')
+        self.groups={g:[n for n in self.sources if group_of(n)==g] for g in GROUPS};self.groups['cortex']=['cortex']
+        if any(not n for n in self.groups.values()):raise ValueError('A required dataset family is missing')
+        self.group_cursor=dict(state.get('group_cursor',{}));self.codec=codec;self.excluded=set()
+        self.pending={n:r for n,r in self.pending.items() if n in manifest or n=='cortex'}
+    def initialize(self):
+        for s in self.sources.values():s.initialize()
+        # Do not consume a row merely to build a validation set.
+        return self
+    def batch(self,count):
+        quotas=quota_tuple(count,tuple(self.group_weights[g] for g in GROUPS))
+        schedule=sorted((i/q,g) for g,q in zip(GROUPS,quotas) for i in range(q))
+        blocks=[]
+        for _,group in schedule:
+            members=self.groups[group];cursor=self.group_cursor.get(group,0)
+            name=members[cursor%len(members)];self.group_cursor[group]=cursor+1
+            # One requested source per block; old buffered records stay attached to it.
+            self.mix_queue=[name];blocks.append(self.block())
+        width=min(self.limit+1,1+math.ceil(max(b[3]-1 for b in blocks)/128)*128)
+        return dict(ids=np.asarray([b[0][:width] for b in blocks],np.int64),
+                    weights=np.asarray([b[1][:width] for b in blocks],np.float32),sources=[b[2] for b in blocks])
+    def state_dict(self):
+        s=super().state_dict();s.update(group_weights=self.group_weights,group_cursor=self.group_cursor,
+                                      coverage_version=1,codec=self.codec,excluded=[])
+        return copy.deepcopy(s)
+    def validation(self):
+        # Language probes are illustrative, external to the streamed corpora;
+        # do not call them a representative held-out web benchmark.
+        probes=[record('','The river flows past a quiet village.','language_probe',self.limit,supervise_all=True),
+                record('','A careful experiment changes one condition at a time.','language_probe',self.limit,supervise_all=True)]
+        return probes+self.cortex.validation()
+    def coverage(self):
+        return {n:dict(epoch=s.s['epoch'],shard=s.s['shard'],shards=len(s.spec['files']),rows_read=s.s['rows'],
+                       documents_emitted=s.s.get('documents_emitted',0),pending_document=s.s.get('pending_document') is not None,
+                       buffered_records=len(s.s['pending'])+int(n in self.pending)) for n,s in self.sources.items()}

unified/ablations/unified-v1-blt-none-s42/source/unified_models.py

@@ -0,0 +1,91 @@
+"""Explicit experiment registry; no recurrent Ouro candidate."""
+from dataclasses import asdict
+import numpy as np
+import torch
+from torch import nn
+import torch.nn.functional as F
+from bet_model import BETFog,BETConfig,uniform_steps,verify_architecture
+from causal_arch import ModelCfg,build_model,forward_model,targets_and_mask
+
+CAUSAL=('sparkbet','ouro_1pass','evabyte','blt','byteflow_causal')
+FLOW=('canvas_full','canvas_no_flow','canvas_raster','canvas_patch8')
+VARIANTS=CAUSAL+FLOW
+
+def build(variant,context=1024,seed=42):
+    if variant not in VARIANTS:raise ValueError(f'Unsupported variant {variant}; recurrent ouro is removed')
+    torch.manual_seed(seed)
+    if variant=='sparkbet':
+        m=BETFog(BETConfig());verify_architecture(m,1)
+        return m,dict(variant=variant,config=asdict(m.c),fidelity='checkpoint-compatible SparkBET, one physical pass')
+    if variant in FLOW:
+        from canvas_flow import Config,CanvasModel
+        v=variant.removeprefix('canvas_');v='full' if v=='full' else v
+        c=Config(text_len=context+2,variant=v,patch=8 if v=='patch8' else 4)
+        m=nn.ModuleDict({'content':CanvasModel(c),'planner':CanvasModel(Config(**{**asdict(c),'dim':128,'depth':3,'heads':4}),True)})
+        return m,dict(variant=variant,config=asdict(c),fidelity='custom absorbing discrete flow; separate planner and content')
+    c=ModelCfg(dim=320,ff=864,heads=5,context=context,body_layers=6,loops=1,
+        local_dim=128,local_ff=384,local_heads=2,encoder_layers=1,decoder_layers=1,
+        local_window=128,eva_window=256,eva_chunk=16,hash_buckets=1024,
+        entropy_dim=64,entropy_ff=192,entropy_heads=1,entropy_layers=2,grad_checkpoint=False)
+    m,meta=build_model(variant,c,seed)
+    return m,dict(variant=variant,config=asdict(c),**meta)
+
+def denominators(ids,weights,variant):
+    n=8 if variant=='evabyte' else 1
+    return torch.stack([(targets_and_mask(ids,j+1)[1]*weights[:,j+1:]).sum().clamp_min(1) for j in range(n)])
+
+def causal_loss(model,variant,ids,weights,denom):
+    out={'logits':model(ids[:,:-1],uniform_steps(1))} if variant=='sparkbet' else forward_model(model,ids[:,:-1],variant)
+    heads=out.get('heads') or [out['logits']];loss=out['logits'].float().sum()*0
+    primary=None
+    for j,logits in enumerate(heads):
+        target,valid=targets_and_mask(ids,j+1);mask=valid*weights[:,j+1:]
+        ce=F.cross_entropy(logits[:,:target.shape[1]].float().flatten(0,1),target.flatten(),reduction='none').view_as(target)
+        raw=(ce*mask).sum();loss=loss+raw/denom[j]/len(heads)
+        if j==0:primary=raw.detach()/denom[0]
+    return loss,primary
+
+def canvas_batch(data,step,context,codec='braille',seed=42,variant='canvas_full'):
+    """Every real input byte is retained; only Cortex contributes generated images.
+
+    Bytes (including UTF-8 continuation bytes) have individual flow positions.
+    Display code must UTF-8-decode the reconstructed linguistic lane. Spatial
+    byte slots are not Unicode glyph cells. Do not claim browser-grade shaping.
+    """
+    from canvas_flow import make_example,capacity_map,ARRAY_KEYS,TEXT,IMAGE,RESERVED,FREE,EOS,PAD,POLICIES
+    rows=[]
+    for j,(ids,source) in enumerate(zip(data['ids'],data['sources'])):
+        index=step*len(data['ids'])+j
+        # Non-Cortex content is text-only; preserve all corpus bytes, no filtering.
+        e=make_example(index,'train',128,context+2,192,seed,encoding_override=(codec=='ascii'))
+        # Preserve packed-document separation without introducing an early flow EOS.
+        lane=[int(x) if x<256 else 10 for x in ids if 0<=x<256 or x==258]
+        raw=np.asarray(lane,np.int16)
+        visual=source=='cortex' and index%3!=0
+        if not visual:e['image'].fill(0);e['layout'].fill(TEXT)
+        else:
+            e['layout'][e['layout']==FREE]=TEXT
+        # Image-only pages are additional Cortex presentations, never replace a
+        # corpus/Cortex task's bytes. The paired task bytes stay on this page.
+        if (e['layout']==TEXT).sum()<len(raw)+1:raise RuntimeError('Canvas capacity exhausted; no bytes truncated')
+        text=np.full(context+2,PAD,np.int16);text[:len(raw)]=raw;text[len(raw)]=EOS;e['text']=text
+        if variant=='canvas_raster':e['order']=np.int64(POLICIES.index('row'))
+        e['coords'],e['regions'],e['capacity']=capacity_map(e['layout'],context+2,POLICIES[int(e['order'])])
+        geometry='; '.join(f'{o["family"]} at {o["x"]},{o["y"]}' for o in e['meta']['objects']) if visual else ''
+        prompt=f'{source}; {"mixed" if visual else "text"}; {"ASCII" if e["encoding"] else "Braille"}; order {POLICIES[int(e["order"])]}; margin {e["meta"]["margin"]}; {geometry}'.encode()[:192]
+        e['prompt'].fill(PAD);e['prompt'][:len(prompt)]=list(prompt)
+        rows.append(e)
+        if visual and index%3==1:
+            # Additional image-only page: the original task bytes above are kept.
+            import copy
+            picture=copy.deepcopy(e);picture['layout'][picture['layout']==TEXT]=FREE
+            picture['text'].fill(PAD);picture['text'][0]=EOS
+            picture['coords'].fill(0);picture['regions'].fill(0);picture['capacity'].fill(False)
+            rows.append(picture)
+    return {k:torch.from_numpy(np.stack([e[k] for e in rows])).long() for k in ARRAY_KEYS}
+
+def flow_loss(model,batch):
+    from canvas_flow import make_loss
+    content=make_loss(model['content'],batch)
+    planner=make_loss(model['planner'],batch,planner=True)
+    return content+planner,content.detach()

unified/ablations/unified-v1-blt-none-s42/source/unified_runner.py

@@ -0,0 +1,330 @@
+"""Single-GPU Run All entry point: bounded screens or continuous main training."""
+import contextlib,copy,csv,fcntl,json,math,os,random,signal,time,uuid
+from pathlib import Path
+import numpy as np
+import torch
+from checkpoints import pipeline_hash,atomic_json
+from unified_data import CoverageMixer,complete_manifest,DEFAULT_WEIGHTS
+from unified_models import build,CAUSAL,FLOW,denominators,causal_loss,canvas_batch,flow_loss
+from unified_state import Store,save,ledger,fingerprint,FORMAT,LEGACY_FORMAT
+from trainer import rng_state,restore_rng
+
+ROOT=Path(__file__).parent
+
+def lr_at(step,cfg,plan):
+    value=cfg['lr']*min(1.,(step+1)/max(1,cfg['warmup_steps']))
+    end=plan['end_step'];cool=plan['cooldown_steps']
+    if end is not None and cool and step>=end-cool:
+        # Last optimizer update has the requested floor; no off-by-one zero step.
+        fraction=(end-1-step)/max(1,cool-1)
+        value*=cfg['min_lr_ratio']+(1-cfg['min_lr_ratio'])*max(0.,fraction)
+    return value
+
+def plan_for(start,steps,cooldown):
+    if cooldown<0:raise ValueError('Cooldown must be nonnegative')
+    if steps is not None and steps<1:raise ValueError('Finite steps must be positive; use None for infinite')
+    return dict(start_step=start,end_step=None if steps is None else start+steps,
+                cooldown_steps=0 if steps is None else min(steps,int(cooldown)))
+
+def finite_optimizer(opt):
+    return all(not torch.is_tensor(v) or bool(torch.isfinite(v).all()) for s in opt.state.values() for v in s.values())
+
+def update(model,opt,data,variant,cfg,device,step,codec):
+    """Retry SAME batch + RNG: BF16/FP16 scales -> FP32, OOM -> smaller microbatch.
+
+    No optimizer mutation on a failed backward. Invalid Adam updates restore the
+    pre-update parameters and moments, then retry at a lower learning rate.
+    """
+    saved_rng=rng_state();micro=min(cfg['microbatch'],len(data['ids']))
+    ids=torch.as_tensor(data['ids'],device=device);weights=torch.as_tensor(data['weights'],device=device)
+    flow=variant in FLOW
+    mapped=(data.get('canvas') or canvas_batch(data,step,cfg['context'],codec,cfg['seed'],variant)) if flow else None
+    batch_count=len(mapped['image']) if flow else len(ids)
+    denom=None if flow else denominators(ids,weights,variant)
+    modes=[('bf16',1.)] if device.type=='cuda' and torch.cuda.is_bf16_supported() else ([('fp16',s) for s in cfg['fallback_scales']] if device.type=='cuda' else [])
+    modes += [('fp32',1.),('fp32',1.)]
+    mode_index=0;attempt=0;lr_backoff=1.
+    initial_lrs=[p['lr'] for p in opt.param_groups]
+    while mode_index<len(modes):
+        precision,scale=modes[mode_index];restore_rng(saved_rng);opt.zero_grad(set_to_none=True)
+        total=0.;primary=0.;failed=False;oom=False
+        try:
+            for start in range(0,batch_count,micro):
+                end=min(batch_count,start+micro)
+                with torch.autocast(device.type,dtype=torch.bfloat16 if precision=='bf16' else torch.float16,enabled=precision!='fp32'):
+                    if flow:
+                        batch={k:v[start:end].to(device) for k,v in mapped.items()}
+                        loss,one=flow_loss(model,batch);factor=(end-start)/batch_count;loss=loss*factor;one=one*factor
+                    else:loss,one=causal_loss(model,variant,ids[start:end],weights[start:end],denom)
+                if not bool(torch.isfinite(loss)):failed=True;break
+                (loss*scale).backward();total+=float(loss.detach());primary+=float(one)
+            if not failed:
+                for p in model.parameters():
+                    if p.grad is not None:p.grad.div_(scale)
+                failed=not all(p.grad is None or bool(torch.isfinite(p.grad).all()) for p in model.parameters())
+            if not failed:
+                norm=torch.nn.utils.clip_grad_norm_(model.parameters(),cfg['clip_grad'])
+                failed=not bool(torch.isfinite(norm))
+        except torch.cuda.OutOfMemoryError:
+            oom=True;failed=True
+        if not failed:
+            # ~9M parameter screen: CPU rollback copies are cheap relative to a
+            # corrupt long-running optimizer. Disable only with an explicit choice.
+            snapshot={k:v.detach().cpu().clone() for k,v in model.state_dict().items()}
+            moments=copy.deepcopy(opt.state_dict())
+            for group,lr in zip(opt.param_groups,initial_lrs):group['lr']=lr*lr_backoff
+            opt.step()
+            good=all(bool(torch.isfinite(p).all()) for p in model.parameters()) and finite_optimizer(opt)
+            if good:return dict(loss=primary,objective=total,grad_norm=float(norm),retry_count=attempt,precision=precision,
+                                targets=int(weights[:,1:].sum()),microbatch=micro,lr_backoff=lr_backoff)
+            model.load_state_dict(snapshot);opt.load_state_dict(moments);lr_backoff*=.25
+        attempt+=1;opt.zero_grad(set_to_none=True)
+        if oom:
+            if device.type=='cuda':torch.cuda.empty_cache()
+            if micro==1:raise RuntimeError('OOM at microbatch 1; checkpoint is intact. Use canvas_patch8 or a smaller model.')
+            micro=max(1,micro//2)
+        else:mode_index+=1
+        print(f'Retrying unchanged batch: attempt={attempt}, microbatch={micro}, next precision={modes[min(mode_index,len(modes)-1)][0]}, lr multiplier={lr_backoff}',flush=True)
+    restore_rng(saved_rng)
+    raise RuntimeError('Numerical fallbacks exhausted; no data cursor committed. Resume the last complete checkpoint.')
+
+@torch.no_grad()
+def evaluate(model,variant,panel,cfg,device,step,codec):
+    """Fixed external Cortex/probe panel; not held-out web perplexity."""
+    was=model.training;model.eval();saved=rng_state();torch.manual_seed(97031)
+    losses=[]
+    try:
+        for r in panel:
+            data=dict(ids=np.asarray([r['ids']],np.int64),weights=np.asarray([r['weights']],np.float32),sources=[r['source']])
+            if variant in FLOW:
+                batch={k:v.to(device) for k,v in canvas_batch(data,0,cfg['context'],codec,seed=97031,variant=variant).items()}
+                loss,_=flow_loss(model,batch)
+            else:
+                x=torch.as_tensor(data['ids'],device=device);w=torch.as_tensor(data['weights'],device=device)
+                _,loss=causal_loss(model,variant,x,w,denominators(x,w,variant))
+            losses.append(float(loss))
+        return sum(losses)/len(losses)
+    finally:restore_rng(saved);model.train(was)
+
+def patcher_stage(model,mixer,cfg,device,store):
+    """Train an independent entropy model; persist progress and train-only threshold.
+
+    Its extra data/compute are reported separately. Main ablations restart the
+    same data anchor afterward, so patcher preparation does not shift BLT's data.
+    """
+    stage_began=time.monotonic();previous_seconds=0.
+    path=store.run/'patcher.pt';patcher=model.patcher;patcher.requires_grad_(True)
+    opt=torch.optim.AdamW(patcher.parameters(),lr=cfg['lr'],betas=(.9,.95));start=0
+    if not path.exists() and store.cfg['hub_resume'] and store.prefix+'/patcher.pt' in store.remote_files:
+        from huggingface_hub import hf_hub_download
+        import shutil
+        f=hf_hub_download(store.repo,store.prefix+'/patcher.pt',revision=store.head)
+        shutil.copy2(f,path)
+    if path.exists():
+        state=torch.load(path,map_location='cpu',weights_only=True)
+        if state['budget']!=cfg['patcher_steps']:raise ValueError('Patcher budget differs')
+        if state.get('signature')!=store.signature or state.get('code_sha')!=pipeline_hash(ROOT):raise ValueError('Patcher configuration/code differs')
+        if state.get('manifest')!=mixer.manifest:raise ValueError('Patcher dataset revision differs; use its original data anchor')
+        patcher.load_state_dict(state['model']);opt.load_state_dict(state['optimizer']);start=state['step'];previous_seconds=state.get('patcher_seconds',0.)
+        if hasattr(mixer,'archive'):
+            from prepared_data import PreparedMixer
+            mixer=PreparedMixer(mixer.manifest,cfg['context'],state['mixer'],group_weights=cfg['group_weights'],archive=mixer.archive).initialize()
+        else:mixer=CoverageMixer(mixer.manifest,cfg['context'],state['mixer'],group_weights=cfg['group_weights']).initialize()
+        restore_rng(state['rng'])
+    if hasattr(mixer,'archive'):
+        from prepared_data import ShardPrefetcher
+        mixer=ShardPrefetcher(mixer,cfg,store.run_id+'-patcher',start,cfg['patcher_steps']-start+4,observations=state.get('prepared_observations') if path.exists() else None)
+        mixer.prewarm()
+    for i in range(start,cfg['patcher_steps']):
+        data=mixer.batch(cfg['global_batch']);update(patcher,opt,data,'entropy',cfg,device,i,'none')
+        if hasattr(mixer,'commit_batch'):mixer.commit_batch()
+        if (i+1)%cfg['save_every']==0 or i+1==cfg['patcher_steps']:
+            temp=path.with_suffix('.tmp');torch.save(dict(step=i+1,budget=cfg['patcher_steps'],patcher_seconds=previous_seconds+time.monotonic()-stage_began,signature=store.signature,code_sha=pipeline_hash(ROOT),manifest=mixer.manifest,model=patcher.state_dict(),optimizer=opt.state_dict(),mixer=mixer.state_dict(),prepared_observations=copy.deepcopy(getattr(mixer,'observations',{})),rng=rng_state()),temp);os.replace(temp,path)
+            if hasattr(mixer,'flush_observed'):mixer.flush_observed(store.run_id+'-patcher','entropy-pretraining',i+1,path)
+            if store.cfg['upload']:
+                from huggingface_hub import CommitOperationAdd
+                operations=[CommitOperationAdd(path_in_repo=store.prefix+'/patcher.pt',path_or_fileobj=str(path))]
+                anchor=store.root/(cfg['experiment_id']+'-anchor.pt')
+                if anchor.exists():operations.append(CommitOperationAdd(path_in_repo='unified/experiments/'+cfg['experiment_id']+'/anchor.pt',path_or_fileobj=str(anchor)))
+                result=store.api.create_commit(repo_id=store.repo,parent_commit=store.head,operations=operations,commit_message=f'{store.run_id}: entropy patcher {i+1}')
+                store.head=result.oid
+    patcher.eval();values=[]
+    for _ in range(4):
+        data=mixer.batch(cfg['global_batch']);x=torch.as_tensor(data['ids'],device=device)[:,:-1]
+        if hasattr(mixer,'commit_batch'):
+            mixer.pending_event['phase']='entropy_calibration';mixer.commit_batch()
+        for chunk in x.split(cfg['microbatch']):
+            ent=patcher.entropy(chunk);values.append(ent[chunk<256].cpu())
+    values=torch.cat(values)
+    if not len(values):raise RuntimeError('Empty entropy calibration')
+    patcher.threshold.fill_(float(torch.quantile(values,1-1/model.c.patch_size)))
+    patcher.requires_grad_(False)
+    if hasattr(mixer,'flush_observed'):mixer.flush_observed(store.run_id+'-calibration','entropy-calibration',cfg['patcher_steps'],path)
+    mixer.close()
+    return previous_seconds+time.monotonic()-stage_began
+
+def run_one(cfg,variant,codec,seed,main=False,anchor=None):
+    if variant in FLOW and codec not in ('braille','ascii'):raise ValueError('Canvas codec must be braille or ascii')
+    if codec not in ('none','braille','ascii'):raise ValueError('Unknown codec')
+    cfg={**cfg,'seed':seed};run_id=f'main-{variant}-{codec}' if main else f'{cfg["experiment_id"]}-{variant}-{codec}-s{seed}'
+    # Hyperparameters that change a trial's meaning are immutable on resume.
+    semantic={k:cfg[k] for k in ('context','global_batch','lr','warmup_steps','group_weights','clip_grad','patcher_steps')}
+    signature=fingerprint(dict(variant=variant,codec=codec,seed=seed,semantic=semantic,version=1))
+    store=Store(cfg,run_id,signature,main);state=store.latest();store.restore_ledger()
+    promoted=False
+    if main and state is None and cfg.get('train_from_run_id'):
+        origin=Store(cfg,cfg['train_from_run_id'],signature,False);state=origin.latest()
+        if state is None:raise RuntimeError('Requested ablation checkpoint was not found')
+        state=copy.deepcopy(state);state['initialization']='promoted:'+cfg['train_from_run_id'];state['lineage']=uuid.uuid4().hex;promoted=True
+    random.seed(seed);np.random.seed(seed);torch.manual_seed(seed)
+    device=torch.device('cuda' if torch.cuda.is_available() else 'cpu')
+    if device.type!='cuda' and not cfg.get('allow_cpu_test'):raise RuntimeError('Select a Molab RTX PRO 6000 GPU runtime before Run All')
+    if device.type=='cuda':
+        print(torch.cuda.get_device_name(),flush=True)
+        torch.backends.cuda.matmul.allow_tf32=True;torch.backends.cudnn.allow_tf32=True
+    model,meta=build(variant,cfg['context'],seed);model=model.to(device)
+    legacy=state is not None and state['format']==LEGACY_FORMAT
+    if state and not legacy and state['code_sha']!=pipeline_hash(ROOT):raise RuntimeError('Code fingerprint differs; use a new experiment ID or explicitly migrate the implementation')
+    manifest=complete_manifest(state['manifest'] if state else (anchor or {}).get('manifest'))
+    mix_state=state['mixer'] if state else (anchor or {}).get('mixer')
+    if state and state.get('prepared_dataset') and not cfg.get('prepared_shards',False):raise ValueError('This checkpoint uses prepared record cursors; keep PREPARED_SHARDS enabled')
+    archive=None
+    if cfg.get('prepared_shards',False):
+        from prepared_data import Archive,PreparedMixer,ShardPrefetcher
+        archive=Archive(cfg)
+        mixer=PreparedMixer(manifest,cfg['context'],mix_state,group_weights=cfg['group_weights'],codec=codec,archive=archive).initialize()
+    else:mixer=CoverageMixer(manifest,cfg['context'],mix_state,group_weights=cfg['group_weights'],codec=codec).initialize()
+    start=state['step'] if state else 0;step=start
+    if state and state['config']['context']!=cfg['context']:raise ValueError('Resume context differs; buffered stream records must keep their original context')
+    if state:model.load_state_dict(state['model'],strict=True)
+    initialization=state.get('initialization','migrated_attached_sparkbet') if state else 'fresh_matched_seed'
+    print(f'{run_id}: {initialization}; {sum(p.numel() for p in model.parameters()):,} parameters; one pass',flush=True)
+    patcher_seconds=state.get('patcher_seconds',0.) if state else 0.
+    if variant=='blt' and not state:
+        if cfg['patcher_steps']<1:raise ValueError('BLT requires a positive entropy-patcher training budget')
+        pm=PreparedMixer(manifest,cfg['context'],mix_state,group_weights=cfg['group_weights'],archive=archive).initialize() if archive else CoverageMixer(manifest,cfg['context'],mix_state,group_weights=cfg['group_weights']).initialize()
+        patcher_seconds=patcher_stage(model,pm,cfg,device,store)
+        # Restore the trial's initialization RNG after preparation (patcher saved).
+        random.seed(seed);torch.manual_seed(seed)
+    opt=torch.optim.AdamW([p for p in model.parameters() if p.requires_grad],lr=cfg['lr'],betas=(.9,.95),eps=1e-8,weight_decay=.1,
+                          fused=(device.type=='cuda'))
+    if state:opt.load_state_dict(state['optimizer'])
+    if state:
+        rng=state.get('rng') or (state.get('rng_by_rank') or [None])[0]
+        if rng:restore_rng(rng)
+    steps=cfg['train_steps'] if main else cfg['ablation_steps']
+    plan=state['plan'] if state and not legacy and not promoted else plan_for(start,steps,cfg['cooldown_steps'])
+    if state and not legacy and not promoted:
+        if main and ((steps is None)!=(plan['end_step'] is None)):
+            print('Explicit main schedule change: keeping weights, optimizer and all data cursors',flush=True)
+            plan=plan_for(start,steps,cfg['cooldown_steps'])
+        expected=None if steps is None else steps
+        saved_budget=None if plan['end_step'] is None else plan['end_step']-plan['start_step']
+        if expected!=saved_budget:raise ValueError('Saved run budget differs. Resume with its original budget; use a new experiment ID for a new trial.')
+    lineage=state['lineage'] if state else uuid.uuid4().hex
+    targets=state.get('targets',0) if state else 0;seconds=state.get('seconds',0.) if state else 0.
+    measured_targets=state.get('measured_targets',0) if state else 0
+    if archive and (plan['end_step'] is None or step<plan['end_step']):
+        mapper=(lambda data,i:canvas_batch(data,i,cfg['context'],codec,seed,variant)) if variant in FLOW else None
+        mixer=ShardPrefetcher(mixer,cfg,run_id,step,None if plan['end_step'] is None else plan['end_step']-step,variant,codec,mapper,
+            observations=state.get('prepared_observations') if state else None)
+        mixer.prewarm()
+    preparation_seconds=(state.get('preparation_seconds',0.) if state else 0.)+getattr(mixer,'preparation_seconds',0.)
+    code_sha=pipeline_hash(ROOT);committed=mixer.state_dict();stop=False
+    panel=mixer.validation();last={};probe='';csvpath=Path(cfg['work_dir'])/'experiments.csv'
+    def handler(*_):
+        nonlocal stop;stop=True
+    old_handlers={s:signal.getsignal(s) for s in (signal.SIGINT,signal.SIGTERM)}
+    for s in old_handlers:signal.signal(s,handler)
+    def checkpoint(status,publish=True):
+        payload=dict(format=FORMAT,step=step,lineage=lineage,signature=signature,code_sha=code_sha,config=cfg,meta=meta,
+            manifest=manifest,mixer=committed,model=model.state_dict(),optimizer=opt.state_dict(),rng=rng_state(),
+            plan=plan,targets=targets,measured_targets=measured_targets,seconds=seconds,preparation_seconds=preparation_seconds,patcher_seconds=patcher_seconds,initialization=initialization)
+        if archive:
+            payload['prepared_dataset']=dict(repo=archive.repo,revision=archive.head,recipe=archive.recipe)
+            payload['prepared_observations']=copy.deepcopy(getattr(mixer,'observations',{}))
+        path=save(store.run,payload,cfg['keep_local'])
+        if hasattr(mixer,'flush_observed'):mixer.flush_observed(run_id,lineage,step,path)
+        row=dict(run_id=run_id,mode='train' if main else 'ablation',cohort='canvas_flow' if variant in FLOW else ('codec' if codec!='none' else 'causal'),
+            variant=variant,codec=codec,seed=seed,step=step,start_step=plan['start_step'],status=status,initialization=initialization,
+            params=sum(p.numel() for p in model.parameters()),context=cfg['context'],loss=last.get('loss',''),probe_loss=probe,
+            targets=targets,seconds=seconds,targets_per_second=measured_targets/max(1,seconds),grad_norm=last.get('grad_norm',''),retry_count=last.get('retry_count',0),
+            precision=last.get('precision',''),patcher_steps=cfg['patcher_steps'] if variant=='blt' else 0,
+            dataset_sha=fingerprint(manifest),code_sha=code_sha,coverage_json=json.dumps(mixer.coverage(),sort_keys=True))
+        observations=getattr(mixer,'observations',{})
+        row.update(sample_presentations=observations.get('presentations',''),unique_samples_seen=observations.get('unique_samples',''),preparation_seconds=preparation_seconds,patcher_seconds=patcher_seconds)
+        ledger(csvpath,row)
+        atomic_json(store.run/'coverage.json',mixer.coverage())
+        progress=dict(step=step,prepared_dataset=payload.get('prepared_dataset'),observations=observations,coverage=mixer.coverage())
+        if publish:store.publish(path,model,manifest,{**meta,'data_progress':progress})
+    try:
+        if not state or promoted:checkpoint('initialized')
+        # Completed finite trials are immutable; Run All proceeds to the next one.
+        if plan['end_step'] is not None and step>=plan['end_step']:
+            print(f'{run_id}: already complete',flush=True);return True
+        while not stop and not (Path(cfg['work_dir'])/'STOP').exists():
+            if plan['end_step'] is not None and step>=plan['end_step']:break
+            began=time.monotonic();data=mixer.batch(cfg['global_batch'])
+            for g in opt.param_groups:g['lr']=lr_at(step,cfg,plan)
+            last=update(model,opt,data,variant,cfg,device,step,codec)
+            cfg['microbatch']=last['microbatch'];step+=1;targets+=last['targets'];measured_targets+=last['targets'];seconds+=time.monotonic()-began
+            if hasattr(mixer,'commit_batch'):mixer.commit_batch()
+            committed=mixer.state_dict()
+            if step%cfg['log_every']==0:print(f'{run_id}: update={step} loss={last["loss"]:.4f} targets/s={measured_targets/max(1,seconds):.0f}',flush=True)
+            if step%cfg['eval_every']==0:probe=evaluate(model,variant,panel,cfg,device,step,codec)
+            if step%cfg['save_every']==0:checkpoint('running',publish=step%cfg['hub_every']==0)
+            elif step%cfg['hub_every']==0:checkpoint('running')
+        complete=plan['end_step'] is not None and step>=plan['end_step']
+        probe=evaluate(model,variant,panel,cfg,device,step,codec)
+        checkpoint('complete' if complete else 'paused')
+        return complete
+    except BaseException:
+        print('Run failed: no partial update is published. Resume the last complete local/Hub checkpoint.',flush=True)
+        raise
+    finally:
+        mixer.close()
+        for s,h in old_handlers.items():signal.signal(s,h)
+        del model,opt
+        if device.type=='cuda':torch.cuda.empty_cache()
+
+def main(cfg):
+    if cfg['mode'] not in ('train','ablation'):raise ValueError('MODE must be train or ablation')
+    for key in ('context','global_batch','microbatch','ablation_steps','patcher_steps','hub_every','save_every','eval_every','log_every'):
+        if not isinstance(cfg[key],int) or cfg[key]<1:raise ValueError(key+' must be a positive integer')
+    if cfg.get('prepared_shards'):
+        for key in ('prepared_records_per_shard','cortex_records_per_shard','prefetch_shards','dispatch_batches_per_shard','prepared_memory_shards','prepared_workers'):
+            if cfg.get(key,1)<1:raise ValueError(key+' must be positive')
+    if cfg['global_batch']<10:raise ValueError('GLOBAL_BATCH must be at least 10')
+    if not cfg['seeds'] or len(set(cfg['seeds']))!=len(cfg['seeds']):raise ValueError('Supply distinct seeds')
+    if cfg['lr']<=0 or cfg['clip_grad']<=0 or not 0<=cfg['min_lr_ratio']<=1:raise ValueError('Invalid optimizer settings')
+    root=Path(cfg['work_dir']);root.mkdir(parents=True,exist_ok=True)
+    lock=(root/'runner.lock').open('w')
+    try:fcntl.flock(lock,fcntl.LOCK_EX|fcntl.LOCK_NB)
+    except BlockingIOError:raise RuntimeError('This experiment already has an active trainer')
+    if (root/'STOP').exists():
+        print('Removing the previous clean-stop request; Run All explicitly resumes training.',flush=True);(root/'STOP').unlink()
+    if cfg['mode']=='train':run_one(cfg,cfg['train_variant'],cfg['train_codec'],cfg['seeds'][0],True)
+    else:
+        # One immutable starting stream for all fresh architecture/codec trials.
+        anchor_path=root/(cfg['experiment_id']+'-anchor.pt')
+        if anchor_path.exists():anchor=torch.load(anchor_path,map_location='cpu',weights_only=True)
+        else:
+            from huggingface_hub import HfApi,hf_hub_download
+            from dataset_registry import retry
+            remote_anchor='unified/experiments/'+cfg['experiment_id']+'/anchor.pt'
+            info=retry(lambda:HfApi().model_info(cfg['repo'])) if cfg['hub_resume'] else None
+            if info and remote_anchor in [f.rfilename for f in info.siblings]:
+                import shutil
+                f=retry(lambda:hf_hub_download(cfg['repo'],remote_anchor,revision=info.sha))
+                shutil.copy2(f,anchor_path)
+            manifest=complete_manifest();m=CoverageMixer(manifest,cfg['context'],group_weights=cfg['group_weights']).initialize()
+            anchor=torch.load(anchor_path,map_location='cpu',weights_only=True) if anchor_path.exists() else dict(manifest=manifest,mixer=m.state_dict());m.close();tmp=anchor_path.with_suffix('.tmp');torch.save(anchor,tmp);os.replace(tmp,anchor_path)
+        for seed in cfg['seeds']:
+            for variant,codec in cfg['trials']:
+                if not run_one(cfg,variant,codec,seed,anchor=anchor):return
+    lock.close()
+
+if __name__=='__main__':
+    import argparse
+    parser=argparse.ArgumentParser();parser.add_argument('--config',required=True);args=parser.parse_args()
+    main(json.loads(Path(args.config).read_text()))

unified/ablations/unified-v1-blt-none-s42/source/unified_state.py

@@ -0,0 +1,153 @@
+"""Verified resume and single-commit state + inference + experiment ledger."""
+import csv,hashlib,json,os,shutil,time,uuid,copy
+from pathlib import Path
+import torch
+from huggingface_hub import HfApi,hf_hub_download,CommitOperationAdd,CommitOperationDelete
+from checkpoints import digest,atomic_json,verified_load,FORMAT as LEGACY_FORMAT,Publisher
+from dataset_registry import retry
+
+FORMAT='cortex-unified-v1'
+LEGACY_PIPELINE='d99360d8f48a2cc017360a4d48b462b9fbee20902d3b58bcc936f054c46fc07c'
+FIELDS=['run_id','mode','cohort','variant','codec','seed','step','start_step','status','initialization',
+        'sample_presentations','unique_samples_seen','preparation_seconds','patcher_seconds','params','context','loss','probe_loss','targets','seconds','targets_per_second','grad_norm',
+        'retry_count','precision','patcher_steps','dataset_sha','code_sha','coverage_json']
+
+def fingerprint(value):return hashlib.sha256(json.dumps(value,sort_keys=True,default=str).encode()).hexdigest()
+
+def ledger(path,row):
+    """One ledger; idempotent (run_id, step) upsert across retries/resumes."""
+    path=Path(path);path.parent.mkdir(parents=True,exist_ok=True)
+    rows=list(csv.DictReader(path.open())) if path.exists() else []
+    key=(str(row['run_id']),str(row['step']))
+    rows=[r for r in rows if (r['run_id'],r['step'])!=key];rows.append({k:row.get(k,'') for k in FIELDS})
+    tmp=path.with_suffix('.tmp')
+    with tmp.open('w',newline='') as f:
+        w=csv.DictWriter(f,fieldnames=FIELDS);w.writeheader();w.writerows(rows);f.flush();os.fsync(f.fileno())
+    os.replace(tmp,path)
+
+def save(run,state,keep=3):
+    root=Path(run)/'checkpoints';root.mkdir(parents=True,exist_ok=True)
+    dest=root/f'checkpoint-{state["step"]:012d}'
+    temp=root/('.partial-'+uuid.uuid4().hex);temp.mkdir()
+    try:
+        with (temp/'training.pt').open('wb') as f:torch.save(state,f);f.flush();os.fsync(f.fileno())
+        meta=dict(format=FORMAT,step=state['step'],lineage=state['lineage'],signature=state['signature'],
+                  sha256=digest(temp/'training.pt'),bytes=(temp/'training.pt').stat().st_size,saved_at=time.time())
+        atomic_json(temp/'metadata.json',meta);(temp/'COMPLETE').write_text('complete\n')
+        if dest.exists():
+            # Completed update is immutable; status/ledger can still be republished.
+            shutil.rmtree(temp);return dest
+        os.replace(temp,dest)
+        for p in sorted(root.glob('checkpoint-*'))[:-keep]:shutil.rmtree(p)
+        return dest
+    except BaseException:
+        shutil.rmtree(temp,ignore_errors=True);raise
+
+def load(path):
+    path=Path(path);meta=json.loads((path/'metadata.json').read_text())
+    if meta['format']==LEGACY_FORMAT:return verified_load(path)
+    f=path/'training.pt'
+    if meta['format']!=FORMAT or not (path/'COMPLETE').exists():raise ValueError('Unrecognized/incomplete checkpoint')
+    if f.stat().st_size!=meta['bytes'] or digest(f)!=meta['sha256']:raise ValueError('Checkpoint checksum mismatch')
+    s=torch.load(f,map_location='cpu',weights_only=True)
+    if any(s[k]!=meta[k] for k in ('format','step','lineage','signature')):raise ValueError('State/metadata mismatch')
+    return s
+
+class Store:
+    def __init__(self,cfg,run_id,signature,main=False,api=None):
+        self.cfg=cfg;self.run_id=run_id;self.signature=signature;self.main=main
+        self.repo=cfg['repo'];self.root=Path(cfg['work_dir']);self.run=self.root/run_id
+        self.run.mkdir(parents=True,exist_ok=True);self.api=api or HfApi()
+        self.prefix='unified/main/'+run_id if main else 'unified/ablations/'+run_id
+        self.head=None;self.remote_files=[]
+    def inventory(self):
+        if not self.cfg['hub_resume'] and not self.cfg['upload']:return
+        # Authentication/repository errors must fail, never silently start fresh.
+        info=retry(lambda:self.api.model_info(self.repo));self.head=info.sha
+        self.remote_files=[f.rfilename for f in info.siblings]
+    def latest(self):
+        self.inventory();items=[]
+        for p in (self.run/'checkpoints').glob('checkpoint-*/metadata.json'):
+            if (p.parent/'COMPLETE').exists():items.append((json.loads(p.read_text()),p.parent,None))
+        if self.cfg['hub_resume']:
+            for f in self.remote_files:
+                eligible=f.startswith(self.prefix+'/checkpoint-') or (self.main and self.cfg.get('train_variant','sparkbet')=='sparkbet' and not self.cfg.get('train_from_run_id') and f.startswith('checkpoints/sparkbet9m/checkpoint-'))
+                if eligible and f.endswith('/metadata.json') and f.rsplit('/',1)[0]+'/COMPLETE' in self.remote_files:
+                    local=retry(lambda f=f:hf_hub_download(self.repo,f,revision=self.head,cache_dir=str(self.root/'hub_cache')))
+                    items.append((json.loads(Path(local).read_text()),None,f.rsplit('/',1)[0]))
+        unified=[x for x in items if x[0]['format']==FORMAT]
+        # The unified lineage succeeds the old pipeline. Never prefer a legacy
+        # writer's higher step over a deliberately migrated unified lineage.
+        if unified:items=unified
+        if len({x[0]['lineage'] for x in items})>1:raise RuntimeError('Conflicting lineages; select a distinct work directory/repository')
+        for meta,path,remote in sorted(items,key=lambda x:(x[0]['step'],x[0]['saved_at']),reverse=True):
+            if meta['format']==FORMAT and meta['signature']!=self.signature:raise RuntimeError('Saved experiment configuration differs; change EXPERIMENT_ID for a new trial')
+            try:
+                if remote:
+                    path=self.run/'download'/remote.replace('/','_');path.mkdir(parents=True,exist_ok=True)
+                    for n in ('training.pt','metadata.json','COMPLETE'):
+                        f=retry(lambda n=n:hf_hub_download(self.repo,remote+'/'+n,revision=self.head,cache_dir=str(self.root/'hub_cache')))
+                        shutil.copy2(f,path/n)
+                s=load(path)
+            except (ValueError,EOFError,OSError) as e:
+                print(f'Invalid checkpoint at {meta["step"]}: {type(e).__name__}; trying older complete state',flush=True);continue
+            if s['format']==LEGACY_FORMAT:
+                if not self.main or s['pipeline']!=LEGACY_PIPELINE:raise RuntimeError('Legacy pipeline was not the audited attachment; refusing implicit migration')
+                print('Migrating attached SparkBET weights, optimizer and data cursors. New objective: one pass, no auxiliary recurrence. Missing dataset partitions append at the saved revision.',flush=True)
+            print(f'Resuming {self.run_id}, update {s["step"]}, {"Hub" if remote else "local"}',flush=True)
+            return s
+        if items:raise RuntimeError('No checkpoint passed verification; fresh initialization refused')
+        if self.main and self.cfg['hub_resume'] and 'model.safetensors' in self.remote_files and not self.cfg.get('train_from_run_id'):
+            raise RuntimeError('Root weights exist but no compatible full-state checkpoint was found. Select TRAIN_FROM_RUN_ID to promote a compatible ablation; a silent fresh main run is refused.')
+        return None
+    def restore_ledger(self):
+        path=self.root/'experiments.csv'
+        if self.cfg['hub_resume'] and 'experiments.csv' in self.remote_files:
+            f=retry(lambda:hf_hub_download(self.repo,'experiments.csv',revision=self.head,cache_dir=str(self.root/'hub_cache')))
+            remote=list(csv.DictReader(open(f)));local=list(csv.DictReader(path.open())) if path.exists() else []
+            for row in remote+local:ledger(path,row)
+    def publish(self,path,model,manifest,meta):
+        if not self.cfg['upload']:return
+        # Fetch a head before building an optimistic atomic commit. A concurrent
+        # writer causes a clear conflict; never overwrite its root or ledger.
+        previous_head=self.head
+        self.inventory()
+        if previous_head is not None and self.head!=previous_head:
+            self.head=previous_head
+            raise RuntimeError('Hub changed since this run last read/published it; resume to merge state instead of overwriting another writer')
+        dest=self.run/'export';dest.mkdir(exist_ok=True)
+        if self.main and meta['variant']=='sparkbet':
+            export=Publisher(self.repo,self.run)
+            class LocalOnly:
+                def upload_folder(self,**kwargs):pass
+            export.api=LocalOnly();export.export(model,int(Path(path).name.split('-')[-1]),manifest,Path(__file__).parent)
+            config=json.loads((dest/'config.json').read_text());config.update(refinement_cycles=1,precision='bf16 autocast where supported; fp32 master')
+            atomic_json(dest/'config.json',config)
+            shutil.copy2(Path(__file__).parent/'UNIFIED_README.md',dest/'README.md')
+        else:
+            from safetensors.torch import save_file
+            save_file({k:v.detach().cpu().contiguous().clone() for k,v in model.state_dict().items()},str(dest/'model.safetensors'))
+            atomic_json(dest/'experiment_config.json',meta)
+            if self.main:
+                atomic_json(dest/'config.json',dict(architecture=meta['variant'],custom_loader='unified_models.build',config=meta['config'],step=int(Path(path).name.split('-')[-1])))
+                (dest/'README.md').write_text('# Unified canvas / byte experiment\n\nCustom scaled architecture. Fetch the matching `unified/main` source modules and use `unified_models.build(variant, context, seed)`, then load `model.safetensors` strictly. This export does not advertise Transformers AutoModel compatibility. Full optimizer/data/RNG resume is in the same commit.\n')
+        if 'data_progress' in meta:atomic_json(dest/'data_progress.json',meta['data_progress'])
+        operations=[];remote_prefix=self.prefix+'/'+Path(path).name
+        for p in Path(path).iterdir():
+            if p.is_file():operations.append(CommitOperationAdd(path_in_repo=remote_prefix+'/'+p.name,path_or_fileobj=str(p)))
+        for p in dest.iterdir():
+            if p.is_file():operations.append(CommitOperationAdd(path_in_repo=p.name if self.main else self.prefix+'/inference/'+p.name,path_or_fileobj=str(p)))
+        # Save the exact implementation beside every exported configuration.
+        for p in Path(__file__).parent.glob('*.py'):
+            operations.append(CommitOperationAdd(path_in_repo=self.prefix+'/source/'+p.name,path_or_fileobj=str(p)))
+        anchor=self.root/(self.cfg['experiment_id']+'-anchor.pt')
+        if anchor.exists() and not self.main:operations.append(CommitOperationAdd(path_in_repo='unified/experiments/'+self.cfg['experiment_id']+'/anchor.pt',path_or_fileobj=str(anchor)))
+        csvpath=self.root/'experiments.csv'
+        if csvpath.exists():operations.append(CommitOperationAdd(path_in_repo='experiments.csv',path_or_fileobj=str(csvpath)))
+        old=sorted({f.rsplit('/',1)[0] for f in self.remote_files if f.startswith(self.prefix+'/checkpoint-') and f.endswith('/COMPLETE')})
+        keep=set(sorted(set(old+[remote_prefix]))[-self.cfg.get('keep_hub',2):])
+        for f in self.remote_files:
+            if f.startswith(self.prefix+'/checkpoint-') and f.rsplit('/',1)[0] not in keep:operations.append(CommitOperationDelete(path_in_repo=f))
+        result=self.api.create_commit(repo_id=self.repo,operations=operations,parent_commit=self.head,
+                 commit_message=f'{self.run_id}: state and inference synchronized at {Path(path).name}')
+        self.head=result.oid