| { |
| "task": "y = Mv, m=3, entries 0..7, each y_i nine LSB-first bits; A answer-only, T trace of the row accumulator after every shifted partial product", |
| "hf_config": { |
| "vocab_size": 15, |
| "hidden_size": 128, |
| "num_hidden_layers": 5, |
| "num_attention_heads": 4, |
| "intermediate_size": 640, |
| "rotary_pct": 0.0, |
| "use_parallel_residual": false, |
| "max_position_embeddings": 512, |
| "bos_token_id": 0, |
| "eos_token_id": 0, |
| "tie_word_embeddings": false, |
| "hidden_act": "relu", |
| "no_layernorm": true, |
| "initializer_range": 0.02, |
| "attention_dropout": 0.0, |
| "hidden_dropout": 0.0 |
| }, |
| "tokens": "BOS 0, unk 1, d0..d7 2..9, bits 0/1 10/11, ; , = 12..14", |
| "data": "fresh inputs every batch; saved disjoint 2,000-input test set per seed", |
| "train": "TRAIN: all weights from the lego.arch init under manual_seed(seed); 5L x 4H x 32, d_model 128, ReLU MLP 640, NoPE, no LayerNorm", |
| "optimiser": "AdamW lr 1e-3 wd .1 betas (.9,.98) warmup 1000 cosine clip 1, plateau 5000; T r1 and final (10 seeds) 30k batch 256; A r3 120k planned (stop 93k) batch 128; TF32, evals fp32" |
| } |
