File size: 1,074 Bytes

b9c5bae

{
 "task": "y = Mv, m=3, entries 0..7, each y_i nine LSB-first bits; A answer-only, T trace of the row accumulator after every shifted partial product",
 "hf_config": {
  "vocab_size": 15,
  "hidden_size": 128,
  "num_hidden_layers": 5,
  "num_attention_heads": 4,
  "intermediate_size": 640,
  "rotary_pct": 0.0,
  "use_parallel_residual": false,
  "max_position_embeddings": 512,
  "bos_token_id": 0,
  "eos_token_id": 0,
  "tie_word_embeddings": false,
  "hidden_act": "relu",
  "no_layernorm": true,
  "initializer_range": 0.02,
  "attention_dropout": 0.0,
  "hidden_dropout": 0.0
 },
 "tokens": "BOS 0, unk 1, d0..d7 2..9, bits 0/1 10/11, ; , = 12..14",
 "data": "fresh inputs every batch; saved disjoint 2,000-input test set per seed",
 "train": "TRAIN: all weights from the lego.arch init under manual_seed(seed); 5L x 4H x 32, d_model 128, ReLU MLP 640, NoPE, no LayerNorm",
 "optimiser": "AdamW lr 1e-3 wd .1 betas (.9,.98) warmup 1000 cosine clip 1, plateau 5000; T r1 and final (10 seeds) 30k batch 256; A r3 120k planned (stop 93k) batch 128; TF32, evals fp32"
}