Hugging Face's logo

itzPotato
/
arithmetic-relu-2layer-seed2

PyTorch
Safetensors
arithmetic
interpretability
bilinear
transcoder
Model card Files
xet
 Community
arithmetic-relu-2layer-seed2
File size: 831 Bytes
095edfb
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
{
  "d_model": 32,
  "d_mlp": 64,
  "n_heads": 4,
  "d_head": 8,
  "n_layers": 2,
  "mlp_type": "relu",
  "d_vocab": 13,
  "n_ctx": 16,
  "bias": false,
  "norm": false,
  "tie_unembed": false,
  "model_seed": 2,
  "data_config": {
    "n_train": 5000000,
    "n_val": 10000,
    "n_test": 10000,
    "seed": 1234,
    "balanced_ops": true
  },
  "vocab": {
    "plus": 10,
    "minus": 11,
    "equals": 12,
    "n_operand_digits": 4,
    "n_answer_digits": 5
  },
  "train_config": {
    "lr": 0.02,
    "betas": [
      0.9,
      0.98
    ],
    "weight_decay": 0.01,
    "batch_size": 1024,
    "warmup_steps": 200,
    "lr_schedule": "cosine",
    "lr_min_frac": 0.1,
    "eval_every_steps": 200,
    "seed": 2,
    "max_steps": null,
    "grad_clip": 1.0
  },
  "planned_repo_id": "itzPotato/arithmetic-relu-2layer-seed2"
}