File size: 549 Bytes
b8131eb
 
 
 
05def1c
b8131eb
 
 
 
 
 
 
 
 
 
 
 
 
 
05def1c
 
b8131eb
05def1c
 
 
b8131eb
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
{
  "class_name": "EFCProbe",
  "init_args": {
    "num_layers": 6,
    "d_model": 7168,
    "feature_mode": "combined",
    "center": true,
    "d_hidden": 64,
    "d_probe": 256,
    "normalization": "rms",
    "normalization_eps": 1e-06,
    "shrinkage": "fixed",
    "shrinkage_alpha": 0.1,
    "spectral_transform": "eigh",
    "newton_schulz_iterations": 10,
    "jitter": 1e-05,
    "feature_chunk_size": 4,
    "normalize_input": "none",
    "layer_keys": [
      "28",
      "39",
      "51",
      "62",
      "74",
      "84"
    ]
  }
}