File size: 1,155 Bytes
f12d393
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
{
  "steps": 40,
  "unique_training_texts": 372,
  "losses": [
    14.162500381469727,
    12.354166984558105,
    12.982142448425293,
    12.465909004211426,
    12.401785850524902,
    11.859375,
    12.511363983154297,
    12.65625,
    10.3125,
    12.0,
    11.401785850524902,
    11.10546875,
    13.330357551574707,
    10.870535850524902,
    10.474431991577148,
    11.971590995788574,
    8.4921875,
    11.274999618530273,
    10.290178298950195,
    8.956250190734863,
    10.924107551574707,
    10.133928298950195,
    10.276785850524902,
    9.416666984558105,
    11.010416984558105,
    10.364583015441895,
    8.723958015441895,
    9.328125,
    10.334821701049805,
    9.627604484558105,
    9.754687309265137,
    5.798295497894287,
    10.287946701049805,
    11.689583778381348,
    10.24609375,
    9.551562309265137,
    7.107954502105713,
    9.4765625,
    7.7414774894714355,
    12.395833015441895
  ],
  "encoder_parameter_delta_l2": 0.06532496213912964,
  "seconds": 5.581527638000011,
  "newly_initialized_mlm_head": true,
  "limitation": "Small synthetic corpus and new MLM head; not large-scale continued pretraining"
}