Hugging Face's logo

AdaptiveChunking
/
jitter_s1

PyTorch
Safetensors
dynamic-chunking
tokenization
hnet
byte-level
beetle-hnet
legacy-trainer
Model card Files
xet
 Community
jitter_s1
File size: 54,462 Bytes
ddcb837
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
{"event": "start", "run": "jitter_s1", "condition": "A", "params": 22516992, "langs": ["en", "de", "vi", "tr", "fi", "ru", "el", "hi", "ta", "th", "zh", "my"], "premium": {"en": 1.0, "de": 1.184, "vi": 1.39, "tr": 1.12, "fi": 1.104, "ru": 1.971, "el": 2.174, "hi": 2.552, "ta": 3.192, "th": 2.762, "zh": 0.915, "my": 3.524}, "schedule": {"tau1": 0.4, "tau2": 0.7, "p1_objective": "global", "p2_objective": "global", "p3_objective": "global", "freeze_in_p3": false, "freeze_scope": "router"}, "corpus": "/local/scratch/sas245/beetle-hnet/scratch/data/tier1_corpus", "premium_kind": "flores_per_sent", "doc_filtered": {"my": 37511}, "weights": {"en": 0.2946, "de": 0.0909, "vi": 0.0576, "tr": 0.0612, "fi": 0.0424, "ru": 0.1476, "el": 0.0522, "hi": 0.042, "ta": 0.0313, "th": 0.0542, "zh": 0.1038, "my": 0.0222}, "t": 1.5}
{"event": "probe_schedule", "n": 13, "steps": [0, 1, 2, 4, 8, 16, 32, 64, 128, 256, 2000, 4000, 6000], "t": 1.5}
{"event": "probe", "step": 0, "phase": 1, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 2674.0, "chars": 17403.0, "bytes": 17418.0, "cpc": 0.153652, "cps": 20.890625, "mean_p": 0.478126, "boundary_entropy": 0.685438, "bpb": 8.041379}, "de": {"chunks": 2056.0, "chars": 20533.0, "bytes": 20862.0, "cpc": 0.100131, "cps": 16.0625, "mean_p": 0.469682, "boundary_entropy": 0.685273, "bpb": 8.067143}, "vi": {"chunks": 14185.0, "chars": 18435.0, "bytes": 24274.0, "cpc": 0.76946, "cps": 110.820312, "mean_p": 0.507219, "boundary_entropy": 0.688283, "bpb": 7.894113}, "tr": {"chunks": 5645.0, "chars": 18103.0, "bytes": 19742.0, "cpc": 0.311827, "cps": 44.101562, "mean_p": 0.486063, "boundary_entropy": 0.686375, "bpb": 8.01701}, "fi": {"chunks": 4732.0, "chars": 18811.0, "bytes": 19572.0, "cpc": 0.251555, "cps": 36.96875, "mean_p": 0.48451, "boundary_entropy": 0.686595, "bpb": 8.075294}, "ru": {"chunks": 17548.0, "chars": 18950.0, "bytes": 34560.0, "cpc": 0.926016, "cps": 137.09375, "mean_p": 0.502342, "boundary_entropy": 0.690066, "bpb": 8.095633}, "el": {"chunks": 31473.0, "chars": 20523.0, "bytes": 37264.0, "cpc": 1.533548, "cps": 245.882812, "mean_p": 0.517965, "boundary_entropy": 0.689704, "bpb": 7.804942}, "hi": {"chunks": 1659.0, "chars": 16825.0, "bytes": 43173.0, "cpc": 0.098603, "cps": 12.960938, "mean_p": 0.476241, "boundary_entropy": 0.689492, "bpb": 8.047677}, "ta": {"chunks": 39693.0, "chars": 18695.0, "bytes": 50915.0, "cpc": 2.123188, "cps": 310.101562, "mean_p": 0.514597, "boundary_entropy": 0.690542, "bpb": 8.096414}, "th": {"chunks": 44424.0, "chars": 16085.0, "bytes": 46085.0, "cpc": 2.761828, "cps": 347.0625, "mean_p": 0.52078, "boundary_entropy": 0.690224, "bpb": 8.201461}, "zh": {"chunks": 12933.0, "chars": 5906.0, "bytes": 16173.0, "cpc": 2.189807, "cps": 101.039062, "mean_p": 0.528309, "boundary_entropy": 0.684752, "bpb": 8.121995}, "my": {"chunks": 7048.0, "chars": 18948.0, "bytes": 53743.0, "cpc": 0.371965, "cps": 55.0625, "mean_p": 0.491145, "boundary_entropy": 0.691021, "bpb": 8.112229}}, "bpb": {"en": 8.04138, "de": 8.06714, "vi": 7.89411, "tr": 8.01701, "fi": 8.07529, "ru": 8.09563, "el": 7.80494, "hi": 8.04768, "ta": 8.09641, "th": 8.20146, "zh": 8.12199, "my": 8.11223}, "gini_cpc": 0.507116, "gini_cps": 0.500863, "bpic": 2.084973, "worst_lang": "th", "cpc_parity_vs_en": {"en": 1.0, "de": 0.6516785410034256, "vi": 5.0078223656304, "tr": 2.029439514461233, "fi": 1.6371767549398106, "ru": 6.026721581613623, "el": 9.980677296913099, "hi": 0.6417324941848254, "ta": 13.818190381639045, "th": 17.9746032262692, "zh": 14.25176170627287, "my": 2.4208352616119475}, "t": 3.2}
{"event": "train", "step": 0, "phase": 1, "objective": "global", "lm_loss": 5.56843, "bpb": 8.03354, "chunk_loss": 0.018394, "F": 0.00098, "G": 0.49096, "chunks": 1.0, "bytes_per_chunk": 1024.0, "boundary_entropy": 0.6904, "router_noise": 0.0, "grad_norm": 4.288, "lr": 1e-06, "router_trainable": true, "sec_per_step": 0.001, "alpha": 0.03, "t": 3.6}
{"event": "probe", "step": 1, "phase": 1, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 2664.0, "chars": 17403.0, "bytes": 17418.0, "cpc": 0.153077, "cps": 20.8125, "mean_p": 0.478116, "boundary_entropy": 0.68544, "bpb": 8.020959}, "de": {"chunks": 2049.0, "chars": 20533.0, "bytes": 20862.0, "cpc": 0.099791, "cps": 16.007812, "mean_p": 0.469667, "boundary_entropy": 0.685274, "bpb": 8.049532}, "vi": {"chunks": 14185.0, "chars": 18435.0, "bytes": 24274.0, "cpc": 0.76946, "cps": 110.820312, "mean_p": 0.507217, "boundary_entropy": 0.688284, "bpb": 7.879922}, "tr": {"chunks": 5644.0, "chars": 18103.0, "bytes": 19742.0, "cpc": 0.311772, "cps": 44.09375, "mean_p": 0.486068, "boundary_entropy": 0.686377, "bpb": 8.002969}, "fi": {"chunks": 4715.0, "chars": 18811.0, "bytes": 19572.0, "cpc": 0.250651, "cps": 36.835938, "mean_p": 0.484451, "boundary_entropy": 0.686591, "bpb": 8.058056}, "ru": {"chunks": 17619.0, "chars": 18950.0, "bytes": 34560.0, "cpc": 0.929763, "cps": 137.648438, "mean_p": 0.502405, "boundary_entropy": 0.690066, "bpb": 8.091542}, "el": {"chunks": 31509.0, "chars": 20523.0, "bytes": 37264.0, "cpc": 1.535302, "cps": 246.164062, "mean_p": 0.518025, "boundary_entropy": 0.6897, "bpb": 7.798168}, "hi": {"chunks": 1684.0, "chars": 16825.0, "bytes": 43173.0, "cpc": 0.100089, "cps": 13.15625, "mean_p": 0.476383, "boundary_entropy": 0.689508, "bpb": 8.02942}, "ta": {"chunks": 39824.0, "chars": 18695.0, "bytes": 50915.0, "cpc": 2.130195, "cps": 311.125, "mean_p": 0.514695, "boundary_entropy": 0.690537, "bpb": 8.080367}, "th": {"chunks": 44429.0, "chars": 16085.0, "bytes": 46085.0, "cpc": 2.762139, "cps": 347.101562, "mean_p": 0.520846, "boundary_entropy": 0.690219, "bpb": 8.188781}, "zh": {"chunks": 12945.0, "chars": 5906.0, "bytes": 16173.0, "cpc": 2.191839, "cps": 101.132812, "mean_p": 0.52838, "boundary_entropy": 0.684744, "bpb": 8.120358}, "my": {"chunks": 7122.0, "chars": 18948.0, "bytes": 53743.0, "cpc": 0.375871, "cps": 55.640625, "mean_p": 0.491225, "boundary_entropy": 0.691024, "bpb": 8.107101}}, "bpb": {"en": 8.02096, "de": 8.04953, "vi": 7.87992, "tr": 8.00297, "fi": 8.05806, "ru": 8.09154, "el": 7.79817, "hi": 8.02942, "ta": 8.08037, "th": 8.18878, "zh": 8.12036, "my": 8.1071}, "gini_cpc": 0.506909, "gini_cps": 0.500776, "bpic": 2.081366, "worst_lang": "th", "cpc_parity_vs_en": {"en": 1.0, "de": 0.6518977032357931, "vi": 5.026620497633516, "tr": 2.0366966701552127, "fi": 1.637418577207796, "ru": 6.073820330884974, "el": 10.02960142937851, "hi": 0.6538481720948288, "ta": 13.915836242768576, "th": 18.04410612819845, "zh": 14.31853257337934, "my": 2.4554352880429513}, "t": 4.9}
{"event": "probe", "step": 2, "phase": 1, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 2665.0, "chars": 17403.0, "bytes": 17418.0, "cpc": 0.153135, "cps": 20.820312, "mean_p": 0.478086, "boundary_entropy": 0.685442, "bpb": 7.985878}, "de": {"chunks": 2047.0, "chars": 20533.0, "bytes": 20862.0, "cpc": 0.099693, "cps": 15.992188, "mean_p": 0.46964, "boundary_entropy": 0.685274, "bpb": 8.020022}, "vi": {"chunks": 14175.0, "chars": 18435.0, "bytes": 24274.0, "cpc": 0.768918, "cps": 110.742188, "mean_p": 0.507212, "boundary_entropy": 0.688287, "bpb": 7.855724}, "tr": {"chunks": 5645.0, "chars": 18103.0, "bytes": 19742.0, "cpc": 0.311827, "cps": 44.101562, "mean_p": 0.486091, "boundary_entropy": 0.686383, "bpb": 7.979128}, "fi": {"chunks": 4688.0, "chars": 18811.0, "bytes": 19572.0, "cpc": 0.249216, "cps": 36.625, "mean_p": 0.484349, "boundary_entropy": 0.686585, "bpb": 8.030504}, "ru": {"chunks": 17793.0, "chars": 18950.0, "bytes": 34560.0, "cpc": 0.938945, "cps": 139.007812, "mean_p": 0.502572, "boundary_entropy": 0.690065, "bpb": 8.072627}, "el": {"chunks": 31632.0, "chars": 20523.0, "bytes": 37264.0, "cpc": 1.541295, "cps": 247.125, "mean_p": 0.51819, "boundary_entropy": 0.68969, "bpb": 7.781468}, "hi": {"chunks": 1745.0, "chars": 16825.0, "bytes": 43173.0, "cpc": 0.103715, "cps": 13.632812, "mean_p": 0.47671, "boundary_entropy": 0.689541, "bpb": 7.988682}, "ta": {"chunks": 40397.0, "chars": 18695.0, "bytes": 50915.0, "cpc": 2.160845, "cps": 315.601562, "mean_p": 0.515052, "boundary_entropy": 0.690518, "bpb": 8.043904}, "th": {"chunks": 44496.0, "chars": 16085.0, "bytes": 46085.0, "cpc": 2.766304, "cps": 347.625, "mean_p": 0.521205, "boundary_entropy": 0.690191, "bpb": 8.137322}, "zh": {"chunks": 12956.0, "chars": 5906.0, "bytes": 16173.0, "cpc": 2.193701, "cps": 101.21875, "mean_p": 0.528511, "boundary_entropy": 0.68473, "bpb": 8.115871}, "my": {"chunks": 7237.0, "chars": 18948.0, "bytes": 53743.0, "cpc": 0.38194, "cps": 56.539062, "mean_p": 0.491328, "boundary_entropy": 0.691028, "bpb": 8.101076}}, "bpb": {"en": 7.98588, "de": 8.02002, "vi": 7.85572, "tr": 7.97913, "fi": 8.0305, "ru": 8.07263, "el": 7.78147, "hi": 7.98868, "ta": 8.0439, "th": 8.13732, "zh": 8.11587, "my": 8.10108}, "gini_cpc": 0.506485, "gini_cps": 0.50097, "bpic": 2.069168, "worst_lang": "th", "cpc_parity_vs_en": {"en": 1.0, "de": 0.6510170193243129, "vi": 5.021192048936199, "tr": 2.0362931563487194, "fi": 1.6274311575509242, "ru": 6.1315019578529455, "el": 10.064975304785488, "hi": 0.6772784624863051, "ta": 14.110764754850626, "th": 18.064536091973867, "zh": 14.325322357967124, "my": 2.494147327671942}, "t": 6.5}
{"event": "probe", "step": 4, "phase": 1, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 2593.0, "chars": 17403.0, "bytes": 17418.0, "cpc": 0.148997, "cps": 20.257812, "mean_p": 0.477701, "boundary_entropy": 0.685415, "bpb": 7.838458}, "de": {"chunks": 1997.0, "chars": 20533.0, "bytes": 20862.0, "cpc": 0.097258, "cps": 15.601562, "mean_p": 0.469242, "boundary_entropy": 0.685232, "bpb": 7.891691}, "vi": {"chunks": 13915.0, "chars": 18435.0, "bytes": 24274.0, "cpc": 0.754814, "cps": 108.710938, "mean_p": 0.50656, "boundary_entropy": 0.68831, "bpb": 7.759148}, "tr": {"chunks": 5519.0, "chars": 18103.0, "bytes": 19742.0, "cpc": 0.304867, "cps": 43.117188, "mean_p": 0.485785, "boundary_entropy": 0.686374, "bpb": 7.876807}, "fi": {"chunks": 4529.0, "chars": 18811.0, "bytes": 19572.0, "cpc": 0.240763, "cps": 35.382812, "mean_p": 0.483709, "boundary_entropy": 0.686539, "bpb": 7.910918}, "ru": {"chunks": 17253.0, "chars": 18950.0, "bytes": 34560.0, "cpc": 0.910449, "cps": 134.789062, "mean_p": 0.502039, "boundary_entropy": 0.690065, "bpb": 7.942093}, "el": {"chunks": 31592.0, "chars": 20523.0, "bytes": 37264.0, "cpc": 1.539346, "cps": 246.8125, "mean_p": 0.51809, "boundary_entropy": 0.689699, "bpb": 7.710974}, "hi": {"chunks": 1878.0, "chars": 16825.0, "bytes": 43173.0, "cpc": 0.11162, "cps": 14.671875, "mean_p": 0.477394, "boundary_entropy": 0.689611, "bpb": 7.843225}, "ta": {"chunks": 40908.0, "chars": 18695.0, "bytes": 50915.0, "cpc": 2.188179, "cps": 319.59375, "mean_p": 0.515421, "boundary_entropy": 0.690497, "bpb": 7.919492}, "th": {"chunks": 44582.0, "chars": 16085.0, "bytes": 46085.0, "cpc": 2.771651, "cps": 348.296875, "mean_p": 0.521687, "boundary_entropy": 0.690152, "bpb": 7.984667}, "zh": {"chunks": 12996.0, "chars": 5906.0, "bytes": 16173.0, "cpc": 2.200474, "cps": 101.53125, "mean_p": 0.52877, "boundary_entropy": 0.6847, "bpb": 8.098097}, "my": {"chunks": 7299.0, "chars": 18948.0, "bytes": 53743.0, "cpc": 0.385212, "cps": 57.023438, "mean_p": 0.491348, "boundary_entropy": 0.691028, "bpb": 8.039072}}, "bpb": {"en": 7.83846, "de": 7.89169, "vi": 7.75915, "tr": 7.87681, "fi": 7.91092, "ru": 7.94209, "el": 7.71097, "hi": 7.84323, "ta": 7.91949, "th": 7.98467, "zh": 8.0981, "my": 8.03907}, "gini_cpc": 0.508937, "gini_cps": 0.503186, "bpic": 2.073808, "worst_lang": "th", "cpc_parity_vs_en": {"en": 1.0, "de": 0.6527505721086835, "vi": 5.065959017784942, "tr": 2.0461216283502988, "fi": 1.6158909203846255, "ru": 6.110503700342001, "el": 10.331369135968322, "hi": 0.7491385023915687, "ta": 14.686028991378954, "th": 18.60201908943773, "zh": 14.768550196816587, "my": 2.585363368077987}, "t": 8.0}
{"event": "probe", "step": 8, "phase": 1, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 2264.0, "chars": 17403.0, "bytes": 17418.0, "cpc": 0.130093, "cps": 17.6875, "mean_p": 0.475804, "boundary_entropy": 0.685259, "bpb": 7.392165}, "de": {"chunks": 1774.0, "chars": 20533.0, "bytes": 20862.0, "cpc": 0.086398, "cps": 13.859375, "mean_p": 0.467284, "boundary_entropy": 0.685006, "bpb": 7.493377}, "vi": {"chunks": 12826.0, "chars": 18435.0, "bytes": 24274.0, "cpc": 0.695742, "cps": 100.203125, "mean_p": 0.504008, "boundary_entropy": 0.688382, "bpb": 7.484874}, "tr": {"chunks": 5055.0, "chars": 18103.0, "bytes": 19742.0, "cpc": 0.279235, "cps": 39.492188, "mean_p": 0.484228, "boundary_entropy": 0.686305, "bpb": 7.566448}, "fi": {"chunks": 3865.0, "chars": 18811.0, "bytes": 19572.0, "cpc": 0.205465, "cps": 30.195312, "mean_p": 0.480947, "boundary_entropy": 0.686323, "bpb": 7.542082}, "ru": {"chunks": 15176.0, "chars": 18950.0, "bytes": 34560.0, "cpc": 0.800844, "cps": 118.5625, "mean_p": 0.499958, "boundary_entropy": 0.690059, "bpb": 7.441214}, "el": {"chunks": 31671.0, "chars": 20523.0, "bytes": 37264.0, "cpc": 1.543195, "cps": 247.429688, "mean_p": 0.518151, "boundary_entropy": 0.689698, "bpb": 7.560604}, "hi": {"chunks": 2197.0, "chars": 16825.0, "bytes": 43173.0, "cpc": 0.130579, "cps": 17.164062, "mean_p": 0.479041, "boundary_entropy": 0.689771, "bpb": 7.440378}, "ta": {"chunks": 41737.0, "chars": 18695.0, "bytes": 50915.0, "cpc": 2.232522, "cps": 326.070312, "mean_p": 0.516032, "boundary_entropy": 0.690459, "bpb": 7.55434}, "th": {"chunks": 44718.0, "chars": 16085.0, "bytes": 46085.0, "cpc": 2.780106, "cps": 349.359375, "mean_p": 0.522485, "boundary_entropy": 0.690083, "bpb": 7.577396}, "zh": {"chunks": 13107.0, "chars": 5906.0, "bytes": 16173.0, "cpc": 2.219269, "cps": 102.398438, "mean_p": 0.52958, "boundary_entropy": 0.684605, "bpb": 8.021169}, "my": {"chunks": 8247.0, "chars": 18948.0, "bytes": 53743.0, "cpc": 0.435244, "cps": 64.429688, "mean_p": 0.492142, "boundary_entropy": 0.691056, "bpb": 7.85545}}, "bpb": {"en": 7.39216, "de": 7.49338, "vi": 7.48487, "tr": 7.56645, "fi": 7.54208, "ru": 7.44121, "el": 7.5606, "hi": 7.44038, "ta": 7.55434, "th": 7.5774, "zh": 8.02117, "my": 7.85545}, "gini_cpc": 0.517732, "gini_cps": 0.51075, "bpic": 2.101332, "worst_lang": "th", "cpc_parity_vs_en": {"en": 1.0, "de": 0.6641235886934743, "vi": 5.348054093218365, "tr": 2.146437791483324, "fi": 1.579375193598663, "ru": 6.155960170431766, "el": 11.8622925041955, "hi": 1.0037433515874807, "ta": 17.161034228081995, "th": 21.37022053775712, "zh": 17.059156541410246, "my": 3.3456485380110417}, "t": 9.9}
{"event": "probe", "step": 16, "phase": 1, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 1470.0, "chars": 17403.0, "bytes": 17418.0, "cpc": 0.084468, "cps": 11.484375, "mean_p": 0.471159, "boundary_entropy": 0.684859, "bpb": 6.763621}, "de": {"chunks": 1189.0, "chars": 20533.0, "bytes": 20862.0, "cpc": 0.057907, "cps": 9.289062, "mean_p": 0.463065, "boundary_entropy": 0.684565, "bpb": 6.891711}, "vi": {"chunks": 9791.0, "chars": 18435.0, "bytes": 24274.0, "cpc": 0.531109, "cps": 76.492188, "mean_p": 0.497074, "boundary_entropy": 0.688455, "bpb": 7.100707}, "tr": {"chunks": 3864.0, "chars": 18103.0, "bytes": 19742.0, "cpc": 0.213445, "cps": 30.1875, "mean_p": 0.479568, "boundary_entropy": 0.686064, "bpb": 7.112817}, "fi": {"chunks": 2328.0, "chars": 18811.0, "bytes": 19572.0, "cpc": 0.123757, "cps": 18.1875, "mean_p": 0.472976, "boundary_entropy": 0.685561, "bpb": 6.986445}, "ru": {"chunks": 10097.0, "chars": 18950.0, "bytes": 34560.0, "cpc": 0.532823, "cps": 78.882812, "mean_p": 0.494456, "boundary_entropy": 0.689968, "bpb": 6.590353}, "el": {"chunks": 32038.0, "chars": 20523.0, "bytes": 37264.0, "cpc": 1.561078, "cps": 250.296875, "mean_p": 0.518384, "boundary_entropy": 0.689714, "bpb": 7.120871}, "hi": {"chunks": 5305.0, "chars": 16825.0, "bytes": 43173.0, "cpc": 0.315305, "cps": 41.445312, "mean_p": 0.487075, "boundary_entropy": 0.690393, "bpb": 6.736586}, "ta": {"chunks": 46034.0, "chars": 18695.0, "bytes": 50915.0, "cpc": 2.46237, "cps": 359.640625, "mean_p": 0.519724, "boundary_entropy": 0.690229, "bpb": 6.86062}, "th": {"chunks": 45404.0, "chars": 16085.0, "bytes": 46085.0, "cpc": 2.822754, "cps": 354.71875, "mean_p": 0.525404, "boundary_entropy": 0.689823, "bpb": 6.685688}, "zh": {"chunks": 13563.0, "chars": 5906.0, "bytes": 16173.0, "cpc": 2.296478, "cps": 105.960938, "mean_p": 0.532697, "boundary_entropy": 0.684263, "bpb": 7.802035}, "my": {"chunks": 11189.0, "chars": 18948.0, "bytes": 53743.0, "cpc": 0.590511, "cps": 87.414062, "mean_p": 0.493836, "boundary_entropy": 0.691104, "bpb": 7.307892}}, "bpb": {"en": 6.76362, "de": 6.89171, "vi": 7.10071, "tr": 7.11282, "fi": 6.98644, "ru": 6.59035, "el": 7.12087, "hi": 6.73659, "ta": 6.86062, "th": 6.68569, "zh": 7.80204, "my": 7.30789}, "gini_cpc": 0.53621, "gini_cps": 0.531771, "bpic": 2.10554, "worst_lang": "th", "cpc_parity_vs_en": {"en": 1.0, "de": 0.6855454186739713, "vi": 6.287683809080996, "tr": 2.526930816518178, "fi": 1.4651357922361972, "ru": 6.30797447633407, "el": 18.481249807334134, "hi": 3.7328204506170968, "ta": 29.15144112525995, "th": 33.417952332316204, "zh": 27.187489374347777, "my": 6.99092564828338}, "t": 12.0}
{"event": "probe", "step": 32, "phase": 1, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 383.0, "chars": 17403.0, "bytes": 17418.0, "cpc": 0.022008, "cps": 2.992188, "mean_p": 0.433994, "boundary_entropy": 0.677224, "bpb": 6.352728}, "de": {"chunks": 335.0, "chars": 20533.0, "bytes": 20862.0, "cpc": 0.016315, "cps": 2.617188, "mean_p": 0.429047, "boundary_entropy": 0.676857, "bpb": 6.44189}, "vi": {"chunks": 581.0, "chars": 18435.0, "bytes": 24274.0, "cpc": 0.031516, "cps": 4.539062, "mean_p": 0.45394, "boundary_entropy": 0.683796, "bpb": 6.864593}, "tr": {"chunks": 531.0, "chars": 18103.0, "bytes": 19742.0, "cpc": 0.029332, "cps": 4.148438, "mean_p": 0.443549, "boundary_entropy": 0.680185, "bpb": 6.736257}, "fi": {"chunks": 349.0, "chars": 18811.0, "bytes": 19572.0, "cpc": 0.018553, "cps": 2.726562, "mean_p": 0.43495, "boundary_entropy": 0.678082, "bpb": 6.507426}, "ru": {"chunks": 139.0, "chars": 18950.0, "bytes": 34560.0, "cpc": 0.007335, "cps": 1.085938, "mean_p": 0.445588, "boundary_entropy": 0.683888, "bpb": 6.150857}, "el": {"chunks": 8892.0, "chars": 20523.0, "bytes": 37264.0, "cpc": 0.43327, "cps": 69.46875, "mean_p": 0.491312, "boundary_entropy": 0.690112, "bpb": 6.302296}, "hi": {"chunks": 1311.0, "chars": 16825.0, "bytes": 43173.0, "cpc": 0.07792, "cps": 10.242188, "mean_p": 0.481567, "boundary_entropy": 0.690105, "bpb": 6.098238}, "ta": {"chunks": 26982.0, "chars": 18695.0, "bytes": 50915.0, "cpc": 1.443274, "cps": 210.796875, "mean_p": 0.503229, "boundary_entropy": 0.691096, "bpb": 6.387771}, "th": {"chunks": 27042.0, "chars": 16085.0, "bytes": 46085.0, "cpc": 1.681194, "cps": 211.265625, "mean_p": 0.504089, "boundary_entropy": 0.691072, "bpb": 5.606693}, "zh": {"chunks": 11444.0, "chars": 5906.0, "bytes": 16173.0, "cpc": 1.93769, "cps": 89.40625, "mean_p": 0.517991, "boundary_entropy": 0.685907, "bpb": 7.424534}, "my": {"chunks": 1991.0, "chars": 18948.0, "bytes": 53743.0, "cpc": 0.105077, "cps": 15.554688, "mean_p": 0.481385, "boundary_entropy": 0.690415, "bpb": 6.382926}}, "bpb": {"en": 6.35273, "de": 6.44189, "vi": 6.86459, "tr": 6.73626, "fi": 6.50743, "ru": 6.15086, "el": 6.3023, "hi": 6.09824, "ta": 6.38777, "th": 5.60669, "zh": 7.42453, "my": 6.38293}, "gini_cpc": 0.69672, "gini_cps": 0.688793, "bpic": 4.798462, "worst_lang": "zh", "cpc_parity_vs_en": {"en": 1.0, "de": 0.7413405332738905, "vi": 1.4320505112522228, "tr": 1.3328132939320676, "fi": 0.8430219638445534, "ru": 0.3332966374339508, "el": 19.687200083355503, "hi": 3.540568205183912, "ta": 65.58039296470692, "th": 76.39115729021168, "zh": 88.04602479754624, "my": 4.774558624373095}, "t": 14.8}
{"event": "probe", "step": 64, "phase": 1, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 243.0, "chars": 17403.0, "bytes": 17418.0, "cpc": 0.013963, "cps": 1.898438, "mean_p": 0.361583, "boundary_entropy": 0.645794, "bpb": 5.795061}, "de": {"chunks": 211.0, "chars": 20533.0, "bytes": 20862.0, "cpc": 0.010276, "cps": 1.648438, "mean_p": 0.357859, "boundary_entropy": 0.645031, "bpb": 5.937274}, "vi": {"chunks": 184.0, "chars": 18435.0, "bytes": 24274.0, "cpc": 0.009981, "cps": 1.4375, "mean_p": 0.366439, "boundary_entropy": 0.650482, "bpb": 6.511334}, "tr": {"chunks": 188.0, "chars": 18103.0, "bytes": 19742.0, "cpc": 0.010385, "cps": 1.46875, "mean_p": 0.362114, "boundary_entropy": 0.646959, "bpb": 6.32238}, "fi": {"chunks": 166.0, "chars": 18811.0, "bytes": 19572.0, "cpc": 0.008825, "cps": 1.296875, "mean_p": 0.359204, "boundary_entropy": 0.645288, "bpb": 6.008648}, "ru": {"chunks": 130.0, "chars": 18950.0, "bytes": 34560.0, "cpc": 0.00686, "cps": 1.015625, "mean_p": 0.38614, "boundary_entropy": 0.663323, "bpb": 5.589999}, "el": {"chunks": 128.0, "chars": 20523.0, "bytes": 37264.0, "cpc": 0.006237, "cps": 1.0, "mean_p": 0.420718, "boundary_entropy": 0.677257, "bpb": 6.311157}, "hi": {"chunks": 140.0, "chars": 16825.0, "bytes": 43173.0, "cpc": 0.008321, "cps": 1.09375, "mean_p": 0.375218, "boundary_entropy": 0.658523, "bpb": 5.840602}, "ta": {"chunks": 146.0, "chars": 18695.0, "bytes": 50915.0, "cpc": 0.00781, "cps": 1.140625, "mean_p": 0.373999, "boundary_entropy": 0.65836, "bpb": 5.796934}, "th": {"chunks": 131.0, "chars": 16085.0, "bytes": 46085.0, "cpc": 0.008144, "cps": 1.023438, "mean_p": 0.384179, "boundary_entropy": 0.663251, "bpb": 5.662074}, "zh": {"chunks": 186.0, "chars": 5906.0, "bytes": 16173.0, "cpc": 0.031493, "cps": 1.453125, "mean_p": 0.391527, "boundary_entropy": 0.660036, "bpb": 7.3059}, "my": {"chunks": 254.0, "chars": 18948.0, "bytes": 53743.0, "cpc": 0.013405, "cps": 1.984375, "mean_p": 0.437689, "boundary_entropy": 0.683144, "bpb": 5.71819}}, "bpb": {"en": 5.79506, "de": 5.93727, "vi": 6.51133, "tr": 6.32238, "fi": 6.00865, "ru": 5.59, "el": 6.31116, "hi": 5.8406, "ta": 5.79693, "th": 5.66207, "zh": 7.3059, "my": 5.71819}, "gini_cpc": 0.245113, "gini_cps": 0.132139, "bpic": 182.145705, "worst_lang": "zh", "cpc_parity_vs_en": {"en": 1.0, "de": 0.7359492969161957, "vi": 0.7148131405974277, "tr": 0.7437468586817683, "fi": 0.6319955949073663, "ru": 0.49130590572982835, "el": 0.4466701917691864, "hi": 0.5959239080586283, "ta": 0.5593005325910738, "th": 0.5832679016183318, "zh": 2.255471522996074, "my": 0.9600374774887476}, "t": 19.3}
{"event": "probe", "step": 128, "phase": 1, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 133.0, "chars": 17403.0, "bytes": 17418.0, "cpc": 0.007642, "cps": 1.039062, "mean_p": 0.379597, "boundary_entropy": 0.655613, "bpb": 4.548053}, "de": {"chunks": 152.0, "chars": 20533.0, "bytes": 20862.0, "cpc": 0.007403, "cps": 1.1875, "mean_p": 0.37754, "boundary_entropy": 0.655408, "bpb": 4.753989}, "vi": {"chunks": 827.0, "chars": 18435.0, "bytes": 24274.0, "cpc": 0.04486, "cps": 6.460938, "mean_p": 0.418843, "boundary_entropy": 0.671647, "bpb": 5.113376}, "tr": {"chunks": 300.0, "chars": 18103.0, "bytes": 19742.0, "cpc": 0.016572, "cps": 2.34375, "mean_p": 0.396282, "boundary_entropy": 0.663492, "bpb": 5.134856}, "fi": {"chunks": 220.0, "chars": 18811.0, "bytes": 19572.0, "cpc": 0.011695, "cps": 1.71875, "mean_p": 0.382113, "boundary_entropy": 0.656636, "bpb": 4.833313}, "ru": {"chunks": 161.0, "chars": 18950.0, "bytes": 34560.0, "cpc": 0.008496, "cps": 1.257812, "mean_p": 0.416762, "boundary_entropy": 0.674024, "bpb": 3.777848}, "el": {"chunks": 128.0, "chars": 20523.0, "bytes": 37264.0, "cpc": 0.006237, "cps": 1.0, "mean_p": 0.398727, "boundary_entropy": 0.669046, "bpb": 4.63175}, "hi": {"chunks": 135.0, "chars": 16825.0, "bytes": 43173.0, "cpc": 0.008024, "cps": 1.054688, "mean_p": 0.428426, "boundary_entropy": 0.679865, "bpb": 3.545774}, "ta": {"chunks": 131.0, "chars": 18695.0, "bytes": 50915.0, "cpc": 0.007007, "cps": 1.023438, "mean_p": 0.417263, "boundary_entropy": 0.676927, "bpb": 3.739576}, "th": {"chunks": 215.0, "chars": 16085.0, "bytes": 46085.0, "cpc": 0.013366, "cps": 1.679688, "mean_p": 0.428008, "boundary_entropy": 0.677759, "bpb": 3.443156}, "zh": {"chunks": 139.0, "chars": 5906.0, "bytes": 16173.0, "cpc": 0.023535, "cps": 1.085938, "mean_p": 0.412482, "boundary_entropy": 0.670152, "bpb": 6.367455}, "my": {"chunks": 128.0, "chars": 18948.0, "bytes": 53743.0, "cpc": 0.006755, "cps": 1.0, "mean_p": 0.401239, "boundary_entropy": 0.6711, "bpb": 4.022402}}, "bpb": {"en": 4.54805, "de": 4.75399, "vi": 5.11338, "tr": 5.13486, "fi": 4.83331, "ru": 3.77785, "el": 4.63175, "hi": 3.54577, "ta": 3.73958, "th": 3.44316, "zh": 6.36745, "my": 4.0224}, "gini_cpc": 0.353397, "gini_cps": 0.323498, "bpic": 143.792057, "worst_lang": "vi", "cpc_parity_vs_en": {"en": 1.0, "de": 0.9686428119194885, "vi": 5.869956012896358, "tr": 2.168418892893173, "fi": 1.5303236028511553, "ru": 1.111703930009721, "el": 0.8160966661647541, "hi": 1.0499078304974918, "ta": 0.9168922573885906, "th": 1.7490002594300487, "zh": 3.079596638056367, "my": 0.8839324403472265}, "t": 27.5}
{"event": "train", "step": 200, "phase": 1, "objective": "global", "lm_loss": 2.58195, "bpb": 3.72497, "chunk_loss": 0.021673, "F": 0.00098, "G": 0.39935, "chunks": 1.0, "bytes_per_chunk": 1024.0, "boundary_entropy": 0.66927, "router_noise": 0.0, "grad_norm": 1.4347, "lr": 0.000201, "router_trainable": true, "sec_per_step": 0.158, "alpha": 0.03, "t": 35.3}
{"event": "probe", "step": 256, "phase": 1, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 491.0, "chars": 17403.0, "bytes": 17418.0, "cpc": 0.028214, "cps": 3.835938, "mean_p": 0.414525, "boundary_entropy": 0.669415, "bpb": 3.672405}, "de": {"chunks": 840.0, "chars": 20533.0, "bytes": 20862.0, "cpc": 0.04091, "cps": 6.5625, "mean_p": 0.405741, "boundary_entropy": 0.666839, "bpb": 3.88857}, "vi": {"chunks": 1137.0, "chars": 18435.0, "bytes": 24274.0, "cpc": 0.061676, "cps": 8.882812, "mean_p": 0.414811, "boundary_entropy": 0.668569, "bpb": 3.272238}, "tr": {"chunks": 1118.0, "chars": 18103.0, "bytes": 19742.0, "cpc": 0.061758, "cps": 8.734375, "mean_p": 0.413739, "boundary_entropy": 0.669019, "bpb": 3.924511}, "fi": {"chunks": 1019.0, "chars": 18811.0, "bytes": 19572.0, "cpc": 0.05417, "cps": 7.960938, "mean_p": 0.420978, "boundary_entropy": 0.672208, "bpb": 3.919382}, "ru": {"chunks": 195.0, "chars": 18950.0, "bytes": 34560.0, "cpc": 0.01029, "cps": 1.523438, "mean_p": 0.399764, "boundary_entropy": 0.66761, "bpb": 2.665971}, "el": {"chunks": 825.0, "chars": 20523.0, "bytes": 37264.0, "cpc": 0.040199, "cps": 6.445312, "mean_p": 0.438363, "boundary_entropy": 0.681206, "bpb": 2.751758}, "hi": {"chunks": 418.0, "chars": 16825.0, "bytes": 43173.0, "cpc": 0.024844, "cps": 3.265625, "mean_p": 0.425079, "boundary_entropy": 0.677312, "bpb": 2.163683}, "ta": {"chunks": 175.0, "chars": 18695.0, "bytes": 50915.0, "cpc": 0.009361, "cps": 1.367188, "mean_p": 0.439973, "boundary_entropy": 0.682977, "bpb": 2.040198}, "th": {"chunks": 2904.0, "chars": 16085.0, "bytes": 46085.0, "cpc": 0.180541, "cps": 22.6875, "mean_p": 0.456926, "boundary_entropy": 0.685649, "bpb": 2.111542}, "zh": {"chunks": 436.0, "chars": 5906.0, "bytes": 16173.0, "cpc": 0.073823, "cps": 3.40625, "mean_p": 0.40505, "boundary_entropy": 0.664369, "bpb": 5.011176}, "my": {"chunks": 130.0, "chars": 18948.0, "bytes": 53743.0, "cpc": 0.006861, "cps": 1.015625, "mean_p": 0.419887, "boundary_entropy": 0.676671, "bpb": 2.270936}}, "bpb": {"en": 3.6724, "de": 3.88857, "vi": 3.27224, "tr": 3.92451, "fi": 3.91938, "ru": 2.66597, "el": 2.75176, "hi": 2.16368, "ta": 2.0402, "th": 2.11154, "zh": 5.01118, "my": 2.27094}, "gini_cpc": 0.437818, "gini_cps": 0.431668, "bpic": 39.614059, "worst_lang": "th", "cpc_parity_vs_en": {"en": 1.0, "de": 1.4500050239527984, "vi": 2.1860492941291496, "tr": 2.1889401144593177, "fi": 1.9200163573746392, "ru": 0.3647270929501475, "el": 1.4248059873811043, "hi": 0.8805699016169203, "ta": 0.33178382317935734, "th": 6.3990893591643685, "zh": 2.616589984433656, "my": 0.24317706042520917}, "t": 41.9}
{"event": "train", "step": 400, "phase": 1, "objective": "global", "lm_loss": 2.03632, "bpb": 2.93779, "chunk_loss": 0.024239, "F": 0.02814, "G": 0.36063, "chunks": 28.81, "bytes_per_chunk": 35.54, "boundary_entropy": 0.64731, "router_noise": 0.0, "grad_norm": 1.7345, "lr": 0.0002998, "router_trainable": true, "sec_per_step": 0.105, "alpha": 0.03, "t": 56.3}
{"event": "train", "step": 600, "phase": 1, "objective": "global", "lm_loss": 1.87477, "bpb": 2.70472, "chunk_loss": 0.022649, "F": 0.00098, "G": 0.37207, "chunks": 1.0, "bytes_per_chunk": 1024.0, "boundary_entropy": 0.65357, "router_noise": 0.0, "grad_norm": 0.7489, "lr": 0.0002982, "router_trainable": true, "sec_per_step": 0.095, "alpha": 0.03, "t": 75.3}
{"event": "train", "step": 800, "phase": 1, "objective": "global", "lm_loss": 1.67521, "bpb": 2.41682, "chunk_loss": 0.022193, "F": 0.00098, "G": 0.38481, "chunks": 1.0, "bytes_per_chunk": 1024.0, "boundary_entropy": 0.65926, "router_noise": 0.0, "grad_norm": 0.8214, "lr": 0.0002949, "router_trainable": true, "sec_per_step": 0.096, "alpha": 0.03, "t": 94.6}
{"event": "eval", "step": 1000, "phase": 1, "bpb": {"en": 2.6676, "de": 2.98901, "vi": 2.3604, "tr": 2.97268, "fi": 3.23071, "ru": 1.92693, "el": 2.07715, "hi": 1.58912, "ta": 1.37095, "th": 1.62376, "zh": 3.71974, "my": 1.45979}, "bytes_per_chunk": {"en": 88.923, "de": 193.179, "vi": 722.16, "tr": 107.745, "fi": 293.226, "ru": 113.188, "el": 473.184, "hi": 188.052, "ta": 439.102, "th": 946.368, "zh": 826.953, "my": 341.333}, "bpb_mean": 2.33232, "t": 113.4}
{"event": "train", "step": 1000, "phase": 1, "objective": "global", "lm_loss": 1.78443, "bpb": 2.57438, "chunk_loss": 0.025766, "F": 0.04855, "G": 0.33372, "chunks": 49.72, "bytes_per_chunk": 20.596, "boundary_entropy": 0.62891, "router_noise": 0.0, "grad_norm": 0.8492, "lr": 0.0002901, "router_trainable": true, "sec_per_step": 0.094, "alpha": 0.03, "t": 113.4}
{"event": "train", "step": 1200, "phase": 1, "objective": "global", "lm_loss": 1.65627, "bpb": 2.3895, "chunk_loss": 0.023015, "F": 0.00098, "G": 0.36183, "chunks": 1.0, "bytes_per_chunk": 1024.0, "boundary_entropy": 0.64658, "router_noise": 0.0, "grad_norm": 0.8266, "lr": 0.0002837, "router_trainable": true, "sec_per_step": 0.093, "alpha": 0.03, "t": 132.0}
{"event": "train", "step": 1400, "phase": 1, "objective": "global", "lm_loss": 1.61348, "bpb": 2.32776, "chunk_loss": 0.02191, "F": 0.00098, "G": 0.39272, "chunks": 1.0, "bytes_per_chunk": 1024.0, "boundary_entropy": 0.66309, "router_noise": 0.0, "grad_norm": 0.9687, "lr": 0.0002759, "router_trainable": true, "sec_per_step": 0.089, "alpha": 0.03, "t": 149.9}
{"event": "train", "step": 1600, "phase": 1, "objective": "global", "lm_loss": 1.62579, "bpb": 2.34551, "chunk_loss": 0.023298, "F": 0.00098, "G": 0.35393, "chunks": 1.0, "bytes_per_chunk": 1024.0, "boundary_entropy": 0.64151, "router_noise": 0.0, "grad_norm": 0.8963, "lr": 0.0002668, "router_trainable": true, "sec_per_step": 0.088, "alpha": 0.03, "t": 167.5}
{"event": "train", "step": 1800, "phase": 1, "objective": "global", "lm_loss": 1.62437, "bpb": 2.34347, "chunk_loss": 0.023368, "F": 0.00098, "G": 0.35197, "chunks": 1.0, "bytes_per_chunk": 1024.0, "boundary_entropy": 0.63951, "router_noise": 0.0, "grad_norm": 1.2577, "lr": 0.0002564, "router_trainable": true, "sec_per_step": 0.091, "alpha": 0.03, "t": 185.7}
{"event": "probe", "step": 2000, "phase": 1, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 257.0, "chars": 17403.0, "bytes": 17418.0, "cpc": 0.014768, "cps": 2.007812, "mean_p": 0.355916, "boundary_entropy": 0.634868, "bpb": 2.334603}, "de": {"chunks": 336.0, "chars": 20533.0, "bytes": 20862.0, "cpc": 0.016364, "cps": 2.625, "mean_p": 0.356176, "boundary_entropy": 0.636464, "bpb": 2.615902}, "vi": {"chunks": 347.0, "chars": 18435.0, "bytes": 24274.0, "cpc": 0.018823, "cps": 2.710938, "mean_p": 0.356379, "boundary_entropy": 0.638153, "bpb": 2.12749}, "tr": {"chunks": 382.0, "chars": 18103.0, "bytes": 19742.0, "cpc": 0.021101, "cps": 2.984375, "mean_p": 0.35538, "boundary_entropy": 0.635151, "bpb": 2.613461}, "fi": {"chunks": 233.0, "chars": 18811.0, "bytes": 19572.0, "cpc": 0.012386, "cps": 1.820312, "mean_p": 0.36462, "boundary_entropy": 0.642233, "bpb": 2.848301}, "ru": {"chunks": 543.0, "chars": 18950.0, "bytes": 34560.0, "cpc": 0.028654, "cps": 4.242188, "mean_p": 0.350991, "boundary_entropy": 0.638405, "bpb": 1.688301}, "el": {"chunks": 448.0, "chars": 20523.0, "bytes": 37264.0, "cpc": 0.021829, "cps": 3.5, "mean_p": 0.351743, "boundary_entropy": 0.638343, "bpb": 1.7839}, "hi": {"chunks": 1177.0, "chars": 16825.0, "bytes": 43173.0, "cpc": 0.069955, "cps": 9.195312, "mean_p": 0.359596, "boundary_entropy": 0.641316, "bpb": 1.411766}, "ta": {"chunks": 1155.0, "chars": 18695.0, "bytes": 50915.0, "cpc": 0.061781, "cps": 9.023438, "mean_p": 0.365126, "boundary_entropy": 0.645848, "bpb": 1.127496}, "th": {"chunks": 224.0, "chars": 16085.0, "bytes": 46085.0, "cpc": 0.013926, "cps": 1.75, "mean_p": 0.350389, "boundary_entropy": 0.638766, "bpb": 1.318851}, "zh": {"chunks": 137.0, "chars": 5906.0, "bytes": 16173.0, "cpc": 0.023197, "cps": 1.070312, "mean_p": 0.349868, "boundary_entropy": 0.633739, "bpb": 3.50542}, "my": {"chunks": 210.0, "chars": 18948.0, "bytes": 53743.0, "cpc": 0.011083, "cps": 1.640625, "mean_p": 0.321643, "boundary_entropy": 0.613809, "bpb": 1.251973}}, "bpb": {"en": 2.3346, "de": 2.6159, "vi": 2.12749, "tr": 2.61346, "fi": 2.8483, "ru": 1.6883, "el": 1.7839, "hi": 1.41177, "ta": 1.1275, "th": 1.31885, "zh": 3.50542, "my": 1.25197}, "gini_cpc": 0.333492, "gini_cps": 0.361519, "bpic": 70.431455, "worst_lang": "hi", "cpc_parity_vs_en": {"en": 1.0, "de": 1.1080972245304654, "vi": 1.2746100242834482, "tr": 1.4289064886164793, "fi": 0.8387548307172702, "ru": 1.9403568678582794, "el": 1.4781828719832413, "hi": 4.737098189765323, "ta": 4.183574542393095, "th": 0.943013586624559, "zh": 1.5707899768223568, "my": 0.7504934660414043}, "t": 203.2}
{"event": "eval", "step": 2000, "phase": 1, "bpb": {"en": 2.34455, "de": 2.62199, "vi": 2.13169, "tr": 2.62324, "fi": 2.84831, "ru": 1.66786, "el": 1.77523, "hi": 1.40934, "ta": 1.13558, "th": 1.33676, "zh": 3.50947, "my": 1.26013}, "bytes_per_chunk": {"en": 86.345, "de": 75.199, "vi": 144.194, "tr": 80.709, "fi": 131.863, "ru": 97.162, "el": 117.501, "hi": 43.394, "ta": 53.685, "th": 179.92, "zh": 564.966, "my": 227.358}, "bpb_mean": 2.05535, "t": 204.4}
{"event": "train", "step": 2000, "phase": 1, "objective": "global", "lm_loss": 1.55604, "bpb": 2.24489, "chunk_loss": 0.025613, "F": 0.0563, "G": 0.35118, "chunks": 57.66, "bytes_per_chunk": 17.76, "boundary_entropy": 0.63933, "router_noise": 0.0, "grad_norm": 0.8047, "lr": 0.000245, "router_trainable": true, "sec_per_step": 0.094, "alpha": 0.03, "t": 204.4}
{"event": "train", "step": 2200, "phase": 1, "objective": "global", "lm_loss": 1.52653, "bpb": 2.20232, "chunk_loss": 0.022589, "F": 0.00098, "G": 0.37375, "chunks": 1.0, "bytes_per_chunk": 1024.0, "boundary_entropy": 0.65299, "router_noise": 0.0, "grad_norm": 0.7575, "lr": 0.0002325, "router_trainable": true, "sec_per_step": 0.088, "alpha": 0.03, "t": 222.0}
{"event": "phase_transition", "step": 2400, "phase": 2, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0, "t": 239.2}
{"event": "train", "step": 2400, "phase": 2, "objective": "global", "lm_loss": 1.55384, "bpb": 2.24171, "chunk_loss": 0.054117, "F": 0.73846, "G": 0.36101, "chunks": 756.19, "bytes_per_chunk": 1.354, "boundary_entropy": 0.64339, "router_noise": 0.0, "grad_norm": 0.8794, "lr": 0.0002192, "router_trainable": true, "sec_per_step": 0.086, "alpha": 0.03, "t": 239.3}
{"event": "train", "step": 2600, "phase": 2, "objective": "global", "lm_loss": 1.71623, "bpb": 2.47599, "chunk_loss": 0.022556, "F": 0.00098, "G": 0.37466, "chunks": 1.0, "bytes_per_chunk": 1024.0, "boundary_entropy": 0.65261, "router_noise": 0.0, "grad_norm": 0.805, "lr": 0.0002053, "router_trainable": true, "sec_per_step": 0.089, "alpha": 0.03, "t": 257.0}
{"event": "train", "step": 2800, "phase": 2, "objective": "global", "lm_loss": 1.50444, "bpb": 2.17045, "chunk_loss": 0.023924, "F": 0.02304, "G": 0.36283, "chunks": 23.59, "bytes_per_chunk": 43.401, "boundary_entropy": 0.64536, "router_noise": 0.0, "grad_norm": 0.7812, "lr": 0.0001909, "router_trainable": true, "sec_per_step": 0.087, "alpha": 0.03, "t": 274.4}
{"event": "eval", "step": 3000, "phase": 2, "bpb": {"en": 2.21551, "de": 2.45319, "vi": 2.04296, "tr": 2.43092, "fi": 2.65147, "ru": 1.53801, "el": 1.61944, "hi": 1.31357, "ta": 1.08274, "th": 1.23433, "zh": 3.36493, "my": 1.18579}, "bytes_per_chunk": {"en": 40.661, "de": 53.521, "vi": 82.747, "tr": 63.442, "fi": 86.288, "ru": 55.211, "el": 71.998, "hi": 36.419, "ta": 54.534, "th": 127.938, "zh": 89.043, "my": 69.369}, "bpb_mean": 1.92774, "t": 292.8}
{"event": "train", "step": 3000, "phase": 2, "objective": "global", "lm_loss": 1.52656, "bpb": 2.20236, "chunk_loss": 0.022673, "F": 0.00098, "G": 0.3714, "chunks": 1.0, "bytes_per_chunk": 1024.0, "boundary_entropy": 0.6496, "router_noise": 0.0, "grad_norm": 0.8643, "lr": 0.0001761, "router_trainable": true, "sec_per_step": 0.093, "alpha": 0.03, "t": 292.9}
{"event": "train", "step": 3200, "phase": 2, "objective": "global", "lm_loss": 1.36113, "bpb": 1.9637, "chunk_loss": 0.02255, "F": 0.00098, "G": 0.37483, "chunks": 1.0, "bytes_per_chunk": 1024.0, "boundary_entropy": 0.65182, "router_noise": 0.0, "grad_norm": 0.9588, "lr": 0.0001613, "router_trainable": true, "sec_per_step": 0.089, "alpha": 0.03, "t": 310.7}
{"event": "train", "step": 3400, "phase": 2, "objective": "global", "lm_loss": 1.4062, "bpb": 2.02872, "chunk_loss": 0.036152, "F": 0.30194, "G": 0.3746, "chunks": 309.19, "bytes_per_chunk": 3.312, "boundary_entropy": 0.65167, "router_noise": 0.0, "grad_norm": 0.7784, "lr": 0.0001465, "router_trainable": true, "sec_per_step": 0.089, "alpha": 0.03, "t": 328.5}
{"event": "train", "step": 3600, "phase": 2, "objective": "global", "lm_loss": 1.52441, "bpb": 2.19926, "chunk_loss": 0.048164, "F": 0.57361, "G": 0.37125, "chunks": 587.38, "bytes_per_chunk": 1.743, "boundary_entropy": 0.6504, "router_noise": 0.0, "grad_norm": 1.7498, "lr": 0.0001319, "router_trainable": true, "sec_per_step": 0.093, "alpha": 0.03, "t": 347.1}
{"event": "train", "step": 3800, "phase": 2, "objective": "global", "lm_loss": 1.41311, "bpb": 2.03869, "chunk_loss": 0.042968, "F": 0.48413, "G": 0.35482, "chunks": 495.75, "bytes_per_chunk": 2.066, "boundary_entropy": 0.63936, "router_noise": 0.0, "grad_norm": 0.8064, "lr": 0.0001177, "router_trainable": true, "sec_per_step": 0.088, "alpha": 0.03, "t": 364.7}
{"event": "probe", "step": 4000, "phase": 2, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 395.0, "chars": 17403.0, "bytes": 17418.0, "cpc": 0.022697, "cps": 3.085938, "mean_p": 0.367709, "boundary_entropy": 0.639918, "bpb": 2.140504}, "de": {"chunks": 337.0, "chars": 20533.0, "bytes": 20862.0, "cpc": 0.016413, "cps": 2.632812, "mean_p": 0.367454, "boundary_entropy": 0.643109, "bpb": 2.355928}, "vi": {"chunks": 457.0, "chars": 18435.0, "bytes": 24274.0, "cpc": 0.02479, "cps": 3.570312, "mean_p": 0.372393, "boundary_entropy": 0.646221, "bpb": 1.985718}, "tr": {"chunks": 464.0, "chars": 18103.0, "bytes": 19742.0, "cpc": 0.025631, "cps": 3.625, "mean_p": 0.376195, "boundary_entropy": 0.647673, "bpb": 2.32451}, "fi": {"chunks": 290.0, "chars": 18811.0, "bytes": 19572.0, "cpc": 0.015417, "cps": 2.265625, "mean_p": 0.375932, "boundary_entropy": 0.648623, "bpb": 2.539534}, "ru": {"chunks": 833.0, "chars": 18950.0, "bytes": 34560.0, "cpc": 0.043958, "cps": 6.507812, "mean_p": 0.38216, "boundary_entropy": 0.656092, "bpb": 1.489035}, "el": {"chunks": 489.0, "chars": 20523.0, "bytes": 37264.0, "cpc": 0.023827, "cps": 3.820312, "mean_p": 0.381608, "boundary_entropy": 0.656643, "bpb": 1.541612}, "hi": {"chunks": 1202.0, "chars": 16825.0, "bytes": 43173.0, "cpc": 0.071441, "cps": 9.390625, "mean_p": 0.372576, "boundary_entropy": 0.647457, "bpb": 1.269919}, "ta": {"chunks": 826.0, "chars": 18695.0, "bytes": 50915.0, "cpc": 0.044183, "cps": 6.453125, "mean_p": 0.371251, "boundary_entropy": 0.647964, "bpb": 1.03482}, "th": {"chunks": 508.0, "chars": 16085.0, "bytes": 46085.0, "cpc": 0.031582, "cps": 3.96875, "mean_p": 0.377789, "boundary_entropy": 0.654107, "bpb": 1.163126}, "zh": {"chunks": 210.0, "chars": 5906.0, "bytes": 16173.0, "cpc": 0.035557, "cps": 1.640625, "mean_p": 0.370316, "boundary_entropy": 0.64517, "bpb": 3.272949}, "my": {"chunks": 303.0, "chars": 18948.0, "bytes": 53743.0, "cpc": 0.015991, "cps": 2.367188, "mean_p": 0.362003, "boundary_entropy": 0.64003, "bpb": 1.137374}}, "bpb": {"en": 2.1405, "de": 2.35593, "vi": 1.98572, "tr": 2.32451, "fi": 2.53953, "ru": 1.48904, "el": 1.54161, "hi": 1.26992, "ta": 1.03482, "th": 1.16313, "zh": 3.27295, "my": 1.13737}, "gini_cpc": 0.258247, "gini_cps": 0.271935, "bpic": 60.782547, "worst_lang": "hi", "cpc_parity_vs_en": {"en": 1.0, "de": 0.7231102510500231, "vi": 1.0921947451360063, "tr": 1.129261322516654, "fi": 0.6792241813805415, "ru": 1.9367020473598076, "el": 1.049772129694563, "hi": 3.1475774070382, "ta": 1.9466218883408208, "th": 1.3914566208787966, "zh": 1.5665810782426794, "my": 0.704541008970587}, "t": 383.9}
{"event": "eval", "step": 4000, "phase": 2, "bpb": {"en": 2.15032, "de": 2.36116, "vi": 1.98784, "tr": 2.33377, "fi": 2.54249, "ru": 1.46527, "el": 1.53314, "hi": 1.26588, "ta": 1.04159, "th": 1.18211, "zh": 3.27315, "my": 1.14581}, "bytes_per_chunk": {"en": 44.184, "de": 86.09, "vi": 81.059, "tr": 60.194, "fi": 103.737, "ru": 57.614, "el": 95.743, "hi": 40.839, "ta": 64.172, "th": 97.162, "zh": 161.022, "my": 191.486}, "bpb_mean": 1.85688, "t": 385.1}
{"event": "train", "step": 4000, "phase": 2, "objective": "global", "lm_loss": 1.45512, "bpb": 2.0993, "chunk_loss": 0.022719, "F": 0.00098, "G": 0.37011, "chunks": 1.0, "bytes_per_chunk": 1024.0, "boundary_entropy": 0.64993, "router_noise": 0.0, "grad_norm": 0.8842, "lr": 0.000104, "router_trainable": true, "sec_per_step": 0.102, "alpha": 0.03, "t": 385.2}
{"event": "phase_transition", "step": 4200, "phase": 3, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0, "t": 403.4}
{"event": "train", "step": 4200, "phase": 3, "objective": "global", "lm_loss": 1.29843, "bpb": 1.87324, "chunk_loss": 0.055929, "F": 0.70004, "G": 0.3908, "chunks": 716.84, "bytes_per_chunk": 1.428, "boundary_entropy": 0.66019, "router_noise": 0.0, "grad_norm": 0.818, "lr": 9.12e-05, "router_trainable": true, "sec_per_step": 0.091, "alpha": 0.03, "t": 403.4}
{"event": "train", "step": 4400, "phase": 3, "objective": "global", "lm_loss": 1.37957, "bpb": 1.9903, "chunk_loss": 0.022627, "F": 0.00421, "G": 0.37679, "chunks": 4.31, "bytes_per_chunk": 237.449, "boundary_entropy": 0.65397, "router_noise": 0.0, "grad_norm": 0.8301, "lr": 7.92e-05, "router_trainable": true, "sec_per_step": 0.092, "alpha": 0.03, "t": 421.9}
{"event": "train", "step": 4600, "phase": 3, "objective": "global", "lm_loss": 1.49599, "bpb": 2.15826, "chunk_loss": 0.039673, "F": 0.39044, "G": 0.36615, "chunks": 399.81, "bytes_per_chunk": 2.561, "boundary_entropy": 0.64758, "router_noise": 0.0, "grad_norm": 0.7863, "lr": 6.82e-05, "router_trainable": true, "sec_per_step": 0.089, "alpha": 0.03, "t": 439.6}
{"event": "train", "step": 4800, "phase": 3, "objective": "global", "lm_loss": 1.30487, "bpb": 1.88253, "chunk_loss": 0.022297, "F": 0.00098, "G": 0.38191, "chunks": 1.0, "bytes_per_chunk": 1024.0, "boundary_entropy": 0.65608, "router_noise": 0.0, "grad_norm": 0.8666, "lr": 5.85e-05, "router_trainable": true, "sec_per_step": 0.093, "alpha": 0.03, "t": 458.2}
{"event": "eval", "step": 5000, "phase": 3, "bpb": {"en": 2.10913, "de": 2.31166, "vi": 1.96233, "tr": 2.28259, "fi": 2.47792, "ru": 1.42173, "el": 1.49422, "hi": 1.23133, "ta": 1.01495, "th": 1.14546, "zh": 3.22296, "my": 1.11971}, "bytes_per_chunk": {"en": 48.073, "de": 70.907, "vi": 71.742, "tr": 65.194, "fi": 86.947, "ru": 67.459, "el": 144.831, "hi": 49.21, "ta": 76.92, "th": 118.403, "zh": 149.2, "my": 121.814}, "bpb_mean": 1.81617, "t": 476.3}
{"event": "train", "step": 5000, "phase": 3, "objective": "global", "lm_loss": 1.51414, "bpb": 2.18444, "chunk_loss": 0.025256, "F": 0.05795, "G": 0.36871, "chunks": 59.34, "bytes_per_chunk": 17.255, "boundary_entropy": 0.64801, "router_noise": 0.0, "grad_norm": 0.8675, "lr": 5e-05, "router_trainable": true, "sec_per_step": 0.091, "alpha": 0.03, "t": 476.4}
{"event": "train", "step": 5200, "phase": 3, "objective": "global", "lm_loss": 1.40099, "bpb": 2.02121, "chunk_loss": 0.022523, "F": 0.00351, "G": 0.37881, "chunks": 3.59, "bytes_per_chunk": 284.939, "boundary_entropy": 0.65434, "router_noise": 0.0, "grad_norm": 0.8979, "lr": 4.29e-05, "router_trainable": true, "sec_per_step": 0.084, "alpha": 0.03, "t": 493.2}
{"event": "train", "step": 5400, "phase": 3, "objective": "global", "lm_loss": 1.27771, "bpb": 1.84335, "chunk_loss": 0.022471, "F": 0.00098, "G": 0.37704, "chunks": 1.0, "bytes_per_chunk": 1024.0, "boundary_entropy": 0.65352, "router_noise": 0.0, "grad_norm": 0.8255, "lr": 3.73e-05, "router_trainable": true, "sec_per_step": 0.09, "alpha": 0.03, "t": 511.1}
{"event": "train", "step": 5600, "phase": 3, "objective": "global", "lm_loss": 1.36022, "bpb": 1.96238, "chunk_loss": 0.022816, "F": 0.00098, "G": 0.36739, "chunks": 1.0, "bytes_per_chunk": 1024.0, "boundary_entropy": 0.64794, "router_noise": 0.0, "grad_norm": 0.8147, "lr": 3.33e-05, "router_trainable": true, "sec_per_step": 0.089, "alpha": 0.03, "t": 528.9}
{"event": "train", "step": 5800, "phase": 3, "objective": "global", "lm_loss": 1.38027, "bpb": 1.99131, "chunk_loss": 0.028322, "F": 0.12729, "G": 0.36518, "chunks": 130.34, "bytes_per_chunk": 7.856, "boundary_entropy": 0.64685, "router_noise": 0.0, "grad_norm": 0.8599, "lr": 3.08e-05, "router_trainable": true, "sec_per_step": 0.084, "alpha": 0.03, "t": 545.7}
{"event": "probe", "step": 6000, "phase": 3, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 386.0, "chars": 17403.0, "bytes": 17418.0, "cpc": 0.02218, "cps": 3.015625, "mean_p": 0.372235, "boundary_entropy": 0.642723, "bpb": 2.081353}, "de": {"chunks": 371.0, "chars": 20533.0, "bytes": 20862.0, "cpc": 0.018068, "cps": 2.898438, "mean_p": 0.378678, "boundary_entropy": 0.64967, "bpb": 2.281939}, "vi": {"chunks": 535.0, "chars": 18435.0, "bytes": 24274.0, "cpc": 0.029021, "cps": 4.179688, "mean_p": 0.379752, "boundary_entropy": 0.650654, "bpb": 1.945007}, "tr": {"chunks": 366.0, "chars": 18103.0, "bytes": 19742.0, "cpc": 0.020218, "cps": 2.859375, "mean_p": 0.37595, "boundary_entropy": 0.647683, "bpb": 2.245198}, "fi": {"chunks": 283.0, "chars": 18811.0, "bytes": 19572.0, "cpc": 0.015044, "cps": 2.210938, "mean_p": 0.37811, "boundary_entropy": 0.649958, "bpb": 2.445838}, "ru": {"chunks": 744.0, "chars": 18950.0, "bytes": 34560.0, "cpc": 0.039261, "cps": 5.8125, "mean_p": 0.378459, "boundary_entropy": 0.654131, "bpb": 1.425385}, "el": {"chunks": 375.0, "chars": 20523.0, "bytes": 37264.0, "cpc": 0.018272, "cps": 2.929688, "mean_p": 0.372367, "boundary_entropy": 0.651811, "bpb": 1.473511}, "hi": {"chunks": 921.0, "chars": 16825.0, "bytes": 43173.0, "cpc": 0.05474, "cps": 7.195312, "mean_p": 0.375501, "boundary_entropy": 0.650672, "bpb": 1.217113}, "ta": {"chunks": 707.0, "chars": 18695.0, "bytes": 50915.0, "cpc": 0.037818, "cps": 5.523438, "mean_p": 0.362146, "boundary_entropy": 0.640883, "bpb": 0.992749}, "th": {"chunks": 417.0, "chars": 16085.0, "bytes": 46085.0, "cpc": 0.025925, "cps": 3.257812, "mean_p": 0.370949, "boundary_entropy": 0.649942, "bpb": 1.111332}, "zh": {"chunks": 228.0, "chars": 5906.0, "bytes": 16173.0, "cpc": 0.038605, "cps": 1.78125, "mean_p": 0.375965, "boundary_entropy": 0.648166, "bpb": 3.184903}, "my": {"chunks": 280.0, "chars": 18948.0, "bytes": 53743.0, "cpc": 0.014777, "cps": 2.1875, "mean_p": 0.360745, "boundary_entropy": 0.639457, "bpb": 1.09482}}, "bpb": {"en": 2.08135, "de": 2.28194, "vi": 1.94501, "tr": 2.2452, "fi": 2.44584, "ru": 1.42538, "el": 1.47351, "hi": 1.21711, "ta": 0.99275, "th": 1.11133, "zh": 3.1849, "my": 1.09482}, "gini_cpc": 0.23188, "gini_cps": 0.233995, "bpic": 68.373597, "worst_lang": "hi", "cpc_parity_vs_en": {"en": 1.0, "de": 0.8146260953869533, "vi": 1.3084208484930249, "tr": 0.911522408188721, "fi": 0.6782836797893858, "ru": 1.7701111460483683, "el": 0.823810338912061, "hi": 2.4679785047232636, "ta": 1.7050250337085502, "th": 1.168831225307265, "zh": 1.74051680121559, "my": 0.6662411771075679}, "t": 564.4}
{"event": "eval", "step": 6000, "phase": 3, "bpb": {"en": 2.09161, "de": 2.28849, "vi": 1.94771, "tr": 2.25268, "fi": 2.44957, "ru": 1.4018, "el": 1.466, "hi": 1.21198, "ta": 1.00013, "th": 1.1284, "zh": 3.18371, "my": 1.10389}, "bytes_per_chunk": {"en": 46.996, "de": 68.284, "vi": 72.136, "tr": 83.752, "fi": 113.778, "ru": 66.433, "el": 148.271, "hi": 53.119, "ta": 81.563, "th": 134.158, "zh": 132.53, "my": 205.281}, "bpb_mean": 1.79383, "t": 565.6}
{"event": "probe", "step": 6000, "phase": 3, "objective": "global", "router_trainable": true, "per_lang": {"en": {"chunks": 386.0, "chars": 17403.0, "bytes": 17418.0, "cpc": 0.02218, "cps": 3.015625, "mean_p": 0.372235, "boundary_entropy": 0.642723, "bpb": 2.081353}, "de": {"chunks": 371.0, "chars": 20533.0, "bytes": 20862.0, "cpc": 0.018068, "cps": 2.898438, "mean_p": 0.378678, "boundary_entropy": 0.64967, "bpb": 2.281939}, "vi": {"chunks": 535.0, "chars": 18435.0, "bytes": 24274.0, "cpc": 0.029021, "cps": 4.179688, "mean_p": 0.379752, "boundary_entropy": 0.650654, "bpb": 1.945007}, "tr": {"chunks": 366.0, "chars": 18103.0, "bytes": 19742.0, "cpc": 0.020218, "cps": 2.859375, "mean_p": 0.37595, "boundary_entropy": 0.647683, "bpb": 2.245198}, "fi": {"chunks": 283.0, "chars": 18811.0, "bytes": 19572.0, "cpc": 0.015044, "cps": 2.210938, "mean_p": 0.37811, "boundary_entropy": 0.649958, "bpb": 2.445838}, "ru": {"chunks": 744.0, "chars": 18950.0, "bytes": 34560.0, "cpc": 0.039261, "cps": 5.8125, "mean_p": 0.378459, "boundary_entropy": 0.654131, "bpb": 1.425385}, "el": {"chunks": 375.0, "chars": 20523.0, "bytes": 37264.0, "cpc": 0.018272, "cps": 2.929688, "mean_p": 0.372367, "boundary_entropy": 0.651811, "bpb": 1.473511}, "hi": {"chunks": 921.0, "chars": 16825.0, "bytes": 43173.0, "cpc": 0.05474, "cps": 7.195312, "mean_p": 0.375501, "boundary_entropy": 0.650672, "bpb": 1.217113}, "ta": {"chunks": 707.0, "chars": 18695.0, "bytes": 50915.0, "cpc": 0.037818, "cps": 5.523438, "mean_p": 0.362146, "boundary_entropy": 0.640883, "bpb": 0.992749}, "th": {"chunks": 417.0, "chars": 16085.0, "bytes": 46085.0, "cpc": 0.025925, "cps": 3.257812, "mean_p": 0.370949, "boundary_entropy": 0.649942, "bpb": 1.111332}, "zh": {"chunks": 228.0, "chars": 5906.0, "bytes": 16173.0, "cpc": 0.038605, "cps": 1.78125, "mean_p": 0.375965, "boundary_entropy": 0.648166, "bpb": 3.184903}, "my": {"chunks": 280.0, "chars": 18948.0, "bytes": 53743.0, "cpc": 0.014777, "cps": 2.1875, "mean_p": 0.360745, "boundary_entropy": 0.639457, "bpb": 1.09482}}, "bpb": {"en": 2.08135, "de": 2.28194, "vi": 1.94501, "tr": 2.2452, "fi": 2.44584, "ru": 1.42538, "el": 1.47351, "hi": 1.21711, "ta": 0.99275, "th": 1.11133, "zh": 3.1849, "my": 1.09482}, "gini_cpc": 0.23188, "gini_cps": 0.233995, "bpic": 68.373597, "worst_lang": "hi", "cpc_parity_vs_en": {"en": 1.0, "de": 0.8146260953869533, "vi": 1.3084208484930249, "tr": 0.911522408188721, "fi": 0.6782836797893858, "ru": 1.7701111460483683, "el": 0.823810338912061, "hi": 2.4679785047232636, "ta": 1.7050250337085502, "th": 1.168831225307265, "zh": 1.74051680121559, "my": 0.6662411771075679}, "t": 566.5}
{"event": "reps_dumped", "dirs": ["/local/scratch/sas245/beetle-hnet/scratch/reps/jitter_s1/th0.30", "/local/scratch/sas245/beetle-hnet/scratch/reps/jitter_s1/th0.40", "/local/scratch/sas245/beetle-hnet/scratch/reps/jitter_s1/th0.50", "/local/scratch/sas245/beetle-hnet/scratch/reps/jitter_s1/th0.60", "/local/scratch/sas245/beetle-hnet/scratch/reps/jitter_s1/th0.70"], "t": 569.9}
{"event": "final", "run": "jitter_s1", "condition": "A", "step": 6000, "params": 22516992, "per_lang": {"en": {"cpc": 0.022180083893581566, "cps": 3.015625, "bpb": 2.1020406444164355, "bytes_per_chunk": 46.865826405649415}, "de": {"cpc": 0.018068475137583404, "cps": 2.8984375, "bpb": 2.2796363360511163, "bytes_per_chunk": 68.94897422409258}, "vi": {"cpc": 0.029020884187686467, "cps": 4.1796875, "bpb": 1.9494764901690524, "bytes_per_chunk": 70.51620712844654}, "tr": {"cpc": 0.020217643484505332, "cps": 2.859375, "bpb": 2.2458832574689267, "bytes_per_chunk": 85.72400261608895}, "fi": {"cpc": 0.01504438892137579, "cps": 2.2109375, "bpb": 2.4640121534753403, "bytes_per_chunk": 111.19575821845174}, "ru": {"cpc": 0.03926121372031662, "cps": 5.8125, "bpb": 1.4033274435589955, "bytes_per_chunk": 66.50025367833587}, "el": {"cpc": 0.018272182429469375, "cps": 2.9296875, "bpb": 1.4606392699744462, "bytes_per_chunk": 149.24224309706804}, "hi": {"cpc": 0.05473997028231798, "cps": 7.1953125, "bpb": 1.2202438124075716, "bytes_per_chunk": 53.54248366013072}, "ta": {"cpc": 0.03781759828831238, "cps": 5.5234375, "bpb": 0.9998630346897966, "bytes_per_chunk": 80.46163290362185}, "th": {"cpc": 0.025924774634752876, "cps": 3.2578125, "bpb": 1.126236773329332, "bytes_per_chunk": 134.95186615186614}, "zh": {"cpc": 0.03860480866915002, "cps": 1.78125, "bpb": 3.1923234986783178, "bytes_per_chunk": 134.3639159405433}, "my": {"cpc": 0.01477728520160439, "cps": 2.1875, "bpb": 1.114218518197882, "bytes_per_chunk": 210.81141938078005}}, "gini_cpc": 0.2318796127695224, "gini_cps": 0.2339954866678544, "bpic": 68.37359700694816, "seed": 1, "mean_boundary_entropy": 0.6504928469657898, "dechunk": "ema", "router_noise": 0.0, "transitions": [{"step": 0, "phase": 1, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0}, {"step": 2400, "phase": 2, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0}, {"step": 4200, "phase": 3, "objective": "global", "router_trainable": true, "encoder_trainable": true, "frozen_router_params": 0}], "t": 572.1}