Hugging Face's logo

AdaptiveChunking
/
pilot_E_s0

PyTorch
Safetensors
dynamic-chunking
tokenization
hnet
byte-level
beetle-hnet
legacy-trainer
Model card Files
xet
 Community
pilot_E_s0
File size: 2,712 Bytes
896f21f
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
{
  "event": "final",
  "run": "pilot_E_s0",
  "condition": "E",
  "step": 6000,
  "params": 22516992,
  "per_lang": {
    "en": {
      "cpc": 0.20898695627190714,
      "cps": 28.4140625,
      "bpb": 2.027534847106518,
      "bytes_per_chunk": 4.792351075401505
    },
    "de": {
      "cpc": 0.1766911800516242,
      "cps": 28.34375,
      "bpb": 2.209753539885548,
      "bytes_per_chunk": 5.571071842225504
    },
    "vi": {
      "cpc": 0.19528071602929212,
      "cps": 28.125,
      "bpb": 1.9175251448329642,
      "bytes_per_chunk": 6.9727493981992525
    },
    "tr": {
      "cpc": 0.19184665525051098,
      "cps": 27.1328125,
      "bpb": 2.1790023647295897,
      "bytes_per_chunk": 5.453437210705333
    },
    "fi": {
      "cpc": 0.18303120514592525,
      "cps": 26.8984375,
      "bpb": 2.3653980538947033,
      "bytes_per_chunk": 5.287773194420631
    },
    "ru": {
      "cpc": 0.19693931398416886,
      "cps": 29.15625,
      "bpb": 1.3694184250825936,
      "bytes_per_chunk": 8.874054264484352
    },
    "el": {
      "cpc": 0.16678848121619647,
      "cps": 26.7421875,
      "bpb": 1.4537136629856469,
      "bytes_per_chunk": 10.662544996034248
    },
    "hi": {
      "cpc": 0.19292719167904904,
      "cps": 25.359375,
      "bpb": 1.1981697263057156,
      "bytes_per_chunk": 12.130960919965755
    },
    "ta": {
      "cpc": 0.08558438085049479,
      "cps": 12.5,
      "bpb": 0.9989697245544682,
      "bytes_per_chunk": 32.08224207563334
    },
    "th": {
      "cpc": 0.20285980727385763,
      "cps": 25.4921875,
      "bpb": 1.1227061361153114,
      "bytes_per_chunk": 13.105889411058895
    },
    "zh": {
      "cpc": 0.6197087707416187,
      "cps": 28.59375,
      "bpb": 3.112132933082202,
      "bytes_per_chunk": 4.639224153187272
    },
    "my": {
      "cpc": 0.09927169094363521,
      "cps": 14.6953125,
      "bpb": 1.1179256096116879,
      "bytes_per_chunk": 25.561308566135246
    }
  },
  "gini_cpc": 0.2399013066286535,
  "gini_cps": 0.09629140102627898,
  "bpic": 9.946120354532733,
  "seed": 0,
  "mean_boundary_entropy": 0.3112594485282898,
  "dechunk": "ema",
  "router_noise": 0.0,
  "transitions": [
    {
      "step": 0,
      "phase": 1,
      "objective": "global",
      "router_trainable": true,
      "encoder_trainable": true,
      "frozen_router_params": 0
    },
    {
      "step": 2400,
      "phase": 2,
      "objective": "parity_b",
      "router_trainable": true,
      "encoder_trainable": true,
      "frozen_router_params": 0
    },
    {
      "step": 4200,
      "phase": 3,
      "objective": "none",
      "router_trainable": false,
      "encoder_trainable": true,
      "frozen_router_params": 131584
    }
  ]
}