Hugging Face's logo

AdaptiveChunking
/
specfull_A_s0

PyTorch
Safetensors
dynamic-chunking
tokenization
hnet
byte-level
beetle-hnet
legacy-trainer
Model card Files
xet
 Community
specfull_A_s0
File size: 2,716 Bytes
1945191
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
{
  "event": "final",
  "run": "specfull_A_s0",
  "condition": "A",
  "step": 76300,
  "params": 82470528,
  "per_lang": {
    "en": {
      "cpc": 0.19588576682181233,
      "cps": 26.6328125,
      "bpb": 1.4036495348173146,
      "bytes_per_chunk": 4.875442291687008
    },
    "de": {
      "cpc": 0.18224321823406225,
      "cps": 29.234375,
      "bpb": 1.520054643064889,
      "bytes_per_chunk": 5.427355824473866
    },
    "vi": {
      "cpc": 0.2300515324111744,
      "cps": 33.1328125,
      "bpb": 1.162168791481086,
      "bytes_per_chunk": 5.583858307550576
    },
    "tr": {
      "cpc": 0.17803678948240623,
      "cps": 25.1796875,
      "bpb": 1.4175154640604777,
      "bytes_per_chunk": 5.832745558312103
    },
    "fi": {
      "cpc": 0.16240497581202487,
      "cps": 23.8671875,
      "bpb": 1.5665407474244302,
      "bytes_per_chunk": 5.9119668480252585
    },
    "ru": {
      "cpc": 0.3062269129287599,
      "cps": 45.3359375,
      "bpb": 0.8785567863557654,
      "bytes_per_chunk": 5.940536620739667
    },
    "el": {
      "cpc": 0.2899186278809141,
      "cps": 46.484375,
      "bpb": 0.8778654053503812,
      "bytes_per_chunk": 6.037193556187603
    },
    "hi": {
      "cpc": 0.4129569093610698,
      "cps": 54.28125,
      "bpb": 0.7355408939059719,
      "bytes_per_chunk": 5.970471510644718
    },
    "ta": {
      "cpc": 0.4371757154319337,
      "cps": 63.8515625,
      "bpb": 0.6337944581817971,
      "bytes_per_chunk": 6.081804050762128
    },
    "th": {
      "cpc": 0.4639726453217283,
      "cps": 58.3046875,
      "bpb": 0.7144447199675241,
      "bytes_per_chunk": 5.935627030759999
    },
    "zh": {
      "cpc": 0.43362682018286486,
      "cps": 20.0078125,
      "bpb": 2.215191857033059,
      "bytes_per_chunk": 6.144276012398994
    },
    "my": {
      "cpc": 0.44637956512560695,
      "cps": 66.078125,
      "bpb": 0.6806133274729266,
      "bytes_per_chunk": 6.294464754152485
    }
  },
  "gini_cpc": 0.2066257765320496,
  "gini_cps": 0.2201340187647427,
  "bpic": 6.0892488814140195,
  "seed": 0,
  "mean_boundary_entropy": 0.1498613953590393,
  "dechunk": "ema",
  "router_noise": 0.0,
  "transitions": [
    {
      "step": 0,
      "phase": 1,
      "objective": "global",
      "router_trainable": true,
      "encoder_trainable": true,
      "frozen_router_params": 0
    },
    {
      "step": 30520,
      "phase": 2,
      "objective": "global",
      "router_trainable": true,
      "encoder_trainable": true,
      "frozen_router_params": 0
    },
    {
      "step": 53410,
      "phase": 3,
      "objective": "global",
      "router_trainable": true,
      "encoder_trainable": true,
      "frozen_router_params": 0
    }
  ]
}