Hugging Face's logo

AdaptiveChunking
/
spec_E_s2

PyTorch
Safetensors
dynamic-chunking
tokenization
hnet
byte-level
beetle-hnet
legacy-trainer
Model card Files
xet
 Community
spec_E_s2
File size: 2,716 Bytes
7032b51
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
{
  "event": "final",
  "run": "spec_E_s2",
  "condition": "E",
  "step": 11000,
  "params": 82470528,
  "per_lang": {
    "en": {
      "cpc": 0.18381888180198816,
      "cps": 24.9921875,
      "bpb": 1.6438765837477893,
      "bytes_per_chunk": 5.193541357107479
    },
    "de": {
      "cpc": 0.1784931573564506,
      "cps": 28.6328125,
      "bpb": 1.7824091142168186,
      "bytes_per_chunk": 5.5738851708722486
    },
    "vi": {
      "cpc": 0.18014646053702196,
      "cps": 25.9453125,
      "bpb": 1.4739747294799386,
      "bytes_per_chunk": 7.134480482809768
    },
    "tr": {
      "cpc": 0.1762138872010164,
      "cps": 24.921875,
      "bpb": 1.6789048384437173,
      "bytes_per_chunk": 5.853976619287413
    },
    "fi": {
      "cpc": 0.1817021955238956,
      "cps": 26.703125,
      "bpb": 1.8691479514912825,
      "bytes_per_chunk": 5.733808700977712
    },
    "ru": {
      "cpc": 0.19593667546174143,
      "cps": 29.0078125,
      "bpb": 1.0907239198933933,
      "bytes_per_chunk": 9.00026608300073
    },
    "el": {
      "cpc": 0.1705890951615261,
      "cps": 27.3515625,
      "bpb": 1.095154461702511,
      "bytes_per_chunk": 10.631410321403225
    },
    "hi": {
      "cpc": 0.2234769687964339,
      "cps": 29.375,
      "bpb": 0.907511446864184,
      "bytes_per_chunk": 11.288362579394983
    },
    "ta": {
      "cpc": 0.17865739502540787,
      "cps": 26.09375,
      "bpb": 0.7897428494116536,
      "bytes_per_chunk": 14.990150248030764
    },
    "th": {
      "cpc": 0.2431457880012434,
      "cps": 30.5546875,
      "bpb": 0.8658079972935708,
      "bytes_per_chunk": 11.621403555437336
    },
    "zh": {
      "cpc": 0.5130375888926515,
      "cps": 23.671875,
      "bpb": 2.479610840172693,
      "bytes_per_chunk": 5.378995480637533
    },
    "my": {
      "cpc": 0.08982478361832383,
      "cps": 13.296875,
      "bpb": 0.8824134811779903,
      "bytes_per_chunk": 27.510126980795466
    }
  },
  "gini_cpc": 0.19030326488159188,
  "gini_cps": 0.07857023060796653,
  "bpic": 9.654867924528302,
  "seed": 2,
  "mean_boundary_entropy": 0.10425905883312225,
  "dechunk": "ema",
  "router_noise": 0.0,
  "transitions": [
    {
      "step": 0,
      "phase": 1,
      "objective": "global",
      "router_trainable": true,
      "encoder_trainable": true,
      "frozen_router_params": 0
    },
    {
      "step": 4400,
      "phase": 2,
      "objective": "parity_b",
      "router_trainable": true,
      "encoder_trainable": true,
      "frozen_router_params": 0
    },
    {
      "step": 7700,
      "phase": 3,
      "objective": "none",
      "router_trainable": false,
      "encoder_trainable": true,
      "frozen_router_params": 295680
    }
  ]
}