Hugging Face's logo

iskhare
/
mstok-baselines

Model card Files
xet
 Community
iskhare commited on
Commit
f1aa62c
·
verified ·
1 Parent(s): ae8cd63

Publish TinyStories tokenizer hierarchy audit

Browse files
Files changed (16) hide show
  1. ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/detached_apr-step-82813.json +197 -0
  2. ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/eostok_apr-step-82813.json +197 -0
  3. ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/mstok_v1-step-16563.json +174 -0
  4. ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/mstok_v1-step-33126.json +174 -0
  5. ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/mstok_v1-step-49689.json +174 -0
  6. ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/mstok_v1-step-66252.json +174 -0
  7. ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/mstok_v1-step-82813.json +276 -0
  8. ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/naive_e2e-step-16563.json +96 -0
  9. ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/naive_e2e-step-33126.json +96 -0
  10. ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/naive_e2e-step-49689.json +96 -0
  11. ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/naive_e2e-step-66252.json +96 -0
  12. ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/naive_e2e-step-82813.json +197 -0
  13. ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/two_stage-step-16563.json +96 -0
  14. ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/two_stage-step-82813.json +197 -0
  15. ts-fivearm/analysis/tokenizer-hierarchy-v1/summary.json +774 -0
  16. ts-fivearm/analysis/tokenizer-hierarchy-v1/summary.md +25 -0
ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/detached_apr-step-82813.json ADDED
@@ -0,0 +1,197 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": {
3
+ "bytes": 1484530939,
4
+ "path": "outputs/ncm/tinystories-detached-apr-full-v2-5epoch-ddp/checkpoints/step-82813.pt",
5
+ "step": 82813,
6
+ "training_regime": "detached_apr"
7
+ },
8
+ "code_commit": "0775db5cd9708ed9aed8d69ee130d6119527c433",
9
+ "indices": "outputs/ncm/analysis/tokenizer-hierarchy-v1/raw/detached_apr-step-82813.indices.npz",
10
+ "levels": [
11
+ 1,
12
+ 4,
13
+ 9,
14
+ 16,
15
+ 25,
16
+ 36,
17
+ 49,
18
+ 64
19
+ ],
20
+ "protocol": {
21
+ "batch_size": 64,
22
+ "indices_only": false,
23
+ "n_samples": 4096,
24
+ "sample_sha256": "c8b2a6079cc518751ded337e0ae600824783fed7d234ccb8088735d7c53c5282",
25
+ "seed": 42,
26
+ "split": "data/ncm/tinystories/valid_bpe_8192.pickle"
27
+ },
28
+ "student": {
29
+ "codebooks": [
30
+ {
31
+ "effective_codes": 108.3305596831588,
32
+ "level_index": 0,
33
+ "marginal_entropy_nats": 4.685187290409179,
34
+ "observations": 4096,
35
+ "used_codes": 144,
36
+ "utilization": 0.017578125
37
+ },
38
+ {
39
+ "effective_codes": 870.022809348999,
40
+ "level_index": 1,
41
+ "marginal_entropy_nats": 6.768519428947481,
42
+ "observations": 16384,
43
+ "used_codes": 2083,
44
+ "utilization": 0.2542724609375
45
+ },
46
+ {
47
+ "effective_codes": 1755.575921224304,
48
+ "level_index": 2,
49
+ "marginal_entropy_nats": 7.47055224230894,
50
+ "observations": 36864,
51
+ "used_codes": 4171,
52
+ "utilization": 0.5091552734375
53
+ },
54
+ {
55
+ "effective_codes": 2286.5201701298097,
56
+ "level_index": 3,
57
+ "marginal_entropy_nats": 7.734786364458637,
58
+ "observations": 65536,
59
+ "used_codes": 6123,
60
+ "utilization": 0.7474365234375
61
+ },
62
+ {
63
+ "effective_codes": 2014.7145198608748,
64
+ "level_index": 4,
65
+ "marginal_entropy_nats": 7.60823278685518,
66
+ "observations": 102400,
67
+ "used_codes": 5717,
68
+ "utilization": 0.6978759765625
69
+ },
70
+ {
71
+ "effective_codes": 1856.766660204916,
72
+ "level_index": 5,
73
+ "marginal_entropy_nats": 7.526591899286808,
74
+ "observations": 147456,
75
+ "used_codes": 5261,
76
+ "utilization": 0.6422119140625
77
+ },
78
+ {
79
+ "effective_codes": 1934.3235292000836,
80
+ "level_index": 6,
81
+ "marginal_entropy_nats": 7.567512947027877,
82
+ "observations": 200704,
83
+ "used_codes": 5774,
84
+ "utilization": 0.704833984375
85
+ },
86
+ {
87
+ "effective_codes": 1026.9840729647033,
88
+ "level_index": 7,
89
+ "marginal_entropy_nats": 6.934381701497401,
90
+ "observations": 262144,
91
+ "used_codes": 5254,
92
+ "utilization": 0.641357421875
93
+ }
94
+ ],
95
+ "ncp_by_level": [
96
+ {
97
+ "cross_entropy_nats": 4.974203109741211,
98
+ "level_index": 0,
99
+ "level_size": 1,
100
+ "token_accuracy": 0.013427734375
101
+ },
102
+ {
103
+ "cross_entropy_nats": 7.546774573624134,
104
+ "level_index": 1,
105
+ "level_size": 4,
106
+ "token_accuracy": 0.02734375
107
+ },
108
+ {
109
+ "cross_entropy_nats": 4.525883674621582,
110
+ "level_index": 2,
111
+ "level_size": 9,
112
+ "token_accuracy": 0.22767469618055555
113
+ },
114
+ {
115
+ "cross_entropy_nats": 2.784970488399267,
116
+ "level_index": 3,
117
+ "level_size": 16,
118
+ "token_accuracy": 0.437225341796875
119
+ },
120
+ {
121
+ "cross_entropy_nats": 1.3025939130783082,
122
+ "level_index": 4,
123
+ "level_size": 25,
124
+ "token_accuracy": 0.6915234375
125
+ },
126
+ {
127
+ "cross_entropy_nats": 0.6450617793533537,
128
+ "level_index": 5,
129
+ "level_size": 36,
130
+ "token_accuracy": 0.8096788194444444
131
+ },
132
+ {
133
+ "cross_entropy_nats": 0.679096769313423,
134
+ "level_index": 6,
135
+ "level_size": 49,
136
+ "token_accuracy": 0.8046476403061225
137
+ },
138
+ {
139
+ "cross_entropy_nats": 0.2202611332759261,
140
+ "level_index": 7,
141
+ "level_size": 64,
142
+ "token_accuracy": 0.9302215576171875
143
+ }
144
+ ],
145
+ "prefix_reconstruction": [
146
+ {
147
+ "cross_entropy_nats": 5.318077310919762,
148
+ "level_index": 0,
149
+ "through_level": 1,
150
+ "token_accuracy": 0.09982681274414062
151
+ },
152
+ {
153
+ "cross_entropy_nats": 3.8502018973231316,
154
+ "level_index": 1,
155
+ "through_level": 4,
156
+ "token_accuracy": 0.21706771850585938
157
+ },
158
+ {
159
+ "cross_entropy_nats": 2.6491841562092304,
160
+ "level_index": 2,
161
+ "through_level": 9,
162
+ "token_accuracy": 0.3804435729980469
163
+ },
164
+ {
165
+ "cross_entropy_nats": 1.3218422196805477,
166
+ "level_index": 3,
167
+ "through_level": 16,
168
+ "token_accuracy": 0.6571846008300781
169
+ },
170
+ {
171
+ "cross_entropy_nats": 0.48308326303958893,
172
+ "level_index": 4,
173
+ "through_level": 25,
174
+ "token_accuracy": 0.8490104675292969
175
+ },
176
+ {
177
+ "cross_entropy_nats": 0.1673640706576407,
178
+ "level_index": 5,
179
+ "through_level": 36,
180
+ "token_accuracy": 0.9466361999511719
181
+ },
182
+ {
183
+ "cross_entropy_nats": 0.07380200660554692,
184
+ "level_index": 6,
185
+ "through_level": 49,
186
+ "token_accuracy": 0.9765892028808594
187
+ },
188
+ {
189
+ "cross_entropy_nats": 0.00046051875330022085,
190
+ "level_index": 7,
191
+ "through_level": 64,
192
+ "token_accuracy": 0.9999275207519531
193
+ }
194
+ ],
195
+ "weighted_nsp_nats": 1.3530296683311462
196
+ }
197
+ }
ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/eostok_apr-step-82813.json ADDED
@@ -0,0 +1,197 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": {
3
+ "bytes": 1484530939,
4
+ "path": "outputs/ncm/tinystories-eostok-apr-full-v2-5epoch-ddp/checkpoints/step-82813.pt",
5
+ "step": 82813,
6
+ "training_regime": "eostok_apr"
7
+ },
8
+ "code_commit": "0775db5cd9708ed9aed8d69ee130d6119527c433",
9
+ "indices": "outputs/ncm/analysis/tokenizer-hierarchy-v1/raw/eostok_apr-step-82813.indices.npz",
10
+ "levels": [
11
+ 1,
12
+ 4,
13
+ 9,
14
+ 16,
15
+ 25,
16
+ 36,
17
+ 49,
18
+ 64
19
+ ],
20
+ "protocol": {
21
+ "batch_size": 64,
22
+ "indices_only": false,
23
+ "n_samples": 4096,
24
+ "sample_sha256": "c8b2a6079cc518751ded337e0ae600824783fed7d234ccb8088735d7c53c5282",
25
+ "seed": 42,
26
+ "split": "data/ncm/tinystories/valid_bpe_8192.pickle"
27
+ },
28
+ "student": {
29
+ "codebooks": [
30
+ {
31
+ "effective_codes": 158.5840290455278,
32
+ "level_index": 0,
33
+ "marginal_entropy_nats": 5.066284604543368,
34
+ "observations": 4096,
35
+ "used_codes": 201,
36
+ "utilization": 0.0245361328125
37
+ },
38
+ {
39
+ "effective_codes": 791.4991677368068,
40
+ "level_index": 1,
41
+ "marginal_entropy_nats": 6.6739288278193225,
42
+ "observations": 16384,
43
+ "used_codes": 1884,
44
+ "utilization": 0.22998046875
45
+ },
46
+ {
47
+ "effective_codes": 1620.7665092547272,
48
+ "level_index": 2,
49
+ "marginal_entropy_nats": 7.390654470185381,
50
+ "observations": 36864,
51
+ "used_codes": 3887,
52
+ "utilization": 0.4744873046875
53
+ },
54
+ {
55
+ "effective_codes": 2129.7563938724584,
56
+ "level_index": 3,
57
+ "marginal_entropy_nats": 7.663762883088866,
58
+ "observations": 65536,
59
+ "used_codes": 5852,
60
+ "utilization": 0.71435546875
61
+ },
62
+ {
63
+ "effective_codes": 1868.8846361599249,
64
+ "level_index": 4,
65
+ "marginal_entropy_nats": 7.533097080649446,
66
+ "observations": 102400,
67
+ "used_codes": 5409,
68
+ "utilization": 0.6602783203125
69
+ },
70
+ {
71
+ "effective_codes": 1930.470063106789,
72
+ "level_index": 5,
73
+ "marginal_entropy_nats": 7.565518808257861,
74
+ "observations": 147456,
75
+ "used_codes": 5356,
76
+ "utilization": 0.65380859375
77
+ },
78
+ {
79
+ "effective_codes": 1847.4952180756445,
80
+ "level_index": 6,
81
+ "marginal_entropy_nats": 7.521586064497299,
82
+ "observations": 200704,
83
+ "used_codes": 6118,
84
+ "utilization": 0.746826171875
85
+ },
86
+ {
87
+ "effective_codes": 1056.2838119081666,
88
+ "level_index": 7,
89
+ "marginal_entropy_nats": 6.9625121894330375,
90
+ "observations": 262144,
91
+ "used_codes": 5249,
92
+ "utilization": 0.6407470703125
93
+ }
94
+ ],
95
+ "ncp_by_level": [
96
+ {
97
+ "cross_entropy_nats": 5.3176679611206055,
98
+ "level_index": 0,
99
+ "level_size": 1,
100
+ "token_accuracy": 0.001708984375
101
+ },
102
+ {
103
+ "cross_entropy_nats": 7.429441511631012,
104
+ "level_index": 1,
105
+ "level_size": 4,
106
+ "token_accuracy": 0.02410888671875
107
+ },
108
+ {
109
+ "cross_entropy_nats": 4.4729632006751165,
110
+ "level_index": 2,
111
+ "level_size": 9,
112
+ "token_accuracy": 0.21153428819444445
113
+ },
114
+ {
115
+ "cross_entropy_nats": 2.7938546873629093,
116
+ "level_index": 3,
117
+ "level_size": 16,
118
+ "token_accuracy": 0.431854248046875
119
+ },
120
+ {
121
+ "cross_entropy_nats": 1.1903628182411194,
122
+ "level_index": 4,
123
+ "level_size": 25,
124
+ "token_accuracy": 0.71142578125
125
+ },
126
+ {
127
+ "cross_entropy_nats": 0.5721849567360349,
128
+ "level_index": 5,
129
+ "level_size": 36,
130
+ "token_accuracy": 0.8268703884548612
131
+ },
132
+ {
133
+ "cross_entropy_nats": 0.580585431687686,
134
+ "level_index": 6,
135
+ "level_size": 49,
136
+ "token_accuracy": 0.8219367825255102
137
+ },
138
+ {
139
+ "cross_entropy_nats": 0.16295399656519294,
140
+ "level_index": 7,
141
+ "level_size": 64,
142
+ "token_accuracy": 0.9498405456542969
143
+ }
144
+ ],
145
+ "prefix_reconstruction": [
146
+ {
147
+ "cross_entropy_nats": 5.30550742149353,
148
+ "level_index": 0,
149
+ "through_level": 1,
150
+ "token_accuracy": 0.10834884643554688
151
+ },
152
+ {
153
+ "cross_entropy_nats": 3.8148250617086887,
154
+ "level_index": 1,
155
+ "through_level": 4,
156
+ "token_accuracy": 0.20490264892578125
157
+ },
158
+ {
159
+ "cross_entropy_nats": 2.6031500548124313,
160
+ "level_index": 2,
161
+ "through_level": 9,
162
+ "token_accuracy": 0.372833251953125
163
+ },
164
+ {
165
+ "cross_entropy_nats": 1.271256821230054,
166
+ "level_index": 3,
167
+ "through_level": 16,
168
+ "token_accuracy": 0.6538238525390625
169
+ },
170
+ {
171
+ "cross_entropy_nats": 0.46055822307243943,
172
+ "level_index": 4,
173
+ "through_level": 25,
174
+ "token_accuracy": 0.8547477722167969
175
+ },
176
+ {
177
+ "cross_entropy_nats": 0.16380383376963437,
178
+ "level_index": 5,
179
+ "through_level": 36,
180
+ "token_accuracy": 0.9474754333496094
181
+ },
182
+ {
183
+ "cross_entropy_nats": 0.07394133933121338,
184
+ "level_index": 6,
185
+ "through_level": 49,
186
+ "token_accuracy": 0.9766120910644531
187
+ },
188
+ {
189
+ "cross_entropy_nats": 0.00038653903675367474,
190
+ "level_index": 7,
191
+ "through_level": 64,
192
+ "token_accuracy": 0.99993896484375
193
+ }
194
+ ],
195
+ "weighted_nsp_nats": 1.2859343606978655
196
+ }
197
+ }
ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/mstok_v1-step-16563.json ADDED
@@ -0,0 +1,174 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": {
3
+ "bytes": 1747368555,
4
+ "path": "outputs/ncm/tinystories-mstok-v1-full-v2-5epoch-ddp/checkpoints/step-16563.pt",
5
+ "step": 16563,
6
+ "training_regime": "mstok_v1"
7
+ },
8
+ "code_commit": "0775db5cd9708ed9aed8d69ee130d6119527c433",
9
+ "indices": "outputs/ncm/analysis/tokenizer-hierarchy-v1/raw/mstok_v1-step-16563.indices.npz",
10
+ "levels": [
11
+ 1,
12
+ 4,
13
+ 9,
14
+ 16,
15
+ 25,
16
+ 36,
17
+ 49,
18
+ 64
19
+ ],
20
+ "protocol": {
21
+ "batch_size": 64,
22
+ "indices_only": true,
23
+ "n_samples": 4096,
24
+ "sample_sha256": "c8b2a6079cc518751ded337e0ae600824783fed7d234ccb8088735d7c53c5282",
25
+ "seed": 42,
26
+ "split": "data/ncm/tinystories/valid_bpe_8192.pickle"
27
+ },
28
+ "student": {
29
+ "codebooks": [
30
+ {
31
+ "effective_codes": 67.28908945220576,
32
+ "level_index": 0,
33
+ "marginal_entropy_nats": 4.208998105413112,
34
+ "observations": 4096,
35
+ "used_codes": 130,
36
+ "utilization": 0.015869140625
37
+ },
38
+ {
39
+ "effective_codes": 604.1972895518643,
40
+ "level_index": 1,
41
+ "marginal_entropy_nats": 6.40390078293103,
42
+ "observations": 16384,
43
+ "used_codes": 1313,
44
+ "utilization": 0.1602783203125
45
+ },
46
+ {
47
+ "effective_codes": 1185.5557210385268,
48
+ "level_index": 2,
49
+ "marginal_entropy_nats": 7.0779669065422075,
50
+ "observations": 36864,
51
+ "used_codes": 2685,
52
+ "utilization": 0.3277587890625
53
+ },
54
+ {
55
+ "effective_codes": 1691.4407543185398,
56
+ "level_index": 3,
57
+ "marginal_entropy_nats": 7.4333359620788535,
58
+ "observations": 65536,
59
+ "used_codes": 4316,
60
+ "utilization": 0.52685546875
61
+ },
62
+ {
63
+ "effective_codes": 1470.9084097203133,
64
+ "level_index": 4,
65
+ "marginal_entropy_nats": 7.293635454706678,
66
+ "observations": 102400,
67
+ "used_codes": 4460,
68
+ "utilization": 0.54443359375
69
+ },
70
+ {
71
+ "effective_codes": 1187.2916308523695,
72
+ "level_index": 5,
73
+ "marginal_entropy_nats": 7.079430051756454,
74
+ "observations": 147456,
75
+ "used_codes": 4384,
76
+ "utilization": 0.53515625
77
+ },
78
+ {
79
+ "effective_codes": 1104.1597845506058,
80
+ "level_index": 6,
81
+ "marginal_entropy_nats": 7.006839948747115,
82
+ "observations": 200704,
83
+ "used_codes": 3810,
84
+ "utilization": 0.465087890625
85
+ },
86
+ {
87
+ "effective_codes": 1221.136277148685,
88
+ "level_index": 7,
89
+ "marginal_entropy_nats": 7.107537078979034,
90
+ "observations": 262144,
91
+ "used_codes": 5404,
92
+ "utilization": 0.65966796875
93
+ }
94
+ ]
95
+ },
96
+ "teacher": {
97
+ "codebooks": [
98
+ {
99
+ "effective_codes": 65.74338578090334,
100
+ "level_index": 0,
101
+ "marginal_entropy_nats": 4.185759069509546,
102
+ "observations": 4096,
103
+ "used_codes": 131,
104
+ "utilization": 0.0159912109375
105
+ },
106
+ {
107
+ "effective_codes": 657.4906671376275,
108
+ "level_index": 1,
109
+ "marginal_entropy_nats": 6.488430569483691,
110
+ "observations": 16384,
111
+ "used_codes": 1579,
112
+ "utilization": 0.1927490234375
113
+ },
114
+ {
115
+ "effective_codes": 1205.1847381072175,
116
+ "level_index": 2,
117
+ "marginal_entropy_nats": 7.09438814380658,
118
+ "observations": 36864,
119
+ "used_codes": 2889,
120
+ "utilization": 0.3526611328125
121
+ },
122
+ {
123
+ "effective_codes": 1719.7226333524807,
124
+ "level_index": 3,
125
+ "marginal_entropy_nats": 7.4499182971249684,
126
+ "observations": 65536,
127
+ "used_codes": 4562,
128
+ "utilization": 0.556884765625
129
+ },
130
+ {
131
+ "effective_codes": 1469.3813228034799,
132
+ "level_index": 4,
133
+ "marginal_entropy_nats": 7.292596722341464,
134
+ "observations": 102400,
135
+ "used_codes": 4692,
136
+ "utilization": 0.57275390625
137
+ },
138
+ {
139
+ "effective_codes": 1191.2739281018717,
140
+ "level_index": 5,
141
+ "marginal_entropy_nats": 7.082778541315995,
142
+ "observations": 147456,
143
+ "used_codes": 4598,
144
+ "utilization": 0.561279296875
145
+ },
146
+ {
147
+ "effective_codes": 1121.6896124304778,
148
+ "level_index": 6,
149
+ "marginal_entropy_nats": 7.02259141004886,
150
+ "observations": 200704,
151
+ "used_codes": 4344,
152
+ "utilization": 0.5302734375
153
+ },
154
+ {
155
+ "effective_codes": 1210.786690741935,
156
+ "level_index": 7,
157
+ "marginal_entropy_nats": 7.099025584967836,
158
+ "observations": 262144,
159
+ "used_codes": 5414,
160
+ "utilization": 0.660888671875
161
+ }
162
+ ],
163
+ "student_disagreement": [
164
+ 0.190673828125,
165
+ 0.1688232421875,
166
+ 0.12046983506944445,
167
+ 0.0937957763671875,
168
+ 0.152861328125,
169
+ 0.24761962890625,
170
+ 0.3003726881377551,
171
+ 0.14347076416015625
172
+ ]
173
+ }
174
+ }
ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/mstok_v1-step-33126.json ADDED
@@ -0,0 +1,174 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": {
3
+ "bytes": 1747368619,
4
+ "path": "outputs/ncm/tinystories-mstok-v1-full-v2-5epoch-ddp/checkpoints/step-33126.pt",
5
+ "step": 33126,
6
+ "training_regime": "mstok_v1"
7
+ },
8
+ "code_commit": "0775db5cd9708ed9aed8d69ee130d6119527c433",
9
+ "indices": "outputs/ncm/analysis/tokenizer-hierarchy-v1/raw/mstok_v1-step-33126.indices.npz",
10
+ "levels": [
11
+ 1,
12
+ 4,
13
+ 9,
14
+ 16,
15
+ 25,
16
+ 36,
17
+ 49,
18
+ 64
19
+ ],
20
+ "protocol": {
21
+ "batch_size": 64,
22
+ "indices_only": true,
23
+ "n_samples": 4096,
24
+ "sample_sha256": "c8b2a6079cc518751ded337e0ae600824783fed7d234ccb8088735d7c53c5282",
25
+ "seed": 42,
26
+ "split": "data/ncm/tinystories/valid_bpe_8192.pickle"
27
+ },
28
+ "student": {
29
+ "codebooks": [
30
+ {
31
+ "effective_codes": 77.36690250604393,
32
+ "level_index": 0,
33
+ "marginal_entropy_nats": 4.348559072940819,
34
+ "observations": 4096,
35
+ "used_codes": 132,
36
+ "utilization": 0.01611328125
37
+ },
38
+ {
39
+ "effective_codes": 748.2385458613489,
40
+ "level_index": 1,
41
+ "marginal_entropy_nats": 6.617721838712332,
42
+ "observations": 16384,
43
+ "used_codes": 1636,
44
+ "utilization": 0.19970703125
45
+ },
46
+ {
47
+ "effective_codes": 1468.5459995919919,
48
+ "level_index": 2,
49
+ "marginal_entropy_nats": 7.292028074351005,
50
+ "observations": 36864,
51
+ "used_codes": 3248,
52
+ "utilization": 0.396484375
53
+ },
54
+ {
55
+ "effective_codes": 1979.096068920625,
56
+ "level_index": 3,
57
+ "marginal_entropy_nats": 7.590395488598476,
58
+ "observations": 65536,
59
+ "used_codes": 5120,
60
+ "utilization": 0.625
61
+ },
62
+ {
63
+ "effective_codes": 1706.9720994405463,
64
+ "level_index": 4,
65
+ "marginal_entropy_nats": 7.44247637786915,
66
+ "observations": 102400,
67
+ "used_codes": 4888,
68
+ "utilization": 0.5966796875
69
+ },
70
+ {
71
+ "effective_codes": 1430.6634654829722,
72
+ "level_index": 5,
73
+ "marginal_entropy_nats": 7.2658935775331255,
74
+ "observations": 147456,
75
+ "used_codes": 4643,
76
+ "utilization": 0.5667724609375
77
+ },
78
+ {
79
+ "effective_codes": 1424.0415829660406,
80
+ "level_index": 6,
81
+ "marginal_entropy_nats": 7.261254293066661,
82
+ "observations": 200704,
83
+ "used_codes": 4555,
84
+ "utilization": 0.5560302734375
85
+ },
86
+ {
87
+ "effective_codes": 1272.0726884221888,
88
+ "level_index": 7,
89
+ "marginal_entropy_nats": 7.1484028872533525,
90
+ "observations": 262144,
91
+ "used_codes": 5456,
92
+ "utilization": 0.666015625
93
+ }
94
+ ]
95
+ },
96
+ "teacher": {
97
+ "codebooks": [
98
+ {
99
+ "effective_codes": 71.97549602201998,
100
+ "level_index": 0,
101
+ "marginal_entropy_nats": 4.276325728062127,
102
+ "observations": 4096,
103
+ "used_codes": 132,
104
+ "utilization": 0.01611328125
105
+ },
106
+ {
107
+ "effective_codes": 797.1227036945447,
108
+ "level_index": 1,
109
+ "marginal_entropy_nats": 6.681008623896828,
110
+ "observations": 16384,
111
+ "used_codes": 1827,
112
+ "utilization": 0.2230224609375
113
+ },
114
+ {
115
+ "effective_codes": 1498.1416602879995,
116
+ "level_index": 2,
117
+ "marginal_entropy_nats": 7.311980725886429,
118
+ "observations": 36864,
119
+ "used_codes": 3413,
120
+ "utilization": 0.4166259765625
121
+ },
122
+ {
123
+ "effective_codes": 2010.6609457968189,
124
+ "level_index": 3,
125
+ "marginal_entropy_nats": 7.606218775755432,
126
+ "observations": 65536,
127
+ "used_codes": 5283,
128
+ "utilization": 0.6448974609375
129
+ },
130
+ {
131
+ "effective_codes": 1702.0557019204514,
132
+ "level_index": 4,
133
+ "marginal_entropy_nats": 7.439592035930607,
134
+ "observations": 102400,
135
+ "used_codes": 5024,
136
+ "utilization": 0.61328125
137
+ },
138
+ {
139
+ "effective_codes": 1428.5249356006198,
140
+ "level_index": 5,
141
+ "marginal_entropy_nats": 7.264397677311701,
142
+ "observations": 147456,
143
+ "used_codes": 4787,
144
+ "utilization": 0.5843505859375
145
+ },
146
+ {
147
+ "effective_codes": 1441.690474396341,
148
+ "level_index": 6,
149
+ "marginal_entropy_nats": 7.273571644593659,
150
+ "observations": 200704,
151
+ "used_codes": 4851,
152
+ "utilization": 0.5921630859375
153
+ },
154
+ {
155
+ "effective_codes": 1267.2246718668441,
156
+ "level_index": 7,
157
+ "marginal_entropy_nats": 7.144584490462522,
158
+ "observations": 262144,
159
+ "used_codes": 5463,
160
+ "utilization": 0.6668701171875
161
+ }
162
+ ],
163
+ "student_disagreement": [
164
+ 0.18603515625,
165
+ 0.14520263671875,
166
+ 0.10264756944444445,
167
+ 0.0794219970703125,
168
+ 0.143701171875,
169
+ 0.22047254774305555,
170
+ 0.2733876753826531,
171
+ 0.12766265869140625
172
+ ]
173
+ }
174
+ }
ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/mstok_v1-step-49689.json ADDED
@@ -0,0 +1,174 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": {
3
+ "bytes": 1747368555,
4
+ "path": "outputs/ncm/tinystories-mstok-v1-full-v2-5epoch-ddp/checkpoints/step-49689.pt",
5
+ "step": 49689,
6
+ "training_regime": "mstok_v1"
7
+ },
8
+ "code_commit": "0775db5cd9708ed9aed8d69ee130d6119527c433",
9
+ "indices": "outputs/ncm/analysis/tokenizer-hierarchy-v1/raw/mstok_v1-step-49689.indices.npz",
10
+ "levels": [
11
+ 1,
12
+ 4,
13
+ 9,
14
+ 16,
15
+ 25,
16
+ 36,
17
+ 49,
18
+ 64
19
+ ],
20
+ "protocol": {
21
+ "batch_size": 64,
22
+ "indices_only": true,
23
+ "n_samples": 4096,
24
+ "sample_sha256": "c8b2a6079cc518751ded337e0ae600824783fed7d234ccb8088735d7c53c5282",
25
+ "seed": 42,
26
+ "split": "data/ncm/tinystories/valid_bpe_8192.pickle"
27
+ },
28
+ "student": {
29
+ "codebooks": [
30
+ {
31
+ "effective_codes": 75.49219564678937,
32
+ "level_index": 0,
33
+ "marginal_entropy_nats": 4.324029281995347,
34
+ "observations": 4096,
35
+ "used_codes": 132,
36
+ "utilization": 0.01611328125
37
+ },
38
+ {
39
+ "effective_codes": 839.162517426536,
40
+ "level_index": 1,
41
+ "marginal_entropy_nats": 6.732404391435565,
42
+ "observations": 16384,
43
+ "used_codes": 1823,
44
+ "utilization": 0.2225341796875
45
+ },
46
+ {
47
+ "effective_codes": 1640.3213779933099,
48
+ "level_index": 2,
49
+ "marginal_entropy_nats": 7.4026474638112045,
50
+ "observations": 36864,
51
+ "used_codes": 3654,
52
+ "utilization": 0.446044921875
53
+ },
54
+ {
55
+ "effective_codes": 2158.778743159778,
56
+ "level_index": 3,
57
+ "marginal_entropy_nats": 7.677297944095944,
58
+ "observations": 65536,
59
+ "used_codes": 5536,
60
+ "utilization": 0.67578125
61
+ },
62
+ {
63
+ "effective_codes": 1812.4508233818037,
64
+ "level_index": 4,
65
+ "marginal_entropy_nats": 7.502435254433472,
66
+ "observations": 102400,
67
+ "used_codes": 5165,
68
+ "utilization": 0.6304931640625
69
+ },
70
+ {
71
+ "effective_codes": 1541.3286456863423,
72
+ "level_index": 5,
73
+ "marginal_entropy_nats": 7.340400080386016,
74
+ "observations": 147456,
75
+ "used_codes": 4793,
76
+ "utilization": 0.5850830078125
77
+ },
78
+ {
79
+ "effective_codes": 1613.0441533175006,
80
+ "level_index": 6,
81
+ "marginal_entropy_nats": 7.385878451165009,
82
+ "observations": 200704,
83
+ "used_codes": 5001,
84
+ "utilization": 0.6104736328125
85
+ },
86
+ {
87
+ "effective_codes": 1253.2985092396298,
88
+ "level_index": 7,
89
+ "marginal_entropy_nats": 7.133534162148669,
90
+ "observations": 262144,
91
+ "used_codes": 5483,
92
+ "utilization": 0.6693115234375
93
+ }
94
+ ]
95
+ },
96
+ "teacher": {
97
+ "codebooks": [
98
+ {
99
+ "effective_codes": 84.49171516160146,
100
+ "level_index": 0,
101
+ "marginal_entropy_nats": 4.436653484131527,
102
+ "observations": 4096,
103
+ "used_codes": 131,
104
+ "utilization": 0.0159912109375
105
+ },
106
+ {
107
+ "effective_codes": 848.454901708934,
108
+ "level_index": 1,
109
+ "marginal_entropy_nats": 6.743416932653528,
110
+ "observations": 16384,
111
+ "used_codes": 1969,
112
+ "utilization": 0.2403564453125
113
+ },
114
+ {
115
+ "effective_codes": 1648.1661061010936,
116
+ "level_index": 2,
117
+ "marginal_entropy_nats": 7.407418497927262,
118
+ "observations": 36864,
119
+ "used_codes": 3738,
120
+ "utilization": 0.456298828125
121
+ },
122
+ {
123
+ "effective_codes": 2178.2584445432103,
124
+ "level_index": 3,
125
+ "marginal_entropy_nats": 7.686280957859631,
126
+ "observations": 65536,
127
+ "used_codes": 5656,
128
+ "utilization": 0.6904296875
129
+ },
130
+ {
131
+ "effective_codes": 1864.504414866242,
132
+ "level_index": 4,
133
+ "marginal_entropy_nats": 7.530750567471391,
134
+ "observations": 102400,
135
+ "used_codes": 5266,
136
+ "utilization": 0.642822265625
137
+ },
138
+ {
139
+ "effective_codes": 1575.7742920299768,
140
+ "level_index": 5,
141
+ "marginal_entropy_nats": 7.362502044444454,
142
+ "observations": 147456,
143
+ "used_codes": 4930,
144
+ "utilization": 0.601806640625
145
+ },
146
+ {
147
+ "effective_codes": 1632.2954343718852,
148
+ "level_index": 6,
149
+ "marginal_entropy_nats": 7.397742545103995,
150
+ "observations": 200704,
151
+ "used_codes": 5195,
152
+ "utilization": 0.6341552734375
153
+ },
154
+ {
155
+ "effective_codes": 1258.5116906120913,
156
+ "level_index": 7,
157
+ "marginal_entropy_nats": 7.137685103854507,
158
+ "observations": 262144,
159
+ "used_codes": 5490,
160
+ "utilization": 0.670166015625
161
+ }
162
+ ],
163
+ "student_disagreement": [
164
+ 0.154052734375,
165
+ 0.13409423828125,
166
+ 0.09912109375,
167
+ 0.0752410888671875,
168
+ 0.13662109375,
169
+ 0.21138509114583334,
170
+ 0.26183334661989793,
171
+ 0.11057281494140625
172
+ ]
173
+ }
174
+ }
ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/mstok_v1-step-66252.json ADDED
@@ -0,0 +1,174 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": {
3
+ "bytes": 1747368619,
4
+ "path": "outputs/ncm/tinystories-mstok-v1-full-v2-5epoch-ddp/checkpoints/step-66252.pt",
5
+ "step": 66252,
6
+ "training_regime": "mstok_v1"
7
+ },
8
+ "code_commit": "0775db5cd9708ed9aed8d69ee130d6119527c433",
9
+ "indices": "outputs/ncm/analysis/tokenizer-hierarchy-v1/raw/mstok_v1-step-66252.indices.npz",
10
+ "levels": [
11
+ 1,
12
+ 4,
13
+ 9,
14
+ 16,
15
+ 25,
16
+ 36,
17
+ 49,
18
+ 64
19
+ ],
20
+ "protocol": {
21
+ "batch_size": 64,
22
+ "indices_only": true,
23
+ "n_samples": 4096,
24
+ "sample_sha256": "c8b2a6079cc518751ded337e0ae600824783fed7d234ccb8088735d7c53c5282",
25
+ "seed": 42,
26
+ "split": "data/ncm/tinystories/valid_bpe_8192.pickle"
27
+ },
28
+ "student": {
29
+ "codebooks": [
30
+ {
31
+ "effective_codes": 69.448436419019,
32
+ "level_index": 0,
33
+ "marginal_entropy_nats": 4.240584555181889,
34
+ "observations": 4096,
35
+ "used_codes": 133,
36
+ "utilization": 0.0162353515625
37
+ },
38
+ {
39
+ "effective_codes": 913.5978235967098,
40
+ "level_index": 1,
41
+ "marginal_entropy_nats": 6.817390456671139,
42
+ "observations": 16384,
43
+ "used_codes": 1971,
44
+ "utilization": 0.2406005859375
45
+ },
46
+ {
47
+ "effective_codes": 1721.3542499724708,
48
+ "level_index": 2,
49
+ "marginal_entropy_nats": 7.4508666146416,
50
+ "observations": 36864,
51
+ "used_codes": 3833,
52
+ "utilization": 0.4678955078125
53
+ },
54
+ {
55
+ "effective_codes": 2267.11170310472,
56
+ "level_index": 3,
57
+ "marginal_entropy_nats": 7.726261922829014,
58
+ "observations": 65536,
59
+ "used_codes": 5793,
60
+ "utilization": 0.7071533203125
61
+ },
62
+ {
63
+ "effective_codes": 1896.584036298262,
64
+ "level_index": 4,
65
+ "marginal_entropy_nats": 7.547809671403902,
66
+ "observations": 102400,
67
+ "used_codes": 5294,
68
+ "utilization": 0.646240234375
69
+ },
70
+ {
71
+ "effective_codes": 1649.0840278123192,
72
+ "level_index": 5,
73
+ "marginal_entropy_nats": 7.407975278091131,
74
+ "observations": 147456,
75
+ "used_codes": 4936,
76
+ "utilization": 0.6025390625
77
+ },
78
+ {
79
+ "effective_codes": 1712.742955936204,
80
+ "level_index": 6,
81
+ "marginal_entropy_nats": 7.445851432153024,
82
+ "observations": 200704,
83
+ "used_codes": 5312,
84
+ "utilization": 0.6484375
85
+ },
86
+ {
87
+ "effective_codes": 1254.7366363021547,
88
+ "level_index": 7,
89
+ "marginal_entropy_nats": 7.1346809779918985,
90
+ "observations": 262144,
91
+ "used_codes": 5496,
92
+ "utilization": 0.6708984375
93
+ }
94
+ ]
95
+ },
96
+ "teacher": {
97
+ "codebooks": [
98
+ {
99
+ "effective_codes": 72.43724634960437,
100
+ "level_index": 0,
101
+ "marginal_entropy_nats": 4.282720619449034,
102
+ "observations": 4096,
103
+ "used_codes": 135,
104
+ "utilization": 0.0164794921875
105
+ },
106
+ {
107
+ "effective_codes": 941.2491154752972,
108
+ "level_index": 1,
109
+ "marginal_entropy_nats": 6.847207839379582,
110
+ "observations": 16384,
111
+ "used_codes": 2074,
112
+ "utilization": 0.253173828125
113
+ },
114
+ {
115
+ "effective_codes": 1744.0139037020806,
116
+ "level_index": 2,
117
+ "marginal_entropy_nats": 7.463944576743385,
118
+ "observations": 36864,
119
+ "used_codes": 3931,
120
+ "utilization": 0.4798583984375
121
+ },
122
+ {
123
+ "effective_codes": 2304.2944688240314,
124
+ "level_index": 3,
125
+ "marginal_entropy_nats": 7.742529821298399,
126
+ "observations": 65536,
127
+ "used_codes": 5931,
128
+ "utilization": 0.7239990234375
129
+ },
130
+ {
131
+ "effective_codes": 1913.943459633333,
132
+ "level_index": 4,
133
+ "marginal_entropy_nats": 7.556921031155117,
134
+ "observations": 102400,
135
+ "used_codes": 5405,
136
+ "utilization": 0.6597900390625
137
+ },
138
+ {
139
+ "effective_codes": 1686.7743213288984,
140
+ "level_index": 5,
141
+ "marginal_entropy_nats": 7.430573298451394,
142
+ "observations": 147456,
143
+ "used_codes": 5089,
144
+ "utilization": 0.6212158203125
145
+ },
146
+ {
147
+ "effective_codes": 1736.2791636401194,
148
+ "level_index": 6,
149
+ "marginal_entropy_nats": 7.459499690840443,
150
+ "observations": 200704,
151
+ "used_codes": 5458,
152
+ "utilization": 0.666259765625
153
+ },
154
+ {
155
+ "effective_codes": 1259.7185852381758,
156
+ "level_index": 7,
157
+ "marginal_entropy_nats": 7.138643629951237,
158
+ "observations": 262144,
159
+ "used_codes": 5502,
160
+ "utilization": 0.671630859375
161
+ }
162
+ ],
163
+ "student_disagreement": [
164
+ 0.10986328125,
165
+ 0.13531494140625,
166
+ 0.0920681423611111,
167
+ 0.06634521484375,
168
+ 0.1206640625,
169
+ 0.19108751085069445,
170
+ 0.2456951530612245,
171
+ 0.102874755859375
172
+ ]
173
+ }
174
+ }
ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/mstok_v1-step-82813.json ADDED
@@ -0,0 +1,276 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": {
3
+ "bytes": 1747368555,
4
+ "path": "outputs/ncm/tinystories-mstok-v1-full-v2-5epoch-ddp/checkpoints/step-82813.pt",
5
+ "step": 82813,
6
+ "training_regime": "mstok_v1"
7
+ },
8
+ "code_commit": "0775db5cd9708ed9aed8d69ee130d6119527c433",
9
+ "indices": "outputs/ncm/analysis/tokenizer-hierarchy-v1/raw/mstok_v1-step-82813.indices.npz",
10
+ "levels": [
11
+ 1,
12
+ 4,
13
+ 9,
14
+ 16,
15
+ 25,
16
+ 36,
17
+ 49,
18
+ 64
19
+ ],
20
+ "protocol": {
21
+ "batch_size": 64,
22
+ "indices_only": false,
23
+ "n_samples": 4096,
24
+ "sample_sha256": "c8b2a6079cc518751ded337e0ae600824783fed7d234ccb8088735d7c53c5282",
25
+ "seed": 42,
26
+ "split": "data/ncm/tinystories/valid_bpe_8192.pickle"
27
+ },
28
+ "student": {
29
+ "codebooks": [
30
+ {
31
+ "effective_codes": 69.68504650388776,
32
+ "level_index": 0,
33
+ "marginal_entropy_nats": 4.243985753915553,
34
+ "observations": 4096,
35
+ "used_codes": 136,
36
+ "utilization": 0.0166015625
37
+ },
38
+ {
39
+ "effective_codes": 966.7419966958126,
40
+ "level_index": 1,
41
+ "marginal_entropy_nats": 6.873931651886615,
42
+ "observations": 16384,
43
+ "used_codes": 2026,
44
+ "utilization": 0.247314453125
45
+ },
46
+ {
47
+ "effective_codes": 1789.2728736744123,
48
+ "level_index": 2,
49
+ "marginal_entropy_nats": 7.489564600482452,
50
+ "observations": 36864,
51
+ "used_codes": 3973,
52
+ "utilization": 0.4849853515625
53
+ },
54
+ {
55
+ "effective_codes": 2331.341263046575,
56
+ "level_index": 3,
57
+ "marginal_entropy_nats": 7.754199030313414,
58
+ "observations": 65536,
59
+ "used_codes": 5980,
60
+ "utilization": 0.72998046875
61
+ },
62
+ {
63
+ "effective_codes": 1957.1131990780943,
64
+ "level_index": 4,
65
+ "marginal_entropy_nats": 7.5792258088903655,
66
+ "observations": 102400,
67
+ "used_codes": 5418,
68
+ "utilization": 0.661376953125
69
+ },
70
+ {
71
+ "effective_codes": 1720.1990511441002,
72
+ "level_index": 5,
73
+ "marginal_entropy_nats": 7.450195290520959,
74
+ "observations": 147456,
75
+ "used_codes": 5064,
76
+ "utilization": 0.6181640625
77
+ },
78
+ {
79
+ "effective_codes": 1802.5322817865544,
80
+ "level_index": 6,
81
+ "marginal_entropy_nats": 7.496947778450357,
82
+ "observations": 200704,
83
+ "used_codes": 5492,
84
+ "utilization": 0.67041015625
85
+ },
86
+ {
87
+ "effective_codes": 1252.0793852276154,
88
+ "level_index": 7,
89
+ "marginal_entropy_nats": 7.132560956381244,
90
+ "observations": 262144,
91
+ "used_codes": 5513,
92
+ "utilization": 0.6729736328125
93
+ }
94
+ ],
95
+ "ncp_by_level": [
96
+ {
97
+ "cross_entropy_nats": 5.1687397956848145,
98
+ "level_index": 0,
99
+ "level_size": 1,
100
+ "token_accuracy": 0.006103515625
101
+ },
102
+ {
103
+ "cross_entropy_nats": 7.271853156387806,
104
+ "level_index": 1,
105
+ "level_size": 4,
106
+ "token_accuracy": 0.03570556640625
107
+ },
108
+ {
109
+ "cross_entropy_nats": 4.608915514416164,
110
+ "level_index": 2,
111
+ "level_size": 9,
112
+ "token_accuracy": 0.20939127604166666
113
+ },
114
+ {
115
+ "cross_entropy_nats": 2.803427193313837,
116
+ "level_index": 3,
117
+ "level_size": 16,
118
+ "token_accuracy": 0.4294586181640625
119
+ },
120
+ {
121
+ "cross_entropy_nats": 1.2834816360473633,
122
+ "level_index": 4,
123
+ "level_size": 25,
124
+ "token_accuracy": 0.694091796875
125
+ },
126
+ {
127
+ "cross_entropy_nats": 0.4765111414922608,
128
+ "level_index": 5,
129
+ "level_size": 36,
130
+ "token_accuracy": 0.8569403754340278
131
+ },
132
+ {
133
+ "cross_entropy_nats": 0.3875969636196993,
134
+ "level_index": 6,
135
+ "level_size": 49,
136
+ "token_accuracy": 0.8817512356505102
137
+ },
138
+ {
139
+ "cross_entropy_nats": 0.07621806376846507,
140
+ "level_index": 7,
141
+ "level_size": 64,
142
+ "token_accuracy": 0.9738731384277344
143
+ }
144
+ ],
145
+ "prefix_reconstruction": [
146
+ {
147
+ "cross_entropy_nats": 5.167205706238747,
148
+ "level_index": 0,
149
+ "through_level": 1,
150
+ "token_accuracy": 0.12995529174804688
151
+ },
152
+ {
153
+ "cross_entropy_nats": 3.917147032916546,
154
+ "level_index": 1,
155
+ "through_level": 4,
156
+ "token_accuracy": 0.21495819091796875
157
+ },
158
+ {
159
+ "cross_entropy_nats": 2.7244248501956463,
160
+ "level_index": 2,
161
+ "through_level": 9,
162
+ "token_accuracy": 0.36641693115234375
163
+ },
164
+ {
165
+ "cross_entropy_nats": 1.314411474391818,
166
+ "level_index": 3,
167
+ "through_level": 16,
168
+ "token_accuracy": 0.6632041931152344
169
+ },
170
+ {
171
+ "cross_entropy_nats": 0.47825157130137086,
172
+ "level_index": 4,
173
+ "through_level": 25,
174
+ "token_accuracy": 0.8531761169433594
175
+ },
176
+ {
177
+ "cross_entropy_nats": 0.15295705071184784,
178
+ "level_index": 5,
179
+ "through_level": 36,
180
+ "token_accuracy": 0.9521369934082031
181
+ },
182
+ {
183
+ "cross_entropy_nats": 0.06281172827584669,
184
+ "level_index": 6,
185
+ "through_level": 49,
186
+ "token_accuracy": 0.9808197021484375
187
+ },
188
+ {
189
+ "cross_entropy_nats": 0.0008627551530935307,
190
+ "level_index": 7,
191
+ "through_level": 64,
192
+ "token_accuracy": 0.9998588562011719
193
+ }
194
+ ],
195
+ "weighted_nsp_nats": 1.217957016080618
196
+ },
197
+ "teacher": {
198
+ "codebooks": [
199
+ {
200
+ "effective_codes": 76.27400016575254,
201
+ "level_index": 0,
202
+ "marginal_entropy_nats": 4.334332122234322,
203
+ "observations": 4096,
204
+ "used_codes": 132,
205
+ "utilization": 0.01611328125
206
+ },
207
+ {
208
+ "effective_codes": 988.0211590307065,
209
+ "level_index": 1,
210
+ "marginal_entropy_nats": 6.89570411354153,
211
+ "observations": 16384,
212
+ "used_codes": 2171,
213
+ "utilization": 0.2650146484375
214
+ },
215
+ {
216
+ "effective_codes": 1807.9592655179672,
217
+ "level_index": 2,
218
+ "marginal_entropy_nats": 7.499954010564447,
219
+ "observations": 36864,
220
+ "used_codes": 4076,
221
+ "utilization": 0.49755859375
222
+ },
223
+ {
224
+ "effective_codes": 2375.7904168975747,
225
+ "level_index": 3,
226
+ "marginal_entropy_nats": 7.773085468215506,
227
+ "observations": 65536,
228
+ "used_codes": 6092,
229
+ "utilization": 0.74365234375
230
+ },
231
+ {
232
+ "effective_codes": 1993.4214582335937,
233
+ "level_index": 4,
234
+ "marginal_entropy_nats": 7.597607767115527,
235
+ "observations": 102400,
236
+ "used_codes": 5499,
237
+ "utilization": 0.6712646484375
238
+ },
239
+ {
240
+ "effective_codes": 1746.7027662192083,
241
+ "level_index": 5,
242
+ "marginal_entropy_nats": 7.4654851561161575,
243
+ "observations": 147456,
244
+ "used_codes": 5203,
245
+ "utilization": 0.6351318359375
246
+ },
247
+ {
248
+ "effective_codes": 1818.3455757758572,
249
+ "level_index": 6,
250
+ "marginal_entropy_nats": 7.505682342358719,
251
+ "observations": 200704,
252
+ "used_codes": 5634,
253
+ "utilization": 0.687744140625
254
+ },
255
+ {
256
+ "effective_codes": 1259.9655546128,
257
+ "level_index": 7,
258
+ "marginal_entropy_nats": 7.1388396619629555,
259
+ "observations": 262144,
260
+ "used_codes": 5513,
261
+ "utilization": 0.6729736328125
262
+ }
263
+ ],
264
+ "student_disagreement": [
265
+ 0.132080078125,
266
+ 0.13287353515625,
267
+ 0.0932888454861111,
268
+ 0.06683349609375,
269
+ 0.12109375,
270
+ 0.19209798177083334,
271
+ 0.24783262914540816,
272
+ 0.10200881958007812
273
+ ],
274
+ "weighted_nsp_nats": 1.1813653074204922
275
+ }
276
+ }
ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/naive_e2e-step-16563.json ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": {
3
+ "bytes": 1484531003,
4
+ "path": "outputs/ncm/tinystories-naive-e2e-full-v2-5epoch-ddp/checkpoints/step-16563.pt",
5
+ "step": 16563,
6
+ "training_regime": "naive_e2e"
7
+ },
8
+ "code_commit": "0775db5cd9708ed9aed8d69ee130d6119527c433",
9
+ "indices": "outputs/ncm/analysis/tokenizer-hierarchy-v1/raw/naive_e2e-step-16563.indices.npz",
10
+ "levels": [
11
+ 1,
12
+ 4,
13
+ 9,
14
+ 16,
15
+ 25,
16
+ 36,
17
+ 49,
18
+ 64
19
+ ],
20
+ "protocol": {
21
+ "batch_size": 64,
22
+ "indices_only": true,
23
+ "n_samples": 4096,
24
+ "sample_sha256": "c8b2a6079cc518751ded337e0ae600824783fed7d234ccb8088735d7c53c5282",
25
+ "seed": 42,
26
+ "split": "data/ncm/tinystories/valid_bpe_8192.pickle"
27
+ },
28
+ "student": {
29
+ "codebooks": [
30
+ {
31
+ "effective_codes": 20.121468779605593,
32
+ "level_index": 0,
33
+ "marginal_entropy_nats": 3.00178734354153,
34
+ "observations": 4096,
35
+ "used_codes": 75,
36
+ "utilization": 0.0091552734375
37
+ },
38
+ {
39
+ "effective_codes": 521.4961541725338,
40
+ "level_index": 1,
41
+ "marginal_entropy_nats": 6.256701899864323,
42
+ "observations": 16384,
43
+ "used_codes": 1196,
44
+ "utilization": 0.14599609375
45
+ },
46
+ {
47
+ "effective_codes": 1105.2735066475032,
48
+ "level_index": 2,
49
+ "marginal_entropy_nats": 7.00784810065255,
50
+ "observations": 36864,
51
+ "used_codes": 2565,
52
+ "utilization": 0.3131103515625
53
+ },
54
+ {
55
+ "effective_codes": 1554.5037261655407,
56
+ "level_index": 3,
57
+ "marginal_entropy_nats": 7.348911626514617,
58
+ "observations": 65536,
59
+ "used_codes": 4295,
60
+ "utilization": 0.5242919921875
61
+ },
62
+ {
63
+ "effective_codes": 1468.668732825885,
64
+ "level_index": 4,
65
+ "marginal_entropy_nats": 7.292111645519711,
66
+ "observations": 102400,
67
+ "used_codes": 4457,
68
+ "utilization": 0.5440673828125
69
+ },
70
+ {
71
+ "effective_codes": 1287.587461370222,
72
+ "level_index": 5,
73
+ "marginal_entropy_nats": 7.160525561392012,
74
+ "observations": 147456,
75
+ "used_codes": 4533,
76
+ "utilization": 0.5533447265625
77
+ },
78
+ {
79
+ "effective_codes": 1278.8541285441574,
80
+ "level_index": 6,
81
+ "marginal_entropy_nats": 7.153719743897153,
82
+ "observations": 200704,
83
+ "used_codes": 4159,
84
+ "utilization": 0.5076904296875
85
+ },
86
+ {
87
+ "effective_codes": 1043.006674783982,
88
+ "level_index": 7,
89
+ "marginal_entropy_nats": 6.949862854581428,
90
+ "observations": 262144,
91
+ "used_codes": 5206,
92
+ "utilization": 0.635498046875
93
+ }
94
+ ]
95
+ }
96
+ }
ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/naive_e2e-step-33126.json ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": {
3
+ "bytes": 1484531003,
4
+ "path": "outputs/ncm/tinystories-naive-e2e-full-v2-5epoch-ddp/checkpoints/step-33126.pt",
5
+ "step": 33126,
6
+ "training_regime": "naive_e2e"
7
+ },
8
+ "code_commit": "0775db5cd9708ed9aed8d69ee130d6119527c433",
9
+ "indices": "outputs/ncm/analysis/tokenizer-hierarchy-v1/raw/naive_e2e-step-33126.indices.npz",
10
+ "levels": [
11
+ 1,
12
+ 4,
13
+ 9,
14
+ 16,
15
+ 25,
16
+ 36,
17
+ 49,
18
+ 64
19
+ ],
20
+ "protocol": {
21
+ "batch_size": 64,
22
+ "indices_only": true,
23
+ "n_samples": 4096,
24
+ "sample_sha256": "c8b2a6079cc518751ded337e0ae600824783fed7d234ccb8088735d7c53c5282",
25
+ "seed": 42,
26
+ "split": "data/ncm/tinystories/valid_bpe_8192.pickle"
27
+ },
28
+ "student": {
29
+ "codebooks": [
30
+ {
31
+ "effective_codes": 28.557763401494782,
32
+ "level_index": 0,
33
+ "marginal_entropy_nats": 3.351928822132226,
34
+ "observations": 4096,
35
+ "used_codes": 103,
36
+ "utilization": 0.0125732421875
37
+ },
38
+ {
39
+ "effective_codes": 656.1985024002424,
40
+ "level_index": 1,
41
+ "marginal_entropy_nats": 6.486463338293739,
42
+ "observations": 16384,
43
+ "used_codes": 1527,
44
+ "utilization": 0.1864013671875
45
+ },
46
+ {
47
+ "effective_codes": 1345.8468551354667,
48
+ "level_index": 2,
49
+ "marginal_entropy_nats": 7.20477872597181,
50
+ "observations": 36864,
51
+ "used_codes": 3186,
52
+ "utilization": 0.388916015625
53
+ },
54
+ {
55
+ "effective_codes": 1892.3973800319613,
56
+ "level_index": 3,
57
+ "marginal_entropy_nats": 7.545599759287614,
58
+ "observations": 65536,
59
+ "used_codes": 5161,
60
+ "utilization": 0.6300048828125
61
+ },
62
+ {
63
+ "effective_codes": 1668.2064135413514,
64
+ "level_index": 4,
65
+ "marginal_entropy_nats": 7.419504324387911,
66
+ "observations": 102400,
67
+ "used_codes": 4996,
68
+ "utilization": 0.60986328125
69
+ },
70
+ {
71
+ "effective_codes": 1566.2549124590562,
72
+ "level_index": 5,
73
+ "marginal_entropy_nats": 7.356442642651684,
74
+ "observations": 147456,
75
+ "used_codes": 4871,
76
+ "utilization": 0.5946044921875
77
+ },
78
+ {
79
+ "effective_codes": 1528.0879949258792,
80
+ "level_index": 6,
81
+ "marginal_entropy_nats": 7.3317725563706,
82
+ "observations": 200704,
83
+ "used_codes": 4957,
84
+ "utilization": 0.6051025390625
85
+ },
86
+ {
87
+ "effective_codes": 1098.190804495899,
88
+ "level_index": 7,
89
+ "marginal_entropy_nats": 7.001419381558851,
90
+ "observations": 262144,
91
+ "used_codes": 5264,
92
+ "utilization": 0.642578125
93
+ }
94
+ ]
95
+ }
96
+ }
ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/naive_e2e-step-49689.json ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": {
3
+ "bytes": 1484530939,
4
+ "path": "outputs/ncm/tinystories-naive-e2e-full-v2-5epoch-ddp/checkpoints/step-49689.pt",
5
+ "step": 49689,
6
+ "training_regime": "naive_e2e"
7
+ },
8
+ "code_commit": "0775db5cd9708ed9aed8d69ee130d6119527c433",
9
+ "indices": "outputs/ncm/analysis/tokenizer-hierarchy-v1/raw/naive_e2e-step-49689.indices.npz",
10
+ "levels": [
11
+ 1,
12
+ 4,
13
+ 9,
14
+ 16,
15
+ 25,
16
+ 36,
17
+ 49,
18
+ 64
19
+ ],
20
+ "protocol": {
21
+ "batch_size": 64,
22
+ "indices_only": true,
23
+ "n_samples": 4096,
24
+ "sample_sha256": "c8b2a6079cc518751ded337e0ae600824783fed7d234ccb8088735d7c53c5282",
25
+ "seed": 42,
26
+ "split": "data/ncm/tinystories/valid_bpe_8192.pickle"
27
+ },
28
+ "student": {
29
+ "codebooks": [
30
+ {
31
+ "effective_codes": 59.33344990760548,
32
+ "level_index": 0,
33
+ "marginal_entropy_nats": 4.083173226356969,
34
+ "observations": 4096,
35
+ "used_codes": 131,
36
+ "utilization": 0.0159912109375
37
+ },
38
+ {
39
+ "effective_codes": 725.5378930552196,
40
+ "level_index": 1,
41
+ "marginal_entropy_nats": 6.586913301222481,
42
+ "observations": 16384,
43
+ "used_codes": 1719,
44
+ "utilization": 0.2098388671875
45
+ },
46
+ {
47
+ "effective_codes": 1494.3550765337498,
48
+ "level_index": 2,
49
+ "marginal_entropy_nats": 7.309450005816728,
50
+ "observations": 36864,
51
+ "used_codes": 3552,
52
+ "utilization": 0.43359375
53
+ },
54
+ {
55
+ "effective_codes": 2070.6898562359584,
56
+ "level_index": 3,
57
+ "marginal_entropy_nats": 7.635637094621448,
58
+ "observations": 65536,
59
+ "used_codes": 5622,
60
+ "utilization": 0.686279296875
61
+ },
62
+ {
63
+ "effective_codes": 1814.4010648986693,
64
+ "level_index": 4,
65
+ "marginal_entropy_nats": 7.503510700448312,
66
+ "observations": 102400,
67
+ "used_codes": 5253,
68
+ "utilization": 0.6412353515625
69
+ },
70
+ {
71
+ "effective_codes": 1710.4382992087862,
72
+ "level_index": 5,
73
+ "marginal_entropy_nats": 7.44450493198032,
74
+ "observations": 147456,
75
+ "used_codes": 5024,
76
+ "utilization": 0.61328125
77
+ },
78
+ {
79
+ "effective_codes": 1670.026650470522,
80
+ "level_index": 6,
81
+ "marginal_entropy_nats": 7.4205948636490495,
82
+ "observations": 200704,
83
+ "used_codes": 5449,
84
+ "utilization": 0.6651611328125
85
+ },
86
+ {
87
+ "effective_codes": 1116.7667100818362,
88
+ "level_index": 7,
89
+ "marginal_entropy_nats": 7.01819292325602,
90
+ "observations": 262144,
91
+ "used_codes": 5289,
92
+ "utilization": 0.6456298828125
93
+ }
94
+ ]
95
+ }
96
+ }
ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/naive_e2e-step-66252.json ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": {
3
+ "bytes": 1484531067,
4
+ "path": "outputs/ncm/tinystories-naive-e2e-full-v2-5epoch-ddp/checkpoints/step-66252.pt",
5
+ "step": 66252,
6
+ "training_regime": "naive_e2e"
7
+ },
8
+ "code_commit": "0775db5cd9708ed9aed8d69ee130d6119527c433",
9
+ "indices": "outputs/ncm/analysis/tokenizer-hierarchy-v1/raw/naive_e2e-step-66252.indices.npz",
10
+ "levels": [
11
+ 1,
12
+ 4,
13
+ 9,
14
+ 16,
15
+ 25,
16
+ 36,
17
+ 49,
18
+ 64
19
+ ],
20
+ "protocol": {
21
+ "batch_size": 64,
22
+ "indices_only": true,
23
+ "n_samples": 4096,
24
+ "sample_sha256": "c8b2a6079cc518751ded337e0ae600824783fed7d234ccb8088735d7c53c5282",
25
+ "seed": 42,
26
+ "split": "data/ncm/tinystories/valid_bpe_8192.pickle"
27
+ },
28
+ "student": {
29
+ "codebooks": [
30
+ {
31
+ "effective_codes": 114.41751145044778,
32
+ "level_index": 0,
33
+ "marginal_entropy_nats": 4.739854139350655,
34
+ "observations": 4096,
35
+ "used_codes": 176,
36
+ "utilization": 0.021484375
37
+ },
38
+ {
39
+ "effective_codes": 777.2280948452863,
40
+ "level_index": 1,
41
+ "marginal_entropy_nats": 6.655733865647265,
42
+ "observations": 16384,
43
+ "used_codes": 1858,
44
+ "utilization": 0.226806640625
45
+ },
46
+ {
47
+ "effective_codes": 1574.893396396412,
48
+ "level_index": 2,
49
+ "marginal_entropy_nats": 7.361942864141337,
50
+ "observations": 36864,
51
+ "used_codes": 3756,
52
+ "utilization": 0.45849609375
53
+ },
54
+ {
55
+ "effective_codes": 2184.7778765389235,
56
+ "level_index": 3,
57
+ "marginal_entropy_nats": 7.689269444027543,
58
+ "observations": 65536,
59
+ "used_codes": 5830,
60
+ "utilization": 0.711669921875
61
+ },
62
+ {
63
+ "effective_codes": 1918.217640704595,
64
+ "level_index": 4,
65
+ "marginal_entropy_nats": 7.559151721740248,
66
+ "observations": 102400,
67
+ "used_codes": 5408,
68
+ "utilization": 0.66015625
69
+ },
70
+ {
71
+ "effective_codes": 1813.0875854535543,
72
+ "level_index": 5,
73
+ "marginal_entropy_nats": 7.502786519270025,
74
+ "observations": 147456,
75
+ "used_codes": 5146,
76
+ "utilization": 0.628173828125
77
+ },
78
+ {
79
+ "effective_codes": 1744.1940764382023,
80
+ "level_index": 6,
81
+ "marginal_entropy_nats": 7.464047880638857,
82
+ "observations": 200704,
83
+ "used_codes": 5737,
84
+ "utilization": 0.7003173828125
85
+ },
86
+ {
87
+ "effective_codes": 1113.4096867892742,
88
+ "level_index": 7,
89
+ "marginal_entropy_nats": 7.015182375899521,
90
+ "observations": 262144,
91
+ "used_codes": 5308,
92
+ "utilization": 0.64794921875
93
+ }
94
+ ]
95
+ }
96
+ }
ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/naive_e2e-step-82813.json ADDED
@@ -0,0 +1,197 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": {
3
+ "bytes": 1484530939,
4
+ "path": "outputs/ncm/tinystories-naive-e2e-full-v2-5epoch-ddp/checkpoints/step-82813.pt",
5
+ "step": 82813,
6
+ "training_regime": "naive_e2e"
7
+ },
8
+ "code_commit": "0775db5cd9708ed9aed8d69ee130d6119527c433",
9
+ "indices": "outputs/ncm/analysis/tokenizer-hierarchy-v1/raw/naive_e2e-step-82813.indices.npz",
10
+ "levels": [
11
+ 1,
12
+ 4,
13
+ 9,
14
+ 16,
15
+ 25,
16
+ 36,
17
+ 49,
18
+ 64
19
+ ],
20
+ "protocol": {
21
+ "batch_size": 64,
22
+ "indices_only": false,
23
+ "n_samples": 4096,
24
+ "sample_sha256": "c8b2a6079cc518751ded337e0ae600824783fed7d234ccb8088735d7c53c5282",
25
+ "seed": 42,
26
+ "split": "data/ncm/tinystories/valid_bpe_8192.pickle"
27
+ },
28
+ "student": {
29
+ "codebooks": [
30
+ {
31
+ "effective_codes": 111.16375626606238,
32
+ "level_index": 0,
33
+ "marginal_entropy_nats": 4.711004395829672,
34
+ "observations": 4096,
35
+ "used_codes": 190,
36
+ "utilization": 0.023193359375
37
+ },
38
+ {
39
+ "effective_codes": 807.5258724827488,
40
+ "level_index": 1,
41
+ "marginal_entropy_nats": 6.693975094809849,
42
+ "observations": 16384,
43
+ "used_codes": 1944,
44
+ "utilization": 0.2373046875
45
+ },
46
+ {
47
+ "effective_codes": 1639.721897525267,
48
+ "level_index": 2,
49
+ "marginal_entropy_nats": 7.402281931759117,
50
+ "observations": 36864,
51
+ "used_codes": 3930,
52
+ "utilization": 0.479736328125
53
+ },
54
+ {
55
+ "effective_codes": 2233.3919303387443,
56
+ "level_index": 3,
57
+ "marginal_entropy_nats": 7.711276753831861,
58
+ "observations": 65536,
59
+ "used_codes": 5976,
60
+ "utilization": 0.7294921875
61
+ },
62
+ {
63
+ "effective_codes": 1977.4127190623635,
64
+ "level_index": 4,
65
+ "marginal_entropy_nats": 7.58954456165756,
66
+ "observations": 102400,
67
+ "used_codes": 5485,
68
+ "utilization": 0.6695556640625
69
+ },
70
+ {
71
+ "effective_codes": 1873.9451470480856,
72
+ "level_index": 5,
73
+ "marginal_entropy_nats": 7.535801191851347,
74
+ "observations": 147456,
75
+ "used_codes": 5272,
76
+ "utilization": 0.6435546875
77
+ },
78
+ {
79
+ "effective_codes": 1798.3059801141767,
80
+ "level_index": 6,
81
+ "marginal_entropy_nats": 7.494600378591956,
82
+ "observations": 200704,
83
+ "used_codes": 5927,
84
+ "utilization": 0.7235107421875
85
+ },
86
+ {
87
+ "effective_codes": 1121.9008024531531,
88
+ "level_index": 7,
89
+ "marginal_entropy_nats": 7.022779670813301,
90
+ "observations": 262144,
91
+ "used_codes": 5309,
92
+ "utilization": 0.6480712890625
93
+ }
94
+ ],
95
+ "ncp_by_level": [
96
+ {
97
+ "cross_entropy_nats": 5.318140983581543,
98
+ "level_index": 0,
99
+ "level_size": 1,
100
+ "token_accuracy": 0.013427734375
101
+ },
102
+ {
103
+ "cross_entropy_nats": 7.314859472215176,
104
+ "level_index": 1,
105
+ "level_size": 4,
106
+ "token_accuracy": 0.02679443359375
107
+ },
108
+ {
109
+ "cross_entropy_nats": 4.473740233315362,
110
+ "level_index": 2,
111
+ "level_size": 9,
112
+ "token_accuracy": 0.21085611979166666
113
+ },
114
+ {
115
+ "cross_entropy_nats": 2.763992566615343,
116
+ "level_index": 3,
117
+ "level_size": 16,
118
+ "token_accuracy": 0.425384521484375
119
+ },
120
+ {
121
+ "cross_entropy_nats": 1.2383683383464814,
122
+ "level_index": 4,
123
+ "level_size": 25,
124
+ "token_accuracy": 0.698427734375
125
+ },
126
+ {
127
+ "cross_entropy_nats": 0.5924244845906893,
128
+ "level_index": 5,
129
+ "level_size": 36,
130
+ "token_accuracy": 0.8195461697048612
131
+ },
132
+ {
133
+ "cross_entropy_nats": 0.5968850407065177,
134
+ "level_index": 6,
135
+ "level_size": 49,
136
+ "token_accuracy": 0.8168546715561225
137
+ },
138
+ {
139
+ "cross_entropy_nats": 0.1905308347195387,
140
+ "level_index": 7,
141
+ "level_size": 64,
142
+ "token_accuracy": 0.9392585754394531
143
+ }
144
+ ],
145
+ "prefix_reconstruction": [
146
+ {
147
+ "cross_entropy_nats": 5.325701013207436,
148
+ "level_index": 0,
149
+ "through_level": 1,
150
+ "token_accuracy": 0.1072235107421875
151
+ },
152
+ {
153
+ "cross_entropy_nats": 3.876078113913536,
154
+ "level_index": 1,
155
+ "through_level": 4,
156
+ "token_accuracy": 0.21405029296875
157
+ },
158
+ {
159
+ "cross_entropy_nats": 2.695797584950924,
160
+ "level_index": 2,
161
+ "through_level": 9,
162
+ "token_accuracy": 0.34746551513671875
163
+ },
164
+ {
165
+ "cross_entropy_nats": 1.2972525209188461,
166
+ "level_index": 3,
167
+ "through_level": 16,
168
+ "token_accuracy": 0.6556320190429688
169
+ },
170
+ {
171
+ "cross_entropy_nats": 0.47663837019354105,
172
+ "level_index": 4,
173
+ "through_level": 25,
174
+ "token_accuracy": 0.8517951965332031
175
+ },
176
+ {
177
+ "cross_entropy_nats": 0.16667770082131028,
178
+ "level_index": 5,
179
+ "through_level": 36,
180
+ "token_accuracy": 0.9468650817871094
181
+ },
182
+ {
183
+ "cross_entropy_nats": 0.07311888103140518,
184
+ "level_index": 6,
185
+ "through_level": 49,
186
+ "token_accuracy": 0.9764022827148438
187
+ },
188
+ {
189
+ "cross_entropy_nats": 0.0005318695339155965,
190
+ "level_index": 7,
191
+ "through_level": 64,
192
+ "token_accuracy": 0.9998855590820312
193
+ }
194
+ ],
195
+ "weighted_nsp_nats": 1.3002303428947926
196
+ }
197
+ }
ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/two_stage-step-16563.json ADDED
@@ -0,0 +1,96 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": {
3
+ "bytes": 1363098091,
4
+ "path": "outputs/ncm/tinystories-two-stage-full-v2-5epoch-ddp/checkpoints/step-16563.pt",
5
+ "step": 16563,
6
+ "training_regime": "two_stage"
7
+ },
8
+ "code_commit": "0775db5cd9708ed9aed8d69ee130d6119527c433",
9
+ "indices": "outputs/ncm/analysis/tokenizer-hierarchy-v1/raw/two_stage-step-16563.indices.npz",
10
+ "levels": [
11
+ 1,
12
+ 4,
13
+ 9,
14
+ 16,
15
+ 25,
16
+ 36,
17
+ 49,
18
+ 64
19
+ ],
20
+ "protocol": {
21
+ "batch_size": 64,
22
+ "indices_only": true,
23
+ "n_samples": 4096,
24
+ "sample_sha256": "c8b2a6079cc518751ded337e0ae600824783fed7d234ccb8088735d7c53c5282",
25
+ "seed": 42,
26
+ "split": "data/ncm/tinystories/valid_bpe_8192.pickle"
27
+ },
28
+ "student": {
29
+ "codebooks": [
30
+ {
31
+ "effective_codes": 81.28478103295326,
32
+ "level_index": 0,
33
+ "marginal_entropy_nats": 4.397958803863936,
34
+ "observations": 4096,
35
+ "used_codes": 146,
36
+ "utilization": 0.017822265625
37
+ },
38
+ {
39
+ "effective_codes": 956.6936405026054,
40
+ "level_index": 1,
41
+ "marginal_entropy_nats": 6.86348321533539,
42
+ "observations": 16384,
43
+ "used_codes": 2035,
44
+ "utilization": 0.2484130859375
45
+ },
46
+ {
47
+ "effective_codes": 1767.8659819323707,
48
+ "level_index": 2,
49
+ "marginal_entropy_nats": 7.477528438250015,
50
+ "observations": 36864,
51
+ "used_codes": 3986,
52
+ "utilization": 0.486572265625
53
+ },
54
+ {
55
+ "effective_codes": 2329.0922694427577,
56
+ "level_index": 3,
57
+ "marginal_entropy_nats": 7.7532338867220645,
58
+ "observations": 65536,
59
+ "used_codes": 5897,
60
+ "utilization": 0.7198486328125
61
+ },
62
+ {
63
+ "effective_codes": 2010.0747485163015,
64
+ "level_index": 4,
65
+ "marginal_entropy_nats": 7.605927188678221,
66
+ "observations": 102400,
67
+ "used_codes": 5504,
68
+ "utilization": 0.671875
69
+ },
70
+ {
71
+ "effective_codes": 1685.4008610507324,
72
+ "level_index": 5,
73
+ "marginal_entropy_nats": 7.42975871422618,
74
+ "observations": 147456,
75
+ "used_codes": 5051,
76
+ "utilization": 0.6165771484375
77
+ },
78
+ {
79
+ "effective_codes": 1799.4724066084204,
80
+ "level_index": 6,
81
+ "marginal_entropy_nats": 7.495248793480108,
82
+ "observations": 200704,
83
+ "used_codes": 5559,
84
+ "utilization": 0.6785888671875
85
+ },
86
+ {
87
+ "effective_codes": 1247.1762537093414,
88
+ "level_index": 7,
89
+ "marginal_entropy_nats": 7.128637277880896,
90
+ "observations": 262144,
91
+ "used_codes": 5486,
92
+ "utilization": 0.669677734375
93
+ }
94
+ ]
95
+ }
96
+ }
ts-fivearm/analysis/tokenizer-hierarchy-v1/raw/two_stage-step-82813.json ADDED
@@ -0,0 +1,197 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "checkpoint": {
3
+ "bytes": 1363101739,
4
+ "path": "outputs/ncm/tinystories-two-stage-full-v2-5epoch-ddp/checkpoints/step-82813.pt",
5
+ "step": 82813,
6
+ "training_regime": "two_stage"
7
+ },
8
+ "code_commit": "0775db5cd9708ed9aed8d69ee130d6119527c433",
9
+ "indices": "outputs/ncm/analysis/tokenizer-hierarchy-v1/raw/two_stage-step-82813.indices.npz",
10
+ "levels": [
11
+ 1,
12
+ 4,
13
+ 9,
14
+ 16,
15
+ 25,
16
+ 36,
17
+ 49,
18
+ 64
19
+ ],
20
+ "protocol": {
21
+ "batch_size": 64,
22
+ "indices_only": false,
23
+ "n_samples": 4096,
24
+ "sample_sha256": "c8b2a6079cc518751ded337e0ae600824783fed7d234ccb8088735d7c53c5282",
25
+ "seed": 42,
26
+ "split": "data/ncm/tinystories/valid_bpe_8192.pickle"
27
+ },
28
+ "student": {
29
+ "codebooks": [
30
+ {
31
+ "effective_codes": 81.28478103295326,
32
+ "level_index": 0,
33
+ "marginal_entropy_nats": 4.397958803863936,
34
+ "observations": 4096,
35
+ "used_codes": 146,
36
+ "utilization": 0.017822265625
37
+ },
38
+ {
39
+ "effective_codes": 956.6936405026054,
40
+ "level_index": 1,
41
+ "marginal_entropy_nats": 6.86348321533539,
42
+ "observations": 16384,
43
+ "used_codes": 2035,
44
+ "utilization": 0.2484130859375
45
+ },
46
+ {
47
+ "effective_codes": 1767.8659819323707,
48
+ "level_index": 2,
49
+ "marginal_entropy_nats": 7.477528438250015,
50
+ "observations": 36864,
51
+ "used_codes": 3986,
52
+ "utilization": 0.486572265625
53
+ },
54
+ {
55
+ "effective_codes": 2329.0922694427577,
56
+ "level_index": 3,
57
+ "marginal_entropy_nats": 7.7532338867220645,
58
+ "observations": 65536,
59
+ "used_codes": 5897,
60
+ "utilization": 0.7198486328125
61
+ },
62
+ {
63
+ "effective_codes": 2010.0747485163015,
64
+ "level_index": 4,
65
+ "marginal_entropy_nats": 7.605927188678221,
66
+ "observations": 102400,
67
+ "used_codes": 5504,
68
+ "utilization": 0.671875
69
+ },
70
+ {
71
+ "effective_codes": 1685.4008610507324,
72
+ "level_index": 5,
73
+ "marginal_entropy_nats": 7.42975871422618,
74
+ "observations": 147456,
75
+ "used_codes": 5051,
76
+ "utilization": 0.6165771484375
77
+ },
78
+ {
79
+ "effective_codes": 1799.4724066084204,
80
+ "level_index": 6,
81
+ "marginal_entropy_nats": 7.495248793480108,
82
+ "observations": 200704,
83
+ "used_codes": 5559,
84
+ "utilization": 0.6785888671875
85
+ },
86
+ {
87
+ "effective_codes": 1247.1762537093414,
88
+ "level_index": 7,
89
+ "marginal_entropy_nats": 7.128637277880896,
90
+ "observations": 262144,
91
+ "used_codes": 5486,
92
+ "utilization": 0.669677734375
93
+ }
94
+ ],
95
+ "ncp_by_level": [
96
+ {
97
+ "cross_entropy_nats": 5.224560737609863,
98
+ "level_index": 0,
99
+ "level_size": 1,
100
+ "token_accuracy": 0.0078125
101
+ },
102
+ {
103
+ "cross_entropy_nats": 7.23945339769125,
104
+ "level_index": 1,
105
+ "level_size": 4,
106
+ "token_accuracy": 0.0316162109375
107
+ },
108
+ {
109
+ "cross_entropy_nats": 4.429411782158746,
110
+ "level_index": 2,
111
+ "level_size": 9,
112
+ "token_accuracy": 0.2239312065972222
113
+ },
114
+ {
115
+ "cross_entropy_nats": 2.6503800339996815,
116
+ "level_index": 3,
117
+ "level_size": 16,
118
+ "token_accuracy": 0.4469146728515625
119
+ },
120
+ {
121
+ "cross_entropy_nats": 1.1432661604881287,
122
+ "level_index": 4,
123
+ "level_size": 25,
124
+ "token_accuracy": 0.726494140625
125
+ },
126
+ {
127
+ "cross_entropy_nats": 0.3236362718873554,
128
+ "level_index": 5,
129
+ "level_size": 36,
130
+ "token_accuracy": 0.9003499348958334
131
+ },
132
+ {
133
+ "cross_entropy_nats": 0.20877404936722346,
134
+ "level_index": 6,
135
+ "level_size": 49,
136
+ "token_accuracy": 0.9271364795918368
137
+ },
138
+ {
139
+ "cross_entropy_nats": 0.043378556438256055,
140
+ "level_index": 7,
141
+ "level_size": 64,
142
+ "token_accuracy": 0.9844551086425781
143
+ }
144
+ ],
145
+ "prefix_reconstruction": [
146
+ {
147
+ "cross_entropy_nats": 5.201899446547031,
148
+ "level_index": 0,
149
+ "through_level": 1,
150
+ "token_accuracy": 0.12760543823242188
151
+ },
152
+ {
153
+ "cross_entropy_nats": 3.914573274552822,
154
+ "level_index": 1,
155
+ "through_level": 4,
156
+ "token_accuracy": 0.21005630493164062
157
+ },
158
+ {
159
+ "cross_entropy_nats": 2.6983997523784637,
160
+ "level_index": 2,
161
+ "through_level": 9,
162
+ "token_accuracy": 0.3702201843261719
163
+ },
164
+ {
165
+ "cross_entropy_nats": 1.296346077695489,
166
+ "level_index": 3,
167
+ "through_level": 16,
168
+ "token_accuracy": 0.6683502197265625
169
+ },
170
+ {
171
+ "cross_entropy_nats": 0.47104807337746024,
172
+ "level_index": 4,
173
+ "through_level": 25,
174
+ "token_accuracy": 0.8548316955566406
175
+ },
176
+ {
177
+ "cross_entropy_nats": 0.1591827953234315,
178
+ "level_index": 5,
179
+ "through_level": 36,
180
+ "token_accuracy": 0.9495277404785156
181
+ },
182
+ {
183
+ "cross_entropy_nats": 0.06788585550384596,
184
+ "level_index": 6,
185
+ "through_level": 49,
186
+ "token_accuracy": 0.9783401489257812
187
+ },
188
+ {
189
+ "cross_entropy_nats": 0.0007997321670245583,
190
+ "level_index": 7,
191
+ "through_level": 64,
192
+ "token_accuracy": 0.999847412109375
193
+ }
194
+ ],
195
+ "weighted_nsp_nats": 1.098238380625844
196
+ }
197
+ }
ts-fivearm/analysis/tokenizer-hierarchy-v1/summary.json ADDED
@@ -0,0 +1,774 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "dynamics": {
3
+ "mstok_v1": [
4
+ {
5
+ "from_step": 16563,
6
+ "student_assignment_churn": {
7
+ "overall": 0.41810039445465685,
8
+ "per_level": [
9
+ 0.493408203125,
10
+ 0.47552490234375,
11
+ 0.3756239149305556,
12
+ 0.29278564453125,
13
+ 0.407421875,
14
+ 0.5340847439236112,
15
+ 0.5690868144132653,
16
+ 0.2739677429199219
17
+ ]
18
+ },
19
+ "teacher_assignment_churn": {
20
+ "overall": 0.4192528818167892,
21
+ "per_level": [
22
+ 0.434814453125,
23
+ 0.48663330078125,
24
+ 0.3776041666666667,
25
+ 0.2930145263671875,
26
+ 0.406513671875,
27
+ 0.53472900390625,
28
+ 0.5736806441326531,
29
+ 0.2740020751953125
30
+ ]
31
+ },
32
+ "to_step": 33126
33
+ },
34
+ {
35
+ "from_step": 33126,
36
+ "student_assignment_churn": {
37
+ "overall": 0.34024825750612747,
38
+ "per_level": [
39
+ 0.331298828125,
40
+ 0.36895751953125,
41
+ 0.2736002604166667,
42
+ 0.18817138671875,
43
+ 0.314306640625,
44
+ 0.4352552625868056,
45
+ 0.48842574139030615,
46
+ 0.22922897338867188
47
+ ]
48
+ },
49
+ "teacher_assignment_churn": {
50
+ "overall": 0.3222608379289216,
51
+ "per_level": [
52
+ 0.277587890625,
53
+ 0.338134765625,
54
+ 0.2542588975694444,
55
+ 0.1715087890625,
56
+ 0.293642578125,
57
+ 0.4131740993923611,
58
+ 0.46812220982142855,
59
+ 0.21758270263671875
60
+ ]
61
+ },
62
+ "to_step": 49689
63
+ },
64
+ {
65
+ "from_step": 49689,
66
+ "student_assignment_churn": {
67
+ "overall": 0.29595348881740197,
68
+ "per_level": [
69
+ 0.262939453125,
70
+ 0.30609130859375,
71
+ 0.21666124131944445,
72
+ 0.156768798828125,
73
+ 0.273828125,
74
+ 0.3777737087673611,
75
+ 0.4442861128826531,
76
+ 0.19083404541015625
77
+ ]
78
+ },
79
+ "teacher_assignment_churn": {
80
+ "overall": 0.27728989545036764,
81
+ "per_level": [
82
+ 0.22998046875,
83
+ 0.2723388671875,
84
+ 0.19227430555555555,
85
+ 0.1429290771484375,
86
+ 0.26482421875,
87
+ 0.34783935546875,
88
+ 0.41881576849489793,
89
+ 0.18071365356445312
90
+ ]
91
+ },
92
+ "to_step": 66252
93
+ },
94
+ {
95
+ "from_step": 66252,
96
+ "student_assignment_churn": {
97
+ "overall": 0.26662789139093135,
98
+ "per_level": [
99
+ 0.234130859375,
100
+ 0.2845458984375,
101
+ 0.2009548611111111,
102
+ 0.135467529296875,
103
+ 0.2309765625,
104
+ 0.3464491102430556,
105
+ 0.40744579081632654,
106
+ 0.16925430297851562
107
+ ]
108
+ },
109
+ "teacher_assignment_churn": {
110
+ "overall": 0.23729631012561275,
111
+ "per_level": [
112
+ 0.189208984375,
113
+ 0.24163818359375,
114
+ 0.16935221354166666,
115
+ 0.1133880615234375,
116
+ 0.203779296875,
117
+ 0.30596923828125,
118
+ 0.3672622369260204,
119
+ 0.15326690673828125
120
+ ]
121
+ },
122
+ "to_step": 82813
123
+ }
124
+ ],
125
+ "naive_e2e": [
126
+ {
127
+ "from_step": 16563,
128
+ "student_assignment_churn": {
129
+ "overall": 0.3840762867647059,
130
+ "per_level": [
131
+ 0.813720703125,
132
+ 0.4951171875,
133
+ 0.3398708767361111,
134
+ 0.2527008056640625,
135
+ 0.3735546875,
136
+ 0.5207722981770834,
137
+ 0.5285843829719388,
138
+ 0.22606277465820312
139
+ ]
140
+ },
141
+ "to_step": 33126
142
+ },
143
+ {
144
+ "from_step": 33126,
145
+ "student_assignment_churn": {
146
+ "overall": 0.3187100279564951,
147
+ "per_level": [
148
+ 0.817138671875,
149
+ 0.392578125,
150
+ 0.2629937065972222,
151
+ 0.189483642578125,
152
+ 0.30017578125,
153
+ 0.4165242513020833,
154
+ 0.47269610969387754,
155
+ 0.1807708740234375
156
+ ]
157
+ },
158
+ "to_step": 49689
159
+ },
160
+ {
161
+ "from_step": 49689,
162
+ "student_assignment_churn": {
163
+ "overall": 0.29489554610906865,
164
+ "per_level": [
165
+ 0.894287109375,
166
+ 0.33917236328125,
167
+ 0.23372395833333334,
168
+ 0.1722869873046875,
169
+ 0.2653515625,
170
+ 0.3750067816840278,
171
+ 0.4526865433673469,
172
+ 0.16768646240234375
173
+ ]
174
+ },
175
+ "to_step": 66252
176
+ },
177
+ {
178
+ "from_step": 66252,
179
+ "student_assignment_churn": {
180
+ "overall": 0.2485674689797794,
181
+ "per_level": [
182
+ 0.610595703125,
183
+ 0.24847412109375,
184
+ 0.1809353298611111,
185
+ 0.1231536865234375,
186
+ 0.2081640625,
187
+ 0.31365966796875,
188
+ 0.40664361447704084,
189
+ 0.1419219970703125
190
+ ]
191
+ },
192
+ "to_step": 82813
193
+ }
194
+ ],
195
+ "two_stage": [
196
+ {
197
+ "from_step": 16563,
198
+ "student_assignment_churn": {
199
+ "overall": 0.0,
200
+ "per_level": [
201
+ 0.0,
202
+ 0.0,
203
+ 0.0,
204
+ 0.0,
205
+ 0.0,
206
+ 0.0,
207
+ 0.0,
208
+ 0.0
209
+ ]
210
+ },
211
+ "to_step": 82813
212
+ }
213
+ ]
214
+ },
215
+ "final": {
216
+ "detached_apr": {
217
+ "full_reconstruction_accuracy": 0.9999275207519531,
218
+ "full_reconstruction_ce_nats": 0.00046051875330022085,
219
+ "marginal_rate_nats_per_story": 1498.522508364222,
220
+ "mean_code_utilization": 0.5268402099609375,
221
+ "ncp_by_level": [
222
+ {
223
+ "cross_entropy_nats": 4.974203109741211,
224
+ "level_index": 0,
225
+ "level_size": 1,
226
+ "token_accuracy": 0.013427734375
227
+ },
228
+ {
229
+ "cross_entropy_nats": 7.546774573624134,
230
+ "level_index": 1,
231
+ "level_size": 4,
232
+ "token_accuracy": 0.02734375
233
+ },
234
+ {
235
+ "cross_entropy_nats": 4.525883674621582,
236
+ "level_index": 2,
237
+ "level_size": 9,
238
+ "token_accuracy": 0.22767469618055555
239
+ },
240
+ {
241
+ "cross_entropy_nats": 2.784970488399267,
242
+ "level_index": 3,
243
+ "level_size": 16,
244
+ "token_accuracy": 0.437225341796875
245
+ },
246
+ {
247
+ "cross_entropy_nats": 1.3025939130783082,
248
+ "level_index": 4,
249
+ "level_size": 25,
250
+ "token_accuracy": 0.6915234375
251
+ },
252
+ {
253
+ "cross_entropy_nats": 0.6450617793533537,
254
+ "level_index": 5,
255
+ "level_size": 36,
256
+ "token_accuracy": 0.8096788194444444
257
+ },
258
+ {
259
+ "cross_entropy_nats": 0.679096769313423,
260
+ "level_index": 6,
261
+ "level_size": 49,
262
+ "token_accuracy": 0.8046476403061225
263
+ },
264
+ {
265
+ "cross_entropy_nats": 0.2202611332759261,
266
+ "level_index": 7,
267
+ "level_size": 64,
268
+ "token_accuracy": 0.9302215576171875
269
+ }
270
+ ],
271
+ "prefix_reconstruction": [
272
+ {
273
+ "cross_entropy_nats": 5.318077310919762,
274
+ "level_index": 0,
275
+ "through_level": 1,
276
+ "token_accuracy": 0.09982681274414062
277
+ },
278
+ {
279
+ "cross_entropy_nats": 3.8502018973231316,
280
+ "level_index": 1,
281
+ "through_level": 4,
282
+ "token_accuracy": 0.21706771850585938
283
+ },
284
+ {
285
+ "cross_entropy_nats": 2.6491841562092304,
286
+ "level_index": 2,
287
+ "through_level": 9,
288
+ "token_accuracy": 0.3804435729980469
289
+ },
290
+ {
291
+ "cross_entropy_nats": 1.3218422196805477,
292
+ "level_index": 3,
293
+ "through_level": 16,
294
+ "token_accuracy": 0.6571846008300781
295
+ },
296
+ {
297
+ "cross_entropy_nats": 0.48308326303958893,
298
+ "level_index": 4,
299
+ "through_level": 25,
300
+ "token_accuracy": 0.8490104675292969
301
+ },
302
+ {
303
+ "cross_entropy_nats": 0.1673640706576407,
304
+ "level_index": 5,
305
+ "through_level": 36,
306
+ "token_accuracy": 0.9466361999511719
307
+ },
308
+ {
309
+ "cross_entropy_nats": 0.07380200660554692,
310
+ "level_index": 6,
311
+ "through_level": 49,
312
+ "token_accuracy": 0.9765892028808594
313
+ },
314
+ {
315
+ "cross_entropy_nats": 0.00046051875330022085,
316
+ "level_index": 7,
317
+ "through_level": 64,
318
+ "token_accuracy": 0.9999275207519531
319
+ }
320
+ ],
321
+ "weighted_nsp_nats": 1.3530296683311462
322
+ },
323
+ "eostok_apr": {
324
+ "full_reconstruction_accuracy": 0.99993896484375,
325
+ "full_reconstruction_ce_nats": 0.00038653903675367474,
326
+ "marginal_rate_nats_per_story": 1495.742697674512,
327
+ "mean_code_utilization": 0.51812744140625,
328
+ "ncp_by_level": [
329
+ {
330
+ "cross_entropy_nats": 5.3176679611206055,
331
+ "level_index": 0,
332
+ "level_size": 1,
333
+ "token_accuracy": 0.001708984375
334
+ },
335
+ {
336
+ "cross_entropy_nats": 7.429441511631012,
337
+ "level_index": 1,
338
+ "level_size": 4,
339
+ "token_accuracy": 0.02410888671875
340
+ },
341
+ {
342
+ "cross_entropy_nats": 4.4729632006751165,
343
+ "level_index": 2,
344
+ "level_size": 9,
345
+ "token_accuracy": 0.21153428819444445
346
+ },
347
+ {
348
+ "cross_entropy_nats": 2.7938546873629093,
349
+ "level_index": 3,
350
+ "level_size": 16,
351
+ "token_accuracy": 0.431854248046875
352
+ },
353
+ {
354
+ "cross_entropy_nats": 1.1903628182411194,
355
+ "level_index": 4,
356
+ "level_size": 25,
357
+ "token_accuracy": 0.71142578125
358
+ },
359
+ {
360
+ "cross_entropy_nats": 0.5721849567360349,
361
+ "level_index": 5,
362
+ "level_size": 36,
363
+ "token_accuracy": 0.8268703884548612
364
+ },
365
+ {
366
+ "cross_entropy_nats": 0.580585431687686,
367
+ "level_index": 6,
368
+ "level_size": 49,
369
+ "token_accuracy": 0.8219367825255102
370
+ },
371
+ {
372
+ "cross_entropy_nats": 0.16295399656519294,
373
+ "level_index": 7,
374
+ "level_size": 64,
375
+ "token_accuracy": 0.9498405456542969
376
+ }
377
+ ],
378
+ "prefix_reconstruction": [
379
+ {
380
+ "cross_entropy_nats": 5.30550742149353,
381
+ "level_index": 0,
382
+ "through_level": 1,
383
+ "token_accuracy": 0.10834884643554688
384
+ },
385
+ {
386
+ "cross_entropy_nats": 3.8148250617086887,
387
+ "level_index": 1,
388
+ "through_level": 4,
389
+ "token_accuracy": 0.20490264892578125
390
+ },
391
+ {
392
+ "cross_entropy_nats": 2.6031500548124313,
393
+ "level_index": 2,
394
+ "through_level": 9,
395
+ "token_accuracy": 0.372833251953125
396
+ },
397
+ {
398
+ "cross_entropy_nats": 1.271256821230054,
399
+ "level_index": 3,
400
+ "through_level": 16,
401
+ "token_accuracy": 0.6538238525390625
402
+ },
403
+ {
404
+ "cross_entropy_nats": 0.46055822307243943,
405
+ "level_index": 4,
406
+ "through_level": 25,
407
+ "token_accuracy": 0.8547477722167969
408
+ },
409
+ {
410
+ "cross_entropy_nats": 0.16380383376963437,
411
+ "level_index": 5,
412
+ "through_level": 36,
413
+ "token_accuracy": 0.9474754333496094
414
+ },
415
+ {
416
+ "cross_entropy_nats": 0.07394133933121338,
417
+ "level_index": 6,
418
+ "through_level": 49,
419
+ "token_accuracy": 0.9766120910644531
420
+ },
421
+ {
422
+ "cross_entropy_nats": 0.00038653903675367474,
423
+ "level_index": 7,
424
+ "through_level": 64,
425
+ "token_accuracy": 0.99993896484375
426
+ }
427
+ ],
428
+ "weighted_nsp_nats": 1.2859343606978655
429
+ },
430
+ "mstok_v1": {
431
+ "full_reconstruction_accuracy": 0.9998588562011719,
432
+ "full_reconstruction_ce_nats": 0.0008627551530935307,
433
+ "marginal_rate_nats_per_story": 1504.7349962842995,
434
+ "mean_code_utilization": 0.512725830078125,
435
+ "ncp_by_level": [
436
+ {
437
+ "cross_entropy_nats": 5.1687397956848145,
438
+ "level_index": 0,
439
+ "level_size": 1,
440
+ "token_accuracy": 0.006103515625
441
+ },
442
+ {
443
+ "cross_entropy_nats": 7.271853156387806,
444
+ "level_index": 1,
445
+ "level_size": 4,
446
+ "token_accuracy": 0.03570556640625
447
+ },
448
+ {
449
+ "cross_entropy_nats": 4.608915514416164,
450
+ "level_index": 2,
451
+ "level_size": 9,
452
+ "token_accuracy": 0.20939127604166666
453
+ },
454
+ {
455
+ "cross_entropy_nats": 2.803427193313837,
456
+ "level_index": 3,
457
+ "level_size": 16,
458
+ "token_accuracy": 0.4294586181640625
459
+ },
460
+ {
461
+ "cross_entropy_nats": 1.2834816360473633,
462
+ "level_index": 4,
463
+ "level_size": 25,
464
+ "token_accuracy": 0.694091796875
465
+ },
466
+ {
467
+ "cross_entropy_nats": 0.4765111414922608,
468
+ "level_index": 5,
469
+ "level_size": 36,
470
+ "token_accuracy": 0.8569403754340278
471
+ },
472
+ {
473
+ "cross_entropy_nats": 0.3875969636196993,
474
+ "level_index": 6,
475
+ "level_size": 49,
476
+ "token_accuracy": 0.8817512356505102
477
+ },
478
+ {
479
+ "cross_entropy_nats": 0.07621806376846507,
480
+ "level_index": 7,
481
+ "level_size": 64,
482
+ "token_accuracy": 0.9738731384277344
483
+ }
484
+ ],
485
+ "prefix_reconstruction": [
486
+ {
487
+ "cross_entropy_nats": 5.167205706238747,
488
+ "level_index": 0,
489
+ "through_level": 1,
490
+ "token_accuracy": 0.12995529174804688
491
+ },
492
+ {
493
+ "cross_entropy_nats": 3.917147032916546,
494
+ "level_index": 1,
495
+ "through_level": 4,
496
+ "token_accuracy": 0.21495819091796875
497
+ },
498
+ {
499
+ "cross_entropy_nats": 2.7244248501956463,
500
+ "level_index": 2,
501
+ "through_level": 9,
502
+ "token_accuracy": 0.36641693115234375
503
+ },
504
+ {
505
+ "cross_entropy_nats": 1.314411474391818,
506
+ "level_index": 3,
507
+ "through_level": 16,
508
+ "token_accuracy": 0.6632041931152344
509
+ },
510
+ {
511
+ "cross_entropy_nats": 0.47825157130137086,
512
+ "level_index": 4,
513
+ "through_level": 25,
514
+ "token_accuracy": 0.8531761169433594
515
+ },
516
+ {
517
+ "cross_entropy_nats": 0.15295705071184784,
518
+ "level_index": 5,
519
+ "through_level": 36,
520
+ "token_accuracy": 0.9521369934082031
521
+ },
522
+ {
523
+ "cross_entropy_nats": 0.06281172827584669,
524
+ "level_index": 6,
525
+ "through_level": 49,
526
+ "token_accuracy": 0.9808197021484375
527
+ },
528
+ {
529
+ "cross_entropy_nats": 0.0008627551530935307,
530
+ "level_index": 7,
531
+ "through_level": 64,
532
+ "token_accuracy": 0.9998588562011719
533
+ }
534
+ ],
535
+ "student_teacher_disagreement": [
536
+ 0.132080078125,
537
+ 0.13287353515625,
538
+ 0.0932888454861111,
539
+ 0.06683349609375,
540
+ 0.12109375,
541
+ 0.19209798177083334,
542
+ 0.24783262914540816,
543
+ 0.10200881958007812
544
+ ],
545
+ "teacher_weighted_nsp_nats": 1.1813653074204922,
546
+ "weighted_nsp_nats": 1.217957016080618
547
+ },
548
+ "naive_e2e": {
549
+ "full_reconstruction_accuracy": 0.9998855590820312,
550
+ "full_reconstruction_ce_nats": 0.0005318695339155965,
551
+ "marginal_rate_nats_per_story": 1499.2086446533553,
552
+ "mean_code_utilization": 0.5193023681640625,
553
+ "ncp_by_level": [
554
+ {
555
+ "cross_entropy_nats": 5.318140983581543,
556
+ "level_index": 0,
557
+ "level_size": 1,
558
+ "token_accuracy": 0.013427734375
559
+ },
560
+ {
561
+ "cross_entropy_nats": 7.314859472215176,
562
+ "level_index": 1,
563
+ "level_size": 4,
564
+ "token_accuracy": 0.02679443359375
565
+ },
566
+ {
567
+ "cross_entropy_nats": 4.473740233315362,
568
+ "level_index": 2,
569
+ "level_size": 9,
570
+ "token_accuracy": 0.21085611979166666
571
+ },
572
+ {
573
+ "cross_entropy_nats": 2.763992566615343,
574
+ "level_index": 3,
575
+ "level_size": 16,
576
+ "token_accuracy": 0.425384521484375
577
+ },
578
+ {
579
+ "cross_entropy_nats": 1.2383683383464814,
580
+ "level_index": 4,
581
+ "level_size": 25,
582
+ "token_accuracy": 0.698427734375
583
+ },
584
+ {
585
+ "cross_entropy_nats": 0.5924244845906893,
586
+ "level_index": 5,
587
+ "level_size": 36,
588
+ "token_accuracy": 0.8195461697048612
589
+ },
590
+ {
591
+ "cross_entropy_nats": 0.5968850407065177,
592
+ "level_index": 6,
593
+ "level_size": 49,
594
+ "token_accuracy": 0.8168546715561225
595
+ },
596
+ {
597
+ "cross_entropy_nats": 0.1905308347195387,
598
+ "level_index": 7,
599
+ "level_size": 64,
600
+ "token_accuracy": 0.9392585754394531
601
+ }
602
+ ],
603
+ "prefix_reconstruction": [
604
+ {
605
+ "cross_entropy_nats": 5.325701013207436,
606
+ "level_index": 0,
607
+ "through_level": 1,
608
+ "token_accuracy": 0.1072235107421875
609
+ },
610
+ {
611
+ "cross_entropy_nats": 3.876078113913536,
612
+ "level_index": 1,
613
+ "through_level": 4,
614
+ "token_accuracy": 0.21405029296875
615
+ },
616
+ {
617
+ "cross_entropy_nats": 2.695797584950924,
618
+ "level_index": 2,
619
+ "through_level": 9,
620
+ "token_accuracy": 0.34746551513671875
621
+ },
622
+ {
623
+ "cross_entropy_nats": 1.2972525209188461,
624
+ "level_index": 3,
625
+ "through_level": 16,
626
+ "token_accuracy": 0.6556320190429688
627
+ },
628
+ {
629
+ "cross_entropy_nats": 0.47663837019354105,
630
+ "level_index": 4,
631
+ "through_level": 25,
632
+ "token_accuracy": 0.8517951965332031
633
+ },
634
+ {
635
+ "cross_entropy_nats": 0.16667770082131028,
636
+ "level_index": 5,
637
+ "through_level": 36,
638
+ "token_accuracy": 0.9468650817871094
639
+ },
640
+ {
641
+ "cross_entropy_nats": 0.07311888103140518,
642
+ "level_index": 6,
643
+ "through_level": 49,
644
+ "token_accuracy": 0.9764022827148438
645
+ },
646
+ {
647
+ "cross_entropy_nats": 0.0005318695339155965,
648
+ "level_index": 7,
649
+ "through_level": 64,
650
+ "token_accuracy": 0.9998855590820312
651
+ }
652
+ ],
653
+ "weighted_nsp_nats": 1.3002303428947926
654
+ },
655
+ "two_stage": {
656
+ "full_reconstruction_accuracy": 0.999847412109375,
657
+ "full_reconstruction_ce_nats": 0.0007997321670245583,
658
+ "marginal_rate_nats_per_story": 1504.3208598910091,
659
+ "mean_code_utilization": 0.513671875,
660
+ "ncp_by_level": [
661
+ {
662
+ "cross_entropy_nats": 5.224560737609863,
663
+ "level_index": 0,
664
+ "level_size": 1,
665
+ "token_accuracy": 0.0078125
666
+ },
667
+ {
668
+ "cross_entropy_nats": 7.23945339769125,
669
+ "level_index": 1,
670
+ "level_size": 4,
671
+ "token_accuracy": 0.0316162109375
672
+ },
673
+ {
674
+ "cross_entropy_nats": 4.429411782158746,
675
+ "level_index": 2,
676
+ "level_size": 9,
677
+ "token_accuracy": 0.2239312065972222
678
+ },
679
+ {
680
+ "cross_entropy_nats": 2.6503800339996815,
681
+ "level_index": 3,
682
+ "level_size": 16,
683
+ "token_accuracy": 0.4469146728515625
684
+ },
685
+ {
686
+ "cross_entropy_nats": 1.1432661604881287,
687
+ "level_index": 4,
688
+ "level_size": 25,
689
+ "token_accuracy": 0.726494140625
690
+ },
691
+ {
692
+ "cross_entropy_nats": 0.3236362718873554,
693
+ "level_index": 5,
694
+ "level_size": 36,
695
+ "token_accuracy": 0.9003499348958334
696
+ },
697
+ {
698
+ "cross_entropy_nats": 0.20877404936722346,
699
+ "level_index": 6,
700
+ "level_size": 49,
701
+ "token_accuracy": 0.9271364795918368
702
+ },
703
+ {
704
+ "cross_entropy_nats": 0.043378556438256055,
705
+ "level_index": 7,
706
+ "level_size": 64,
707
+ "token_accuracy": 0.9844551086425781
708
+ }
709
+ ],
710
+ "prefix_reconstruction": [
711
+ {
712
+ "cross_entropy_nats": 5.201899446547031,
713
+ "level_index": 0,
714
+ "through_level": 1,
715
+ "token_accuracy": 0.12760543823242188
716
+ },
717
+ {
718
+ "cross_entropy_nats": 3.914573274552822,
719
+ "level_index": 1,
720
+ "through_level": 4,
721
+ "token_accuracy": 0.21005630493164062
722
+ },
723
+ {
724
+ "cross_entropy_nats": 2.6983997523784637,
725
+ "level_index": 2,
726
+ "through_level": 9,
727
+ "token_accuracy": 0.3702201843261719
728
+ },
729
+ {
730
+ "cross_entropy_nats": 1.296346077695489,
731
+ "level_index": 3,
732
+ "through_level": 16,
733
+ "token_accuracy": 0.6683502197265625
734
+ },
735
+ {
736
+ "cross_entropy_nats": 0.47104807337746024,
737
+ "level_index": 4,
738
+ "through_level": 25,
739
+ "token_accuracy": 0.8548316955566406
740
+ },
741
+ {
742
+ "cross_entropy_nats": 0.1591827953234315,
743
+ "level_index": 5,
744
+ "through_level": 36,
745
+ "token_accuracy": 0.9495277404785156
746
+ },
747
+ {
748
+ "cross_entropy_nats": 0.06788585550384596,
749
+ "level_index": 6,
750
+ "through_level": 49,
751
+ "token_accuracy": 0.9783401489257812
752
+ },
753
+ {
754
+ "cross_entropy_nats": 0.0007997321670245583,
755
+ "level_index": 7,
756
+ "through_level": 64,
757
+ "token_accuracy": 0.999847412109375
758
+ }
759
+ ],
760
+ "weighted_nsp_nats": 1.098238380625844
761
+ }
762
+ },
763
+ "protocol": {
764
+ "primary_axes": [
765
+ "reconstruction fidelity",
766
+ "marginal code rate and utilization",
767
+ "trained next-scale modelability",
768
+ "epoch-to-epoch assignment churn",
769
+ "MsTok student-teacher disagreement and teacher churn"
770
+ ],
771
+ "selection": "preregistered before final MsTok checkpoint",
772
+ "study": "tinystories-tokenizer-hierarchy-v1"
773
+ }
774
+ }
ts-fivearm/analysis/tokenizer-hierarchy-v1/summary.md ADDED
@@ -0,0 +1,25 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ # TinyStories tokenizer hierarchy audit
2
+
3
+ Primary held-out results (lower CE/NSP is better; higher accuracy is better):
4
+
5
+ | Arm | Reconstruction CE | Token accuracy | Weighted NSP | Marginal rate | Mean usage |
6
+ |---|---:|---:|---:|---:|---:|
7
+ | two_stage | 0.0008 | 99.98% | 1.0982 | 1504.3 | 51.37% |
8
+ | naive_e2e | 0.0005 | 99.99% | 1.3002 | 1499.2 | 51.93% |
9
+ | eostok_apr | 0.0004 | 99.99% | 1.2859 | 1495.7 | 51.81% |
10
+ | detached_apr | 0.0005 | 99.99% | 1.3530 | 1498.5 | 52.68% |
11
+ | mstok_v1 | 0.0009 | 99.99% | 1.2180 | 1504.7 | 51.27% |
12
+
13
+ ## Assignment stability
14
+
15
+ | Arm | Step interval | Student churn | Teacher churn |
16
+ |---|---:|---:|---:|
17
+ | two_stage | 16563–82813 | 0.00% | — |
18
+ | naive_e2e | 16563–33126 | 38.41% | — |
19
+ | naive_e2e | 33126–49689 | 31.87% | — |
20
+ | naive_e2e | 49689–66252 | 29.49% | — |
21
+ | naive_e2e | 66252–82813 | 24.86% | — |
22
+ | mstok_v1 | 16563–33126 | 41.81% | 41.93% |
23
+ | mstok_v1 | 33126–49689 | 34.02% | 32.23% |
24
+ | mstok_v1 | 49689–66252 | 29.60% | 27.73% |
25
+ | mstok_v1 | 66252–82813 | 26.66% | 23.73% |