ruhai-lin commited on
Commit
dec79dd
·
verified ·
1 Parent(s): d2c007d

Clean up model releases

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. README.md +28 -32
  2. eval/core_comparison.csv +24 -24
  3. gdn-100m/fineweb-edu-10B-s1337/.gitkeep +0 -0
  4. gdn-100m/fineweb-edu-10B-s1337/config.json +0 -37
  5. gdn-100m/fineweb-edu-10B-s1337/eval/core_eval.csv +0 -24
  6. gdn-100m/fineweb-edu-10B-s1337/generation_config.json +0 -7
  7. gdn-100m/fineweb-edu-10B-s1337/model.safetensors +0 -3
  8. gdn-100m/fineweb-edu-10B-s1337/special_tokens_map.json +0 -23
  9. gdn-100m/fineweb-edu-10B-s1337/tokenizer.json +0 -0
  10. gdn-100m/fineweb-edu-10B-s1337/tokenizer.model +0 -3
  11. gdn-100m/fineweb-edu-10B-s1337/tokenizer_config.json +0 -43
  12. gdn-100m/fineweb-edu-1B-s1337/.gitkeep +0 -0
  13. gdn-100m/fineweb-edu-1B-s1337/config.json +0 -38
  14. gdn-100m/fineweb-edu-1B-s1337/eval/core_eval.csv +0 -24
  15. gdn-100m/fineweb-edu-1B-s1337/generation_config.json +0 -7
  16. gdn-100m/fineweb-edu-1B-s1337/model.safetensors +0 -3
  17. gdn-100m/fineweb-edu-1B-s1337/special_tokens_map.json +0 -23
  18. gdn-100m/fineweb-edu-1B-s1337/tokenizer.json +0 -0
  19. gdn-100m/fineweb-edu-1B-s1337/tokenizer.model +0 -3
  20. gdn-100m/fineweb-edu-1B-s1337/tokenizer_config.json +0 -43
  21. gdn-100m/fineweb-edu-1B-s2026/.gitkeep +0 -0
  22. gdn-100m/fineweb-edu-1B-s2026/config.json +0 -38
  23. gdn-100m/fineweb-edu-1B-s2026/eval/core_eval.csv +0 -24
  24. gdn-100m/fineweb-edu-1B-s2026/generation_config.json +0 -7
  25. gdn-100m/fineweb-edu-1B-s2026/model.safetensors +0 -3
  26. gdn-100m/fineweb-edu-1B-s2026/special_tokens_map.json +0 -23
  27. gdn-100m/fineweb-edu-1B-s2026/tokenizer.json +0 -0
  28. gdn-100m/fineweb-edu-1B-s2026/tokenizer.model +0 -3
  29. gdn-100m/fineweb-edu-1B-s2026/tokenizer_config.json +0 -43
  30. gdn-100m/fineweb-edu-1B-s42/.gitkeep +0 -0
  31. gdn-100m/fineweb-edu-1B-s42/config.json +0 -38
  32. gdn-100m/fineweb-edu-1B-s42/eval/core_eval.csv +0 -24
  33. gdn-100m/fineweb-edu-1B-s42/generation_config.json +0 -7
  34. gdn-100m/fineweb-edu-1B-s42/model.safetensors +0 -3
  35. gdn-100m/fineweb-edu-1B-s42/special_tokens_map.json +0 -23
  36. gdn-100m/fineweb-edu-1B-s42/tokenizer.json +0 -0
  37. gdn-100m/fineweb-edu-1B-s42/tokenizer.model +0 -3
  38. gdn-100m/fineweb-edu-1B-s42/tokenizer_config.json +0 -43
  39. gdn-13m/fineweb-edu-10B-s1337/.gitkeep +0 -0
  40. gdn-13m/fineweb-edu-10B-s1337/config.json +0 -38
  41. gdn-13m/fineweb-edu-10B-s1337/eval/core_eval.csv +0 -24
  42. gdn-13m/fineweb-edu-10B-s1337/generation_config.json +0 -7
  43. gdn-13m/fineweb-edu-10B-s1337/model.safetensors +0 -3
  44. gdn-13m/fineweb-edu-10B-s1337/special_tokens_map.json +0 -23
  45. gdn-13m/fineweb-edu-10B-s1337/tokenizer.json +0 -0
  46. gdn-13m/fineweb-edu-10B-s1337/tokenizer.model +0 -3
  47. gdn-13m/fineweb-edu-10B-s1337/tokenizer_config.json +0 -43
  48. gdn-13m/fineweb-edu-10B-s2026/.gitkeep +0 -0
  49. gdn-13m/fineweb-edu-10B-s2026/config.json +0 -38
  50. gdn-13m/fineweb-edu-10B-s2026/eval/core_eval.csv +0 -24
README.md CHANGED
@@ -6,7 +6,7 @@ tags:
6
  ---
7
  # MixerLoop
8
 
9
- Canonical GDN and MixerLoop T4 checkpoints, organized by architecture, dataset,
10
  processed-token budget and training seed. Directories containing only `.gitkeep`
11
  are planned experiments, not available models. Historical releases and their
12
  original evaluation results are preserved unchanged under `legacy/`.
@@ -15,7 +15,7 @@ original evaluation results are preserved unchanged under `legacy/`.
15
 
16
  Directories containing `model.safetensors` provide HF weights; the architectures require FLA and the
17
  MixerLoop model registration code. Install the dependencies from
18
- [MixerLoop commit 24b6a923](https://github.com/ruhai-lin/MixerLoop/tree/24b6a9237c1220c39358b0987341596bdcf3926c)
19
  and run from that checkout (or install it as a package). Plain Transformers
20
  without these registrations does not recognize these model types.
21
 
@@ -46,7 +46,7 @@ First-batch runs use context 1024 and global batch 128. The 1B runs contain
46
  7,630 optimizer steps (1,000,079,360 processed tokens); the 10B run contains
47
  76,294 steps (10,000,007,168 processed tokens), all seed 1337.
48
  Actual parameters: MixerLoop-13m 12,896,380; MixerLoop-100m 100,447,266;
49
- GDN-100m 100,444,194; MixerLoop-450m 445,775,120. Other directories are reserved experiment labels,
50
  not claims about completed training or measured parameter counts.
51
 
52
  ## CORE v2 comparison
@@ -55,34 +55,30 @@ Scores are centered CORE v2 scores, stored as fractions. The 13M and 100M models
55
 
56
  Because 13M models show greater run-to-run noise, we ran three seeds (42, 1337, 2026) and report their mean.
57
 
58
- The rightmost column is MixerLoop 450M trained on FineWeb-Edu 100B / seed1337, so its dataset and training budget differ from the smaller models.
59
-
60
- | Task | 13M GDN | 13M MixerLoop | 13M FullLoop | 100M GDN | 100M MixerLoop | 100M FullLoop | 450M MixerLoop |
61
- |---|---:|---:|---:|---:|---:|---:|---:|
62
- | hellaswag_zeroshot | 0.034898 | 0.033526 | 0.029675 | 0.132842 | 0.133771 | 0.167231 | 0.323773 |
63
- | jeopardy | 0.000315 | 0.000157 | 0.000630 | 0.007085 | 0.006613 | 0.010392 | 0.051488 |
64
- | bigbench_qa_wikidata | 0.040106 | 0.031396 | 0.071863 | 0.214950 | 0.257074 | 0.282811 | 0.536932 |
65
- | arc_easy | 0.155069 | 0.152076 | 0.151702 | 0.317621 | 0.341190 | 0.385522 | 0.512907 |
66
- | arc_challenge | -0.037922 | -0.054228 | -0.037543 | -0.001138 | 0.018203 | 0.045506 | 0.130830 |
67
- | copa | -0.113333 | -0.093333 | -0.120000 | 0.000000 | 0.020000 | 0.020000 | 0.300000 |
68
- | commonsense_qa | -0.180258 | -0.234217 | -0.246564 | -0.244278 | -0.015177 | -0.306011 | -0.251137 |
69
- | piqa | 0.162858 | 0.167211 | 0.166123 | 0.316649 | 0.334059 | 0.355822 | 0.404788 |
70
- | openbook_qa | 0.035556 | 0.041778 | 0.042667 | 0.090667 | 0.088000 | 0.098667 | 0.165333 |
71
- | lambada_openai | 0.105764 | 0.117407 | 0.116825 | 0.253639 | 0.246264 | 0.263536 | 0.398603 |
72
- | hellaswag | 0.028923 | 0.029277 | 0.023479 | 0.125938 | 0.126734 | 0.163513 | 0.313948 |
73
- | winograd | 0.072039 | 0.030525 | 0.050061 | 0.172161 | 0.113553 | 0.172161 | 0.384615 |
74
- | winogrande | 0.027624 | 0.020258 | 0.031834 | 0.027624 | 0.000789 | 0.029203 | 0.097080 |
75
- | bigbench_dyck_languages | 0.001333 | 0.008333 | 0.005667 | 0.128000 | 0.172000 | 0.093000 | 0.032000 |
76
- | agi_eval_lsat_ar | -0.051208 | -0.004831 | -0.016425 | -0.014493 | 0.072464 | 0.049275 | 0.020290 |
77
- | bigbench_cs_algorithms | 0.239394 | 0.306566 | 0.221717 | 0.351515 | 0.351515 | 0.352273 | 0.344697 |
78
- | bigbench_operators | 0.134921 | 0.139683 | 0.139682 | 0.095238 | 0.090476 | 0.090476 | 0.142857 |
79
- | bigbench_repeat_copy_logic | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.062500 |
80
- | squad | 0.013844 | 0.008514 | 0.011542 | 0.070766 | 0.047114 | 0.049669 | 0.143519 |
81
- | coqa | 0.044135 | 0.031484 | 0.036077 | 0.113366 | 0.116623 | 0.123888 | 0.180509 |
82
- | boolq | -0.408874 | -0.361661 | -0.354418 | -0.187832 | -0.059070 | -0.054241 | -0.329470 |
83
- | bigbench_language_identification | 0.001689 | 0.007733 | 0.002755 | -0.003467 | 0.010000 | 0.002533 | 0.002933 |
84
- | **Core** | **0.013949** | **0.017166** | **0.014880** | **0.089402** | **0.112373** | **0.108874** | **0.180409** |
85
 
86
  [Download the comparison CSV](eval/core_comparison.csv).
87
-
88
- The 450M run also provides [six-task zero-shot lm-eval scores](mixerloop-450m/fineweb-edu-100B-s1337/eval/lm_eval.csv): ARC-C and HellaSwag use `acc_norm`; ARC-E, PIQA, WinoGrande and BoolQ use `acc`. `Avg` is the arithmetic mean of these six scores. Both evaluation CSVs store scores as fractions.
 
6
  ---
7
  # MixerLoop
8
 
9
+ Canonical GDN, MixerLoop T4 and FullLoop T4 checkpoints, organized by architecture, dataset,
10
  processed-token budget and training seed. Directories containing only `.gitkeep`
11
  are planned experiments, not available models. Historical releases and their
12
  original evaluation results are preserved unchanged under `legacy/`.
 
15
 
16
  Directories containing `model.safetensors` provide HF weights; the architectures require FLA and the
17
  MixerLoop model registration code. Install the dependencies from
18
+ [MixerLoop commit 54a37b8](https://github.com/ruhai-lin/MixerLoop/tree/54a37b807601e690bc8d430cbdfbd87252f9bbbe)
19
  and run from that checkout (or install it as a package). Plain Transformers
20
  without these registrations does not recognize these model types.
21
 
 
46
  7,630 optimizer steps (1,000,079,360 processed tokens); the 10B run contains
47
  76,294 steps (10,000,007,168 processed tokens), all seed 1337.
48
  Actual parameters: MixerLoop-13m 12,896,380; MixerLoop-100m 100,447,266;
49
+ GDN-100m 100,444,194. Other directories are reserved experiment labels,
50
  not claims about completed training or measured parameter counts.
51
 
52
  ## CORE v2 comparison
 
55
 
56
  Because 13M models show greater run-to-run noise, we ran three seeds (42, 1337, 2026) and report their mean.
57
 
58
+ | Task | 13M GDN | 13M MixerLoop | 13M FullLoop | 100M GDN | 100M MixerLoop | 100M FullLoop |
59
+ |---|---:|---:|---:|---:|---:|---:|
60
+ | hellaswag_zeroshot | 0.034898 | 0.033526 | 0.029675 | 0.132842 | 0.133771 | 0.167231 |
61
+ | jeopardy | 0.000315 | 0.000157 | 0.000630 | 0.007085 | 0.006613 | 0.010392 |
62
+ | bigbench_qa_wikidata | 0.040106 | 0.031396 | 0.071863 | 0.214950 | 0.257074 | 0.282811 |
63
+ | arc_easy | 0.155069 | 0.152076 | 0.151702 | 0.317621 | 0.341190 | 0.385522 |
64
+ | arc_challenge | -0.037922 | -0.054228 | -0.037543 | -0.001138 | 0.018203 | 0.045506 |
65
+ | copa | -0.113333 | -0.093333 | -0.120000 | 0.000000 | 0.020000 | 0.020000 |
66
+ | commonsense_qa | -0.180258 | -0.234217 | -0.246564 | -0.244278 | -0.015177 | -0.306011 |
67
+ | piqa | 0.162858 | 0.167211 | 0.166123 | 0.316649 | 0.334059 | 0.355822 |
68
+ | openbook_qa | 0.035556 | 0.041778 | 0.042667 | 0.090667 | 0.088000 | 0.098667 |
69
+ | lambada_openai | 0.105764 | 0.117407 | 0.116825 | 0.253639 | 0.246264 | 0.263536 |
70
+ | hellaswag | 0.028923 | 0.029277 | 0.023479 | 0.125938 | 0.126734 | 0.163513 |
71
+ | winograd | 0.072039 | 0.030525 | 0.050061 | 0.172161 | 0.113553 | 0.172161 |
72
+ | winogrande | 0.027624 | 0.020258 | 0.031834 | 0.027624 | 0.000789 | 0.029203 |
73
+ | bigbench_dyck_languages | 0.001333 | 0.008333 | 0.005667 | 0.128000 | 0.172000 | 0.093000 |
74
+ | agi_eval_lsat_ar | -0.051208 | -0.004831 | -0.016425 | -0.014493 | 0.072464 | 0.049275 |
75
+ | bigbench_cs_algorithms | 0.239394 | 0.306566 | 0.221717 | 0.351515 | 0.351515 | 0.352273 |
76
+ | bigbench_operators | 0.134921 | 0.139683 | 0.139682 | 0.095238 | 0.090476 | 0.090476 |
77
+ | bigbench_repeat_copy_logic | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 | 0.000000 |
78
+ | squad | 0.013844 | 0.008514 | 0.011542 | 0.070766 | 0.047114 | 0.049669 |
79
+ | coqa | 0.044135 | 0.031484 | 0.036077 | 0.113366 | 0.116623 | 0.123888 |
80
+ | boolq | -0.408874 | -0.361661 | -0.354418 | -0.187832 | -0.059070 | -0.054241 |
81
+ | bigbench_language_identification | 0.001689 | 0.007733 | 0.002755 | -0.003467 | 0.010000 | 0.002533 |
82
+ | **Core** | **0.013949** | **0.017166** | **0.014880** | **0.089402** | **0.112373** | **0.108874** |
 
 
83
 
84
  [Download the comparison CSV](eval/core_comparison.csv).
 
 
eval/core_comparison.csv CHANGED
@@ -1,24 +1,24 @@
1
- Task,13M GDN,13M MixerLoop,13M FullLoop,100M GDN,100M MixerLoop,100M FullLoop,450M MixerLoop
2
- hellaswag_zeroshot,0.034898,0.033526,0.029675,0.132842,0.133771,0.167231,0.323773
3
- jeopardy,0.000315,0.000157,0.000630,0.007085,0.006613,0.010392,0.051488
4
- bigbench_qa_wikidata,0.040106,0.031396,0.071863,0.214950,0.257074,0.282811,0.536932
5
- arc_easy,0.155069,0.152076,0.151702,0.317621,0.341190,0.385522,0.512907
6
- arc_challenge,-0.037922,-0.054228,-0.037543,-0.001138,0.018203,0.045506,0.130830
7
- copa,-0.113333,-0.093333,-0.120000,0.000000,0.020000,0.020000,0.300000
8
- commonsense_qa,-0.180258,-0.234217,-0.246564,-0.244278,-0.015177,-0.306011,-0.251137
9
- piqa,0.162858,0.167211,0.166123,0.316649,0.334059,0.355822,0.404788
10
- openbook_qa,0.035556,0.041778,0.042667,0.090667,0.088000,0.098667,0.165333
11
- lambada_openai,0.105764,0.117407,0.116825,0.253639,0.246264,0.263536,0.398603
12
- hellaswag,0.028923,0.029277,0.023479,0.125938,0.126734,0.163513,0.313948
13
- winograd,0.072039,0.030525,0.050061,0.172161,0.113553,0.172161,0.384615
14
- winogrande,0.027624,0.020258,0.031834,0.027624,0.000789,0.029203,0.097080
15
- bigbench_dyck_languages,0.001333,0.008333,0.005667,0.128000,0.172000,0.093000,0.032000
16
- agi_eval_lsat_ar,-0.051208,-0.004831,-0.016425,-0.014493,0.072464,0.049275,0.020290
17
- bigbench_cs_algorithms,0.239394,0.306566,0.221717,0.351515,0.351515,0.352273,0.344697
18
- bigbench_operators,0.134921,0.139683,0.139682,0.095238,0.090476,0.090476,0.142857
19
- bigbench_repeat_copy_logic,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000,0.062500
20
- squad,0.013844,0.008514,0.011542,0.070766,0.047114,0.049669,0.143519
21
- coqa,0.044135,0.031484,0.036077,0.113366,0.116623,0.123888,0.180509
22
- boolq,-0.408874,-0.361661,-0.354418,-0.187832,-0.059070,-0.054241,-0.329470
23
- bigbench_language_identification,0.001689,0.007733,0.002755,-0.003467,0.010000,0.002533,0.002933
24
- Core,0.013949,0.017166,0.014880,0.089402,0.112373,0.108874,0.180409
 
1
+ Task,13M GDN,13M MixerLoop,13M FullLoop,100M GDN,100M MixerLoop,100M FullLoop
2
+ hellaswag_zeroshot,0.034898,0.033526,0.029675,0.132842,0.133771,0.167231
3
+ jeopardy,0.000315,0.000157,0.000630,0.007085,0.006613,0.010392
4
+ bigbench_qa_wikidata,0.040106,0.031396,0.071863,0.214950,0.257074,0.282811
5
+ arc_easy,0.155069,0.152076,0.151702,0.317621,0.341190,0.385522
6
+ arc_challenge,-0.037922,-0.054228,-0.037543,-0.001138,0.018203,0.045506
7
+ copa,-0.113333,-0.093333,-0.120000,0.000000,0.020000,0.020000
8
+ commonsense_qa,-0.180258,-0.234217,-0.246564,-0.244278,-0.015177,-0.306011
9
+ piqa,0.162858,0.167211,0.166123,0.316649,0.334059,0.355822
10
+ openbook_qa,0.035556,0.041778,0.042667,0.090667,0.088000,0.098667
11
+ lambada_openai,0.105764,0.117407,0.116825,0.253639,0.246264,0.263536
12
+ hellaswag,0.028923,0.029277,0.023479,0.125938,0.126734,0.163513
13
+ winograd,0.072039,0.030525,0.050061,0.172161,0.113553,0.172161
14
+ winogrande,0.027624,0.020258,0.031834,0.027624,0.000789,0.029203
15
+ bigbench_dyck_languages,0.001333,0.008333,0.005667,0.128000,0.172000,0.093000
16
+ agi_eval_lsat_ar,-0.051208,-0.004831,-0.016425,-0.014493,0.072464,0.049275
17
+ bigbench_cs_algorithms,0.239394,0.306566,0.221717,0.351515,0.351515,0.352273
18
+ bigbench_operators,0.134921,0.139683,0.139682,0.095238,0.090476,0.090476
19
+ bigbench_repeat_copy_logic,0.000000,0.000000,0.000000,0.000000,0.000000,0.000000
20
+ squad,0.013844,0.008514,0.011542,0.070766,0.047114,0.049669
21
+ coqa,0.044135,0.031484,0.036077,0.113366,0.116623,0.123888
22
+ boolq,-0.408874,-0.361661,-0.354418,-0.187832,-0.059070,-0.054241
23
+ bigbench_language_identification,0.001689,0.007733,0.002755,-0.003467,0.010000,0.002533
24
+ Core,0.013949,0.017166,0.014880,0.089402,0.112373,0.108874
gdn-100m/fineweb-edu-10B-s1337/.gitkeep ADDED
File without changes
gdn-100m/fineweb-edu-10B-s1337/config.json DELETED
@@ -1,37 +0,0 @@
1
- {
2
- "allow_neg_eigval": false,
3
- "architectures": [
4
- "GatedDeltaNetForCausalLM"
5
- ],
6
- "attn": null,
7
- "attn_mode": "chunk",
8
- "attnres_block_size": null,
9
- "bos_token_id": 1,
10
- "conv_size": 4,
11
- "dtype": "float32",
12
- "eos_token_id": 2,
13
- "expand_v": 2,
14
- "fuse_cross_entropy": true,
15
- "fuse_linear_cross_entropy": false,
16
- "fuse_norm": true,
17
- "fuse_swiglu": true,
18
- "head_dim": 64,
19
- "hidden_act": "swish",
20
- "hidden_ratio": 4,
21
- "hidden_size": 768,
22
- "initializer_range": 0.02,
23
- "intermediate_size": 2112,
24
- "max_position_embeddings": 1024,
25
- "model_type": "gated_deltanet",
26
- "norm_eps": 1e-05,
27
- "num_heads": 9,
28
- "num_hidden_layers": 9,
29
- "num_v_heads": null,
30
- "tie_word_embeddings": true,
31
- "transformers_version": "4.57.3",
32
- "use_cache": false,
33
- "use_gate": true,
34
- "use_l2warp": false,
35
- "use_short_conv": true,
36
- "vocab_size": 32000
37
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-100m/fineweb-edu-10B-s1337/eval/core_eval.csv DELETED
@@ -1,24 +0,0 @@
1
- Task,Accuracy,Centered
2
- hellaswag_zeroshot,0.319757,0.093009
3
- jeopardy,0.003307,0.003307
4
- bigbench_qa_wikidata,0.283008,0.283008
5
- arc_easy,0.458333,0.277778
6
- arc_challenge,0.244881,-0.006826
7
- copa,0.500000,0.000000
8
- commonsense_qa,0.266994,-0.227815
9
- piqa,0.616431,0.232862
10
- openbook_qa,0.320000,0.093333
11
- lambada_openai,0.247041,0.247041
12
- hellaswag,0.319259,0.092345
13
- winograd,0.578755,0.157509
14
- winogrande,0.511444,0.022889
15
- bigbench_dyck_languages,0.106000,0.106000
16
- agi_eval_lsat_ar,0.230435,-0.026087
17
- bigbench_cs_algorithms,0.056061,0.056061
18
- bigbench_operators,0.128571,0.128571
19
- bigbench_repeat_copy_logic,0.000000,0.000000
20
- squad,0.019111,0.019111
21
- coqa,0.075410,0.075410
22
- boolq,0.558410,-0.162080
23
- bigbench_language_identification,0.255600,0.007467
24
- Core_v2,,0.066950
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-100m/fineweb-edu-10B-s1337/generation_config.json DELETED
@@ -1,7 +0,0 @@
1
- {
2
- "_from_model_config": true,
3
- "bos_token_id": 1,
4
- "eos_token_id": 2,
5
- "transformers_version": "4.57.3",
6
- "use_cache": false
7
- }
 
 
 
 
 
 
 
 
gdn-100m/fineweb-edu-10B-s1337/model.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:3e2a40057843be1a683c4dea76624dbdd0883286a9ee522a373ceeac2239a34e
3
- size 401794048
 
 
 
 
gdn-100m/fineweb-edu-10B-s1337/special_tokens_map.json DELETED
@@ -1,23 +0,0 @@
1
- {
2
- "bos_token": {
3
- "content": "<s>",
4
- "lstrip": false,
5
- "normalized": false,
6
- "rstrip": false,
7
- "single_word": false
8
- },
9
- "eos_token": {
10
- "content": "</s>",
11
- "lstrip": false,
12
- "normalized": false,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
- "unk_token": {
17
- "content": "<unk>",
18
- "lstrip": false,
19
- "normalized": false,
20
- "rstrip": false,
21
- "single_word": false
22
- }
23
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-100m/fineweb-edu-10B-s1337/tokenizer.json DELETED
The diff for this file is too large to render. See raw diff
 
gdn-100m/fineweb-edu-10B-s1337/tokenizer.model DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
- size 499723
 
 
 
 
gdn-100m/fineweb-edu-10B-s1337/tokenizer_config.json DELETED
@@ -1,43 +0,0 @@
1
- {
2
- "add_bos_token": true,
3
- "add_eos_token": false,
4
- "add_prefix_space": null,
5
- "added_tokens_decoder": {
6
- "0": {
7
- "content": "<unk>",
8
- "lstrip": false,
9
- "normalized": false,
10
- "rstrip": false,
11
- "single_word": false,
12
- "special": true
13
- },
14
- "1": {
15
- "content": "<s>",
16
- "lstrip": false,
17
- "normalized": false,
18
- "rstrip": false,
19
- "single_word": false,
20
- "special": true
21
- },
22
- "2": {
23
- "content": "</s>",
24
- "lstrip": false,
25
- "normalized": false,
26
- "rstrip": false,
27
- "single_word": false,
28
- "special": true
29
- }
30
- },
31
- "bos_token": "<s>",
32
- "clean_up_tokenization_spaces": false,
33
- "eos_token": "</s>",
34
- "extra_special_tokens": {},
35
- "legacy": true,
36
- "model_max_length": 1024,
37
- "pad_token": null,
38
- "sp_model_kwargs": {},
39
- "spaces_between_special_tokens": false,
40
- "tokenizer_class": "LlamaTokenizer",
41
- "unk_token": "<unk>",
42
- "use_default_system_prompt": false
43
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-100m/fineweb-edu-1B-s1337/.gitkeep ADDED
File without changes
gdn-100m/fineweb-edu-1B-s1337/config.json DELETED
@@ -1,38 +0,0 @@
1
- {
2
- "allow_neg_eigval": false,
3
- "architectures": [
4
- "GatedDeltaNetForCausalLM"
5
- ],
6
- "attn": null,
7
- "attn_mode": "chunk",
8
- "attnres_block_size": null,
9
- "bos_token_id": 1,
10
- "conv_size": 4,
11
- "dtype": "float32",
12
- "eos_token_id": 2,
13
- "expand_v": 2,
14
- "fuse_cross_entropy": true,
15
- "fuse_linear_cross_entropy": false,
16
- "fuse_norm": true,
17
- "fuse_swiglu": true,
18
- "head_dim": 64,
19
- "hidden_act": "swish",
20
- "hidden_ratio": 4,
21
- "hidden_size": 768,
22
- "initializer_range": 0.02,
23
- "intermediate_size": 2112,
24
- "loop_count": 1,
25
- "max_position_embeddings": 1024,
26
- "model_type": "gated_deltanet",
27
- "norm_eps": 1e-05,
28
- "num_heads": 9,
29
- "num_hidden_layers": 9,
30
- "num_v_heads": null,
31
- "tie_word_embeddings": true,
32
- "transformers_version": "4.57.3",
33
- "use_cache": false,
34
- "use_gate": true,
35
- "use_l2warp": false,
36
- "use_short_conv": true,
37
- "vocab_size": 32000
38
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-100m/fineweb-edu-1B-s1337/eval/core_eval.csv DELETED
@@ -1,24 +0,0 @@
1
- Task,Accuracy,Centered
2
- hellaswag_zeroshot,0.278729,0.038306
3
- jeopardy,0.000945,0.000945
4
- bigbench_qa_wikidata,0.079573,0.079573
5
- arc_easy,0.377946,0.170595
6
- arc_challenge,0.223549,-0.035267
7
- copa,0.460000,-0.080000
8
- commonsense_qa,0.307125,-0.160594
9
- piqa,0.571273,0.142546
10
- openbook_qa,0.278000,0.037333
11
- lambada_openai,0.145934,0.145934
12
- hellaswag,0.278132,0.037509
13
- winograd,0.534799,0.069597
14
- winogrande,0.514601,0.029203
15
- bigbench_dyck_languages,0.123000,0.123000
16
- agi_eval_lsat_ar,0.217391,-0.043478
17
- bigbench_cs_algorithms,0.238636,0.238636
18
- bigbench_operators,0.119048,0.119048
19
- bigbench_repeat_copy_logic,0.000000,0.000000
20
- squad,0.002554,0.002554
21
- coqa,0.028060,0.028060
22
- boolq,0.446789,-0.455818
23
- bigbench_language_identification,0.255600,0.007467
24
- Core_v2,,0.022507
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-100m/fineweb-edu-1B-s1337/generation_config.json DELETED
@@ -1,7 +0,0 @@
1
- {
2
- "_from_model_config": true,
3
- "bos_token_id": 1,
4
- "eos_token_id": 2,
5
- "transformers_version": "4.57.3",
6
- "use_cache": false
7
- }
 
 
 
 
 
 
 
 
gdn-100m/fineweb-edu-1B-s1337/model.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:e88f0174e4d714fadea5311e308370615adf1ea34478f70d26991393b2d3ccab
3
- size 401794048
 
 
 
 
gdn-100m/fineweb-edu-1B-s1337/special_tokens_map.json DELETED
@@ -1,23 +0,0 @@
1
- {
2
- "bos_token": {
3
- "content": "<s>",
4
- "lstrip": false,
5
- "normalized": false,
6
- "rstrip": false,
7
- "single_word": false
8
- },
9
- "eos_token": {
10
- "content": "</s>",
11
- "lstrip": false,
12
- "normalized": false,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
- "unk_token": {
17
- "content": "<unk>",
18
- "lstrip": false,
19
- "normalized": false,
20
- "rstrip": false,
21
- "single_word": false
22
- }
23
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-100m/fineweb-edu-1B-s1337/tokenizer.json DELETED
The diff for this file is too large to render. See raw diff
 
gdn-100m/fineweb-edu-1B-s1337/tokenizer.model DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
- size 499723
 
 
 
 
gdn-100m/fineweb-edu-1B-s1337/tokenizer_config.json DELETED
@@ -1,43 +0,0 @@
1
- {
2
- "add_bos_token": true,
3
- "add_eos_token": false,
4
- "add_prefix_space": null,
5
- "added_tokens_decoder": {
6
- "0": {
7
- "content": "<unk>",
8
- "lstrip": false,
9
- "normalized": false,
10
- "rstrip": false,
11
- "single_word": false,
12
- "special": true
13
- },
14
- "1": {
15
- "content": "<s>",
16
- "lstrip": false,
17
- "normalized": false,
18
- "rstrip": false,
19
- "single_word": false,
20
- "special": true
21
- },
22
- "2": {
23
- "content": "</s>",
24
- "lstrip": false,
25
- "normalized": false,
26
- "rstrip": false,
27
- "single_word": false,
28
- "special": true
29
- }
30
- },
31
- "bos_token": "<s>",
32
- "clean_up_tokenization_spaces": false,
33
- "eos_token": "</s>",
34
- "extra_special_tokens": {},
35
- "legacy": true,
36
- "model_max_length": 1024,
37
- "pad_token": null,
38
- "sp_model_kwargs": {},
39
- "spaces_between_special_tokens": false,
40
- "tokenizer_class": "LlamaTokenizer",
41
- "unk_token": "<unk>",
42
- "use_default_system_prompt": false
43
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-100m/fineweb-edu-1B-s2026/.gitkeep ADDED
File without changes
gdn-100m/fineweb-edu-1B-s2026/config.json DELETED
@@ -1,38 +0,0 @@
1
- {
2
- "allow_neg_eigval": false,
3
- "architectures": [
4
- "GatedDeltaNetForCausalLM"
5
- ],
6
- "attn": null,
7
- "attn_mode": "chunk",
8
- "attnres_block_size": null,
9
- "bos_token_id": 1,
10
- "conv_size": 4,
11
- "dtype": "float32",
12
- "eos_token_id": 2,
13
- "expand_v": 2,
14
- "fuse_cross_entropy": true,
15
- "fuse_linear_cross_entropy": false,
16
- "fuse_norm": true,
17
- "fuse_swiglu": true,
18
- "head_dim": 64,
19
- "hidden_act": "swish",
20
- "hidden_ratio": 4,
21
- "hidden_size": 768,
22
- "initializer_range": 0.02,
23
- "intermediate_size": 2112,
24
- "loop_count": 1,
25
- "max_position_embeddings": 1024,
26
- "model_type": "gated_deltanet",
27
- "norm_eps": 1e-05,
28
- "num_heads": 9,
29
- "num_hidden_layers": 9,
30
- "num_v_heads": null,
31
- "tie_word_embeddings": true,
32
- "transformers_version": "4.57.3",
33
- "use_cache": false,
34
- "use_gate": true,
35
- "use_l2warp": false,
36
- "use_short_conv": true,
37
- "vocab_size": 32000
38
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-100m/fineweb-edu-1B-s2026/eval/core_eval.csv DELETED
@@ -1,24 +0,0 @@
1
- Task,Accuracy,Centered
2
- hellaswag_zeroshot,0.276539,0.035385
3
- jeopardy,0.000945,0.000945
4
- bigbench_qa_wikidata,0.093499,0.093499
5
- arc_easy,0.373737,0.164983
6
- arc_challenge,0.224403,-0.034130
7
- copa,0.440000,-0.120000
8
- commonsense_qa,0.266994,-0.227815
9
- piqa,0.577258,0.154516
10
- openbook_qa,0.302000,0.069333
11
- lambada_openai,0.151950,0.151950
12
- hellaswag,0.274348,0.032464
13
- winograd,0.556777,0.113553
14
- winogrande,0.518548,0.037096
15
- bigbench_dyck_languages,0.000000,0.000000
16
- agi_eval_lsat_ar,0.239130,-0.014493
17
- bigbench_cs_algorithms,0.276515,0.276515
18
- bigbench_operators,0.090476,0.090476
19
- bigbench_repeat_copy_logic,0.000000,0.000000
20
- squad,0.004825,0.004825
21
- coqa,0.034573,0.034573
22
- boolq,0.518960,-0.265894
23
- bigbench_language_identification,0.255100,0.006800
24
- Core_v2,,0.027481
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-100m/fineweb-edu-1B-s2026/generation_config.json DELETED
@@ -1,7 +0,0 @@
1
- {
2
- "_from_model_config": true,
3
- "bos_token_id": 1,
4
- "eos_token_id": 2,
5
- "transformers_version": "4.57.3",
6
- "use_cache": false
7
- }
 
 
 
 
 
 
 
 
gdn-100m/fineweb-edu-1B-s2026/model.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:e33e7bb4834d140a4e89908f518d3ef43eb5a9c64bd3aeea28def67885fc42fb
3
- size 401794048
 
 
 
 
gdn-100m/fineweb-edu-1B-s2026/special_tokens_map.json DELETED
@@ -1,23 +0,0 @@
1
- {
2
- "bos_token": {
3
- "content": "<s>",
4
- "lstrip": false,
5
- "normalized": false,
6
- "rstrip": false,
7
- "single_word": false
8
- },
9
- "eos_token": {
10
- "content": "</s>",
11
- "lstrip": false,
12
- "normalized": false,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
- "unk_token": {
17
- "content": "<unk>",
18
- "lstrip": false,
19
- "normalized": false,
20
- "rstrip": false,
21
- "single_word": false
22
- }
23
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-100m/fineweb-edu-1B-s2026/tokenizer.json DELETED
The diff for this file is too large to render. See raw diff
 
gdn-100m/fineweb-edu-1B-s2026/tokenizer.model DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
- size 499723
 
 
 
 
gdn-100m/fineweb-edu-1B-s2026/tokenizer_config.json DELETED
@@ -1,43 +0,0 @@
1
- {
2
- "add_bos_token": true,
3
- "add_eos_token": false,
4
- "add_prefix_space": null,
5
- "added_tokens_decoder": {
6
- "0": {
7
- "content": "<unk>",
8
- "lstrip": false,
9
- "normalized": false,
10
- "rstrip": false,
11
- "single_word": false,
12
- "special": true
13
- },
14
- "1": {
15
- "content": "<s>",
16
- "lstrip": false,
17
- "normalized": false,
18
- "rstrip": false,
19
- "single_word": false,
20
- "special": true
21
- },
22
- "2": {
23
- "content": "</s>",
24
- "lstrip": false,
25
- "normalized": false,
26
- "rstrip": false,
27
- "single_word": false,
28
- "special": true
29
- }
30
- },
31
- "bos_token": "<s>",
32
- "clean_up_tokenization_spaces": false,
33
- "eos_token": "</s>",
34
- "extra_special_tokens": {},
35
- "legacy": true,
36
- "model_max_length": 1024,
37
- "pad_token": null,
38
- "sp_model_kwargs": {},
39
- "spaces_between_special_tokens": false,
40
- "tokenizer_class": "LlamaTokenizer",
41
- "unk_token": "<unk>",
42
- "use_default_system_prompt": false
43
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-100m/fineweb-edu-1B-s42/.gitkeep ADDED
File without changes
gdn-100m/fineweb-edu-1B-s42/config.json DELETED
@@ -1,38 +0,0 @@
1
- {
2
- "allow_neg_eigval": false,
3
- "architectures": [
4
- "GatedDeltaNetForCausalLM"
5
- ],
6
- "attn": null,
7
- "attn_mode": "chunk",
8
- "attnres_block_size": null,
9
- "bos_token_id": 1,
10
- "conv_size": 4,
11
- "dtype": "float32",
12
- "eos_token_id": 2,
13
- "expand_v": 2,
14
- "fuse_cross_entropy": true,
15
- "fuse_linear_cross_entropy": false,
16
- "fuse_norm": true,
17
- "fuse_swiglu": true,
18
- "head_dim": 64,
19
- "hidden_act": "swish",
20
- "hidden_ratio": 4,
21
- "hidden_size": 768,
22
- "initializer_range": 0.02,
23
- "intermediate_size": 2112,
24
- "loop_count": 1,
25
- "max_position_embeddings": 1024,
26
- "model_type": "gated_deltanet",
27
- "norm_eps": 1e-05,
28
- "num_heads": 9,
29
- "num_hidden_layers": 9,
30
- "num_v_heads": null,
31
- "tie_word_embeddings": true,
32
- "transformers_version": "4.57.3",
33
- "use_cache": false,
34
- "use_gate": true,
35
- "use_l2warp": false,
36
- "use_short_conv": true,
37
- "vocab_size": 32000
38
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-100m/fineweb-edu-1B-s42/eval/core_eval.csv DELETED
@@ -1,24 +0,0 @@
1
- Task,Accuracy,Centered
2
- hellaswag_zeroshot,0.280323,0.040430
3
- jeopardy,0.000472,0.000472
4
- bigbench_qa_wikidata,0.103735,0.103735
5
- arc_easy,0.372896,0.163861
6
- arc_challenge,0.225256,-0.032992
7
- copa,0.430000,-0.140000
8
- commonsense_qa,0.265356,-0.230559
9
- piqa,0.572361,0.144723
10
- openbook_qa,0.290000,0.053333
11
- lambada_openai,0.155249,0.155249
12
- hellaswag,0.275941,0.034588
13
- winograd,0.523810,0.047619
14
- winogrande,0.508287,0.016575
15
- bigbench_dyck_languages,0.046000,0.046000
16
- agi_eval_lsat_ar,0.234783,-0.020290
17
- bigbench_cs_algorithms,0.250000,0.250000
18
- bigbench_operators,0.104762,0.104762
19
- bigbench_repeat_copy_logic,0.000000,0.000000
20
- squad,0.003217,0.003217
21
- coqa,0.036954,0.036954
22
- boolq,0.557798,-0.163689
23
- bigbench_language_identification,0.251200,0.001600
24
- Core_v2,,0.027981
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-100m/fineweb-edu-1B-s42/generation_config.json DELETED
@@ -1,7 +0,0 @@
1
- {
2
- "_from_model_config": true,
3
- "bos_token_id": 1,
4
- "eos_token_id": 2,
5
- "transformers_version": "4.57.3",
6
- "use_cache": false
7
- }
 
 
 
 
 
 
 
 
gdn-100m/fineweb-edu-1B-s42/model.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:4de1cc0beaaeec8839784f2bccdb7d90769a3840db7f155d788984c88015c365
3
- size 401794048
 
 
 
 
gdn-100m/fineweb-edu-1B-s42/special_tokens_map.json DELETED
@@ -1,23 +0,0 @@
1
- {
2
- "bos_token": {
3
- "content": "<s>",
4
- "lstrip": false,
5
- "normalized": false,
6
- "rstrip": false,
7
- "single_word": false
8
- },
9
- "eos_token": {
10
- "content": "</s>",
11
- "lstrip": false,
12
- "normalized": false,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
- "unk_token": {
17
- "content": "<unk>",
18
- "lstrip": false,
19
- "normalized": false,
20
- "rstrip": false,
21
- "single_word": false
22
- }
23
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-100m/fineweb-edu-1B-s42/tokenizer.json DELETED
The diff for this file is too large to render. See raw diff
 
gdn-100m/fineweb-edu-1B-s42/tokenizer.model DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
- size 499723
 
 
 
 
gdn-100m/fineweb-edu-1B-s42/tokenizer_config.json DELETED
@@ -1,43 +0,0 @@
1
- {
2
- "add_bos_token": true,
3
- "add_eos_token": false,
4
- "add_prefix_space": null,
5
- "added_tokens_decoder": {
6
- "0": {
7
- "content": "<unk>",
8
- "lstrip": false,
9
- "normalized": false,
10
- "rstrip": false,
11
- "single_word": false,
12
- "special": true
13
- },
14
- "1": {
15
- "content": "<s>",
16
- "lstrip": false,
17
- "normalized": false,
18
- "rstrip": false,
19
- "single_word": false,
20
- "special": true
21
- },
22
- "2": {
23
- "content": "</s>",
24
- "lstrip": false,
25
- "normalized": false,
26
- "rstrip": false,
27
- "single_word": false,
28
- "special": true
29
- }
30
- },
31
- "bos_token": "<s>",
32
- "clean_up_tokenization_spaces": false,
33
- "eos_token": "</s>",
34
- "extra_special_tokens": {},
35
- "legacy": true,
36
- "model_max_length": 1024,
37
- "pad_token": null,
38
- "sp_model_kwargs": {},
39
- "spaces_between_special_tokens": false,
40
- "tokenizer_class": "LlamaTokenizer",
41
- "unk_token": "<unk>",
42
- "use_default_system_prompt": false
43
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-13m/fineweb-edu-10B-s1337/.gitkeep ADDED
File without changes
gdn-13m/fineweb-edu-10B-s1337/config.json DELETED
@@ -1,38 +0,0 @@
1
- {
2
- "allow_neg_eigval": false,
3
- "architectures": [
4
- "GatedDeltaNetForCausalLM"
5
- ],
6
- "attn": null,
7
- "attn_mode": "chunk",
8
- "attnres_block_size": null,
9
- "bos_token_id": 1,
10
- "conv_size": 4,
11
- "dtype": "float32",
12
- "eos_token_id": 2,
13
- "expand_v": 2,
14
- "fuse_cross_entropy": true,
15
- "fuse_linear_cross_entropy": false,
16
- "fuse_norm": true,
17
- "fuse_swiglu": true,
18
- "head_dim": 32,
19
- "hidden_act": "swish",
20
- "hidden_ratio": 4,
21
- "hidden_size": 256,
22
- "initializer_range": 0.02,
23
- "intermediate_size": 704,
24
- "loop_count": 1,
25
- "max_position_embeddings": 1024,
26
- "model_type": "gated_deltanet",
27
- "norm_eps": 1e-05,
28
- "num_heads": 6,
29
- "num_hidden_layers": 5,
30
- "num_v_heads": null,
31
- "tie_word_embeddings": true,
32
- "transformers_version": "4.57.3",
33
- "use_cache": false,
34
- "use_gate": true,
35
- "use_l2warp": false,
36
- "use_short_conv": true,
37
- "vocab_size": 32000
38
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-13m/fineweb-edu-10B-s1337/eval/core_eval.csv DELETED
@@ -1,24 +0,0 @@
1
- Task,Accuracy,Centered
2
- hellaswag_zeroshot,0.270165,0.026887
3
- jeopardy,0.000472,0.000472
4
- bigbench_qa_wikidata,0.018749,0.018749
5
- arc_easy,0.343013,0.124018
6
- arc_challenge,0.220990,-0.038680
7
- copa,0.420000,-0.160000
8
- commonsense_qa,0.282555,-0.201750
9
- piqa,0.560936,0.121872
10
- openbook_qa,0.272000,0.029333
11
- lambada_openai,0.100524,0.100524
12
- hellaswag,0.264390,0.019186
13
- winograd,0.487179,-0.025641
14
- winogrande,0.497238,-0.005525
15
- bigbench_dyck_languages,0.015000,0.015000
16
- agi_eval_lsat_ar,0.226087,-0.031884
17
- bigbench_cs_algorithms,0.308333,0.308333
18
- bigbench_operators,0.109524,0.109524
19
- bigbench_repeat_copy_logic,0.000000,0.000000
20
- squad,0.003500,0.003500
21
- coqa,0.015784,0.015784
22
- boolq,0.569113,-0.133913
23
- bigbench_language_identification,0.258300,0.011067
24
- Core_v2,,0.013948
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-13m/fineweb-edu-10B-s1337/generation_config.json DELETED
@@ -1,7 +0,0 @@
1
- {
2
- "_from_model_config": true,
3
- "bos_token_id": 1,
4
- "eos_token_id": 2,
5
- "transformers_version": "4.57.3",
6
- "use_cache": false
7
- }
 
 
 
 
 
 
 
 
gdn-13m/fineweb-edu-10B-s1337/model.safetensors DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:4a1b260e42eb3588a6f7210ac675b7bedb51ca5ab36a3082c70af06488160759
3
- size 51590904
 
 
 
 
gdn-13m/fineweb-edu-10B-s1337/special_tokens_map.json DELETED
@@ -1,23 +0,0 @@
1
- {
2
- "bos_token": {
3
- "content": "<s>",
4
- "lstrip": false,
5
- "normalized": false,
6
- "rstrip": false,
7
- "single_word": false
8
- },
9
- "eos_token": {
10
- "content": "</s>",
11
- "lstrip": false,
12
- "normalized": false,
13
- "rstrip": false,
14
- "single_word": false
15
- },
16
- "unk_token": {
17
- "content": "<unk>",
18
- "lstrip": false,
19
- "normalized": false,
20
- "rstrip": false,
21
- "single_word": false
22
- }
23
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-13m/fineweb-edu-10B-s1337/tokenizer.json DELETED
The diff for this file is too large to render. See raw diff
 
gdn-13m/fineweb-edu-10B-s1337/tokenizer.model DELETED
@@ -1,3 +0,0 @@
1
- version https://git-lfs.github.com/spec/v1
2
- oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
3
- size 499723
 
 
 
 
gdn-13m/fineweb-edu-10B-s1337/tokenizer_config.json DELETED
@@ -1,43 +0,0 @@
1
- {
2
- "add_bos_token": true,
3
- "add_eos_token": false,
4
- "add_prefix_space": null,
5
- "added_tokens_decoder": {
6
- "0": {
7
- "content": "<unk>",
8
- "lstrip": false,
9
- "normalized": false,
10
- "rstrip": false,
11
- "single_word": false,
12
- "special": true
13
- },
14
- "1": {
15
- "content": "<s>",
16
- "lstrip": false,
17
- "normalized": false,
18
- "rstrip": false,
19
- "single_word": false,
20
- "special": true
21
- },
22
- "2": {
23
- "content": "</s>",
24
- "lstrip": false,
25
- "normalized": false,
26
- "rstrip": false,
27
- "single_word": false,
28
- "special": true
29
- }
30
- },
31
- "bos_token": "<s>",
32
- "clean_up_tokenization_spaces": false,
33
- "eos_token": "</s>",
34
- "extra_special_tokens": {},
35
- "legacy": true,
36
- "model_max_length": 1024,
37
- "pad_token": null,
38
- "sp_model_kwargs": {},
39
- "spaces_between_special_tokens": false,
40
- "tokenizer_class": "LlamaTokenizer",
41
- "unk_token": "<unk>",
42
- "use_default_system_prompt": false
43
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-13m/fineweb-edu-10B-s2026/.gitkeep ADDED
File without changes
gdn-13m/fineweb-edu-10B-s2026/config.json DELETED
@@ -1,38 +0,0 @@
1
- {
2
- "allow_neg_eigval": false,
3
- "architectures": [
4
- "GatedDeltaNetForCausalLM"
5
- ],
6
- "attn": null,
7
- "attn_mode": "chunk",
8
- "attnres_block_size": null,
9
- "bos_token_id": 1,
10
- "conv_size": 4,
11
- "dtype": "float32",
12
- "eos_token_id": 2,
13
- "expand_v": 2,
14
- "fuse_cross_entropy": true,
15
- "fuse_linear_cross_entropy": false,
16
- "fuse_norm": true,
17
- "fuse_swiglu": true,
18
- "head_dim": 32,
19
- "hidden_act": "swish",
20
- "hidden_ratio": 4,
21
- "hidden_size": 256,
22
- "initializer_range": 0.02,
23
- "intermediate_size": 704,
24
- "loop_count": 1,
25
- "max_position_embeddings": 1024,
26
- "model_type": "gated_deltanet",
27
- "norm_eps": 1e-05,
28
- "num_heads": 6,
29
- "num_hidden_layers": 5,
30
- "num_v_heads": null,
31
- "tie_word_embeddings": true,
32
- "transformers_version": "4.57.3",
33
- "use_cache": false,
34
- "use_gate": true,
35
- "use_l2warp": false,
36
- "use_short_conv": true,
37
- "vocab_size": 32000
38
- }
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
gdn-13m/fineweb-edu-10B-s2026/eval/core_eval.csv DELETED
@@ -1,24 +0,0 @@
1
- Task,Accuracy,Centered
2
- hellaswag_zeroshot,0.270862,0.027817
3
- jeopardy,0.000472,0.000472
4
- bigbench_qa_wikidata,0.052310,0.052310
5
- arc_easy,0.332912,0.110550
6
- arc_challenge,0.215870,-0.045506
7
- copa,0.510000,0.020000
8
- commonsense_qa,0.257985,-0.242906
9
- piqa,0.562024,0.124048
10
- openbook_qa,0.272000,0.029333
11
- lambada_openai,0.108092,0.108092
12
- hellaswag,0.267875,0.023833
13
- winograd,0.487179,-0.025641
14
- winogrande,0.489345,-0.021310
15
- bigbench_dyck_languages,0.051000,0.051000
16
- agi_eval_lsat_ar,0.226087,-0.031884
17
- bigbench_cs_algorithms,0.313636,0.313636
18
- bigbench_operators,0.123810,0.123810
19
- bigbench_repeat_copy_logic,0.000000,0.000000
20
- squad,0.001703,0.001703
21
- coqa,0.006514,0.006514
22
- boolq,0.469419,-0.396266
23
- bigbench_language_identification,0.260200,0.013600
24
- Core_v2,,0.011055