Upload MixerLoop / FullLoop / GDN checkpoints (15m/42m/110m)

fullloop-110m/config.json

@@ -0,0 +1,31 @@
+{
+  "architectures": [
+    "FullLoopForCausalLM"
+  ],
+  "attn_mode": "chunk",
+  "bos_token_id": 1,
+  "conv_size": 4,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "expand_v": 1.0,
+  "fuse_cross_entropy": true,
+  "fuse_linear_cross_entropy": false,
+  "fuse_norm": true,
+  "fuse_swiglu": true,
+  "head_dim": 64,
+  "hidden_act": "swish",
+  "hidden_ratio": 4,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 2048,
+  "loop_count": 4,
+  "max_position_embeddings": 1024,
+  "model_type": "fullloop",
+  "norm_eps": 1e-05,
+  "num_heads": 12,
+  "num_hidden_layers": 12,
+  "transformers_version": "4.57.3",
+  "use_cache": false,
+  "use_residual": true,
+  "vocab_size": 32000
+}

fullloop-110m/eval/core_eval.csv

@@ -0,0 +1,24 @@
+Task,Accuracy,Centered
+hellaswag_zeroshot,0.433579,0.244772
+jeopardy,0.027870,0.027870
+bigbench_qa_wikidata,0.381477,0.381477
+arc_easy,0.587963,0.450617
+arc_challenge,0.308020,0.077361
+copa,0.560000,0.120000
+commonsense_qa,0.313677,0.142097
+piqa,0.691513,0.383025
+openbook_qa,0.360000,0.146667
+lambada_openai,0.323501,0.323501
+hellaswag,0.432185,0.242913
+winograd,0.593407,0.186813
+winogrande,0.522494,0.044988
+bigbench_dyck_languages,0.199000,0.199000
+agi_eval_lsat_ar,0.256522,0.070652
+bigbench_cs_algorithms,0.431818,0.431818
+bigbench_operators,0.152381,0.152381
+bigbench_repeat_copy_logic,0.031250,0.031250
+squad,0.097919,0.097919
+coqa,0.141927,0.141927
+boolq,0.537920,-0.215999
+bigbench_language_identification,0.249200,0.174037
+CORE,,0.175231

fullloop-110m/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.57.3",
+  "use_cache": false
+}

fullloop-110m/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9200a356411ca5c7b9dacecb3aca79d80e47fd94edfa0014e59630a304b7b22d
+size 467797832

fullloop-110m/provenance.json

@@ -0,0 +1,63 @@
+{
+  "architecture": "fullloop",
+  "best_validation_loss": 2.3416037559509277,
+  "legacy_checkpoint": "climbmix110M.pt",
+  "legacy_sha256": "3420e2f20ea0f3f48db2a25c9e7cd8a82c7e7183dc3502faa60d24f3eb7d9c6e",
+  "model_args": {
+    "conv_size": 4,
+    "dim": 768,
+    "dropout": 0.0,
+    "head_k_dim": 64,
+    "head_v_dim": 64,
+    "hidden_dim": 2048,
+    "loop_count": 4,
+    "max_seq_len": 1024,
+    "n_layers": 12,
+    "norm_eps": 1e-05,
+    "num_heads": 12,
+    "use_residual": true,
+    "vocab_size": 32000
+  },
+  "source_repo": "ruhai-lin/LT2.c",
+  "source_revision": "82cdc6174c9b0b1501a0b1e8aaeaa2a436171b37",
+  "training_config": {
+    "always_save_checkpoint": false,
+    "batch_size": 16,
+    "beta1": 0.9,
+    "beta2": 0.95,
+    "compile": false,
+    "conv_size": 4,
+    "decay_lr": true,
+    "device": "cuda",
+    "dim": 768,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "eval_interval": 2000,
+    "eval_iters": 100,
+    "eval_only": false,
+    "grad_clip": 1.0,
+    "gradient_accumulation_steps": 32,
+    "head_k_dim": 64,
+    "head_v_dim": 64,
+    "hidden_dim": 2048,
+    "init_from": "resume",
+    "learning_rate": 0.0005,
+    "log_interval": 1,
+    "loop_count": 4,
+    "max_iters": 100000,
+    "max_seq_len": 1024,
+    "n_layers": 12,
+    "norm_eps": 1e-05,
+    "num_heads": 12,
+    "out_dir": "outputs/climbmix110M_50B_100k",
+    "use_residual": true,
+    "vocab_size": 32000,
+    "vocab_source": "llama2",
+    "wandb_log": false,
+    "wandb_project": "lt2c",
+    "wandb_run_name": "run2026_06_24_23_42_15",
+    "warmup_iters": 1000,
+    "weight_decay": 0.1
+  },
+  "training_step": 100000
+}

fullloop-110m/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>"
+}

fullloop-110m/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

fullloop-110m/tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1024,
+  "pad_token": null,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>"
+}

fullloop-15m/config.json

@@ -0,0 +1,31 @@
+{
+  "architectures": [
+    "FullLoopForCausalLM"
+  ],
+  "attn_mode": "chunk",
+  "bos_token_id": 1,
+  "conv_size": 4,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "expand_v": 1.0,
+  "fuse_cross_entropy": true,
+  "fuse_linear_cross_entropy": false,
+  "fuse_norm": true,
+  "fuse_swiglu": true,
+  "head_dim": 72,
+  "hidden_act": "swish",
+  "hidden_ratio": 4,
+  "hidden_size": 288,
+  "initializer_range": 0.02,
+  "intermediate_size": 768,
+  "loop_count": 4,
+  "max_position_embeddings": 1024,
+  "model_type": "fullloop",
+  "norm_eps": 1e-05,
+  "num_heads": 4,
+  "num_hidden_layers": 6,
+  "transformers_version": "4.57.3",
+  "use_cache": false,
+  "use_residual": true,
+  "vocab_size": 32000
+}

fullloop-15m/eval/core_eval.csv

@@ -0,0 +1,24 @@
+Task,Accuracy,Centered
+hellaswag_zeroshot,0.284605,0.046140
+jeopardy,0.000472,0.000472
+bigbench_qa_wikidata,0.063826,0.063826
+arc_easy,0.393098,0.190797
+arc_challenge,0.230375,-0.026166
+copa,0.460000,-0.080000
+commonsense_qa,0.199017,-0.001229
+piqa,0.603917,0.207835
+openbook_qa,0.286000,0.048000
+lambada_openai,0.146711,0.146711
+hellaswag,0.281418,0.041891
+winograd,0.534799,0.069597
+winogrande,0.482242,-0.035517
+bigbench_dyck_languages,0.007000,0.007000
+agi_eval_lsat_ar,0.234783,0.043478
+bigbench_cs_algorithms,0.384848,0.384848
+bigbench_operators,0.090476,0.090476
+bigbench_repeat_copy_logic,0.000000,0.000000
+squad,0.016840,0.016840
+coqa,0.055618,0.055618
+boolq,0.528746,-0.240142
+bigbench_language_identification,0.258100,0.183828
+CORE,,0.055196

fullloop-15m/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.57.3",
+  "use_cache": false
+}

fullloop-15m/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9cb78b14c6f12a5796700efddaf34cf464ac26b353fcc7dc8488976331d4c62b
+size 62913768

fullloop-15m/provenance.json

@@ -0,0 +1,63 @@
+{
+  "architecture": "fullloop",
+  "best_validation_loss": 2.9359543323516846,
+  "legacy_checkpoint": "climbmix15M.pt",
+  "legacy_sha256": "927dfa31009123ab146bb69b7134f67f180c14d1bb3477d025b8a7d92d41cca5",
+  "model_args": {
+    "conv_size": 4,
+    "dim": 288,
+    "dropout": 0.0,
+    "head_k_dim": 72,
+    "head_v_dim": 72,
+    "hidden_dim": 768,
+    "loop_count": 4,
+    "max_seq_len": 1024,
+    "n_layers": 6,
+    "norm_eps": 1e-05,
+    "num_heads": 4,
+    "use_residual": true,
+    "vocab_size": 32000
+  },
+  "source_repo": "ruhai-lin/LT2.c",
+  "source_revision": "82cdc6174c9b0b1501a0b1e8aaeaa2a436171b37",
+  "training_config": {
+    "always_save_checkpoint": false,
+    "batch_size": 64,
+    "beta1": 0.9,
+    "beta2": 0.95,
+    "compile": false,
+    "conv_size": 4,
+    "decay_lr": true,
+    "device": "cuda",
+    "dim": 288,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "eval_interval": 2000,
+    "eval_iters": 100,
+    "eval_only": false,
+    "grad_clip": 1.0,
+    "gradient_accumulation_steps": 8,
+    "head_k_dim": 72,
+    "head_v_dim": 72,
+    "hidden_dim": 768,
+    "init_from": "scratch",
+    "learning_rate": 0.0005,
+    "log_interval": 1,
+    "loop_count": 4,
+    "max_iters": 100000,
+    "max_seq_len": 1024,
+    "n_layers": 6,
+    "norm_eps": 1e-05,
+    "num_heads": 4,
+    "out_dir": "outputs/climbmix15M_50B_100k",
+    "use_residual": true,
+    "vocab_size": 32000,
+    "vocab_source": "llama2",
+    "wandb_log": false,
+    "wandb_project": "lt2c",
+    "wandb_run_name": "run2026_06_17_02_08_27",
+    "warmup_iters": 1000,
+    "weight_decay": 0.1
+  },
+  "training_step": 100000
+}

fullloop-15m/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>"
+}

fullloop-15m/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

fullloop-15m/tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1024,
+  "pad_token": null,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>"
+}

fullloop-42m/config.json

@@ -0,0 +1,31 @@
+{
+  "architectures": [
+    "FullLoopForCausalLM"
+  ],
+  "attn_mode": "chunk",
+  "bos_token_id": 1,
+  "conv_size": 4,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "expand_v": 1.0,
+  "fuse_cross_entropy": true,
+  "fuse_linear_cross_entropy": false,
+  "fuse_norm": true,
+  "fuse_swiglu": true,
+  "head_dim": 64,
+  "hidden_act": "swish",
+  "hidden_ratio": 4,
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "intermediate_size": 1376,
+  "loop_count": 4,
+  "max_position_embeddings": 1024,
+  "model_type": "fullloop",
+  "norm_eps": 1e-05,
+  "num_heads": 8,
+  "num_hidden_layers": 8,
+  "transformers_version": "4.57.3",
+  "use_cache": false,
+  "use_residual": true,
+  "vocab_size": 32000
+}

fullloop-42m/eval/core_eval.csv

@@ -0,0 +1,24 @@
+Task,Accuracy,Centered
+hellaswag_zeroshot,0.340669,0.120892
+jeopardy,0.007558,0.007558
+bigbench_qa_wikidata,0.179962,0.179962
+arc_easy,0.480640,0.307520
+arc_challenge,0.248294,-0.002275
+copa,0.450000,-0.100000
+commonsense_qa,0.212121,0.015152
+piqa,0.651251,0.302503
+openbook_qa,0.312000,0.082667
+lambada_openai,0.234232,0.234232
+hellaswag,0.336288,0.115050
+winograd,0.597070,0.194139
+winogrande,0.528019,0.056038
+bigbench_dyck_languages,0.067000,0.067000
+agi_eval_lsat_ar,0.234783,0.043478
+bigbench_cs_algorithms,0.361364,0.361364
+bigbench_operators,0.123810,0.123810
+bigbench_repeat_copy_logic,0.000000,0.000000
+squad,0.059224,0.059224
+coqa,0.099712,0.099712
+boolq,0.586239,-0.088846
+bigbench_language_identification,0.254900,0.180308
+CORE,,0.107249

fullloop-42m/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.57.3",
+  "use_cache": false
+}

fullloop-42m/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8e6c9e6dbc3548bae0c3d20127dcfbac40ed67684b366f6627f7dc73cfb3d8f3
+size 175631976

fullloop-42m/provenance.json

@@ -0,0 +1,63 @@
+{
+  "architecture": "fullloop",
+  "best_validation_loss": 2.5824697017669678,
+  "legacy_checkpoint": "climbmix42M.pt",
+  "legacy_sha256": "f95bea3bcfa16be0dfe04ed5260d53f5e3a777b4a467cb5937d63c6cdede5250",
+  "model_args": {
+    "conv_size": 4,
+    "dim": 512,
+    "dropout": 0.0,
+    "head_k_dim": 64,
+    "head_v_dim": 64,
+    "hidden_dim": 1376,
+    "loop_count": 4,
+    "max_seq_len": 1024,
+    "n_layers": 8,
+    "norm_eps": 1e-05,
+    "num_heads": 8,
+    "use_residual": true,
+    "vocab_size": 32000
+  },
+  "source_repo": "ruhai-lin/LT2.c",
+  "source_revision": "82cdc6174c9b0b1501a0b1e8aaeaa2a436171b37",
+  "training_config": {
+    "always_save_checkpoint": false,
+    "batch_size": 32,
+    "beta1": 0.9,
+    "beta2": 0.95,
+    "compile": false,
+    "conv_size": 4,
+    "decay_lr": true,
+    "device": "cuda",
+    "dim": 512,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "eval_interval": 2000,
+    "eval_iters": 100,
+    "eval_only": false,
+    "grad_clip": 1.0,
+    "gradient_accumulation_steps": 16,
+    "head_k_dim": 64,
+    "head_v_dim": 64,
+    "hidden_dim": 1376,
+    "init_from": "scratch",
+    "learning_rate": 0.0005,
+    "log_interval": 1,
+    "loop_count": 4,
+    "max_iters": 100000,
+    "max_seq_len": 1024,
+    "n_layers": 8,
+    "norm_eps": 1e-05,
+    "num_heads": 8,
+    "out_dir": "outputs/climbmix42M_50B_100k",
+    "use_residual": true,
+    "vocab_size": 32000,
+    "vocab_source": "llama2",
+    "wandb_log": false,
+    "wandb_project": "lt2c",
+    "wandb_run_name": "run2026_06_25_09_41_13",
+    "warmup_iters": 1000,
+    "weight_decay": 0.1
+  },
+  "training_step": 100000
+}

fullloop-42m/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>"
+}

fullloop-42m/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

fullloop-42m/tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1024,
+  "pad_token": null,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>"
+}

gdn-110m/config.json

@@ -0,0 +1,37 @@
+{
+  "allow_neg_eigval": false,
+  "architectures": [
+    "GatedDeltaNetForCausalLM"
+  ],
+  "attn": null,
+  "attn_mode": "chunk",
+  "attnres_block_size": null,
+  "bos_token_id": 1,
+  "conv_size": 4,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "expand_v": 1.0,
+  "fuse_cross_entropy": true,
+  "fuse_linear_cross_entropy": false,
+  "fuse_norm": true,
+  "fuse_swiglu": true,
+  "head_dim": 64,
+  "hidden_act": "swish",
+  "hidden_ratio": 4,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 2048,
+  "max_position_embeddings": 1024,
+  "model_type": "gated_deltanet",
+  "norm_eps": 1e-05,
+  "num_heads": 12,
+  "num_hidden_layers": 12,
+  "num_v_heads": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.3",
+  "use_cache": false,
+  "use_gate": true,
+  "use_l2warp": false,
+  "use_short_conv": true,
+  "vocab_size": 32000
+}

gdn-110m/eval/core_eval.csv

@@ -0,0 +1,24 @@
+Task,Accuracy,Centered
+hellaswag_zeroshot,0.397132,0.196176
+jeopardy,0.013226,0.013226
+bigbench_qa_wikidata,0.314798,0.314798
+arc_easy,0.543350,0.391134
+arc_challenge,0.275597,0.034130
+copa,0.510000,0.020000
+commonsense_qa,0.205569,0.006962
+piqa,0.688248,0.376496
+openbook_qa,0.330000,0.106667
+lambada_openai,0.306035,0.306035
+hellaswag,0.391655,0.188873
+winograd,0.560440,0.120879
+winogrande,0.520916,0.041831
+bigbench_dyck_languages,0.150000,0.150000
+agi_eval_lsat_ar,0.269565,0.086957
+bigbench_cs_algorithms,0.450758,0.450758
+bigbench_operators,0.147619,0.147619
+bigbench_repeat_copy_logic,0.000000,0.000000
+squad,0.064522,0.064522
+coqa,0.115621,0.115621
+boolq,0.544954,-0.197489
+bigbench_language_identification,0.254700,0.180088
+CORE,,0.141604

gdn-110m/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.57.3",
+  "use_cache": false
+}

gdn-110m/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a9f2d623eb6bde918ebb45cacfeab5848b8d72ffb8812e9435540a33be315fdb
+size 467785288

gdn-110m/provenance.json

@@ -0,0 +1,59 @@
+{
+  "architecture": "gdn",
+  "best_validation_loss": 2.4006850719451904,
+  "legacy_checkpoint": "climbmix110M.pt",
+  "legacy_sha256": "2394dee54eb4e87d5038bc45287d70606dad16a0578c0f7b5fa66e150dd2f024",
+  "model_args": {
+    "conv_size": 4,
+    "dim": 768,
+    "dropout": 0.0,
+    "head_k_dim": 64,
+    "head_v_dim": 64,
+    "hidden_dim": 2048,
+    "max_seq_len": 1024,
+    "n_layers": 12,
+    "norm_eps": 1e-05,
+    "num_heads": 12,
+    "vocab_size": 32000
+  },
+  "source_repo": "ruhai-lin/gdn.c",
+  "source_revision": "667dd7d6592152f65ea35e9e0c497f1ecec050ff",
+  "training_config": {
+    "always_save_checkpoint": false,
+    "batch_size": 64,
+    "beta1": 0.9,
+    "beta2": 0.95,
+    "compile": false,
+    "conv_size": 4,
+    "decay_lr": true,
+    "device": "cuda",
+    "dim": 768,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "eval_interval": 2000,
+    "eval_iters": 100,
+    "eval_only": false,
+    "grad_clip": 1.0,
+    "gradient_accumulation_steps": 8,
+    "head_k_dim": 64,
+    "head_v_dim": 64,
+    "hidden_dim": 2048,
+    "init_from": "resume",
+    "learning_rate": 0.0005,
+    "log_interval": 1,
+    "max_iters": 100000,
+    "max_seq_len": 1024,
+    "n_layers": 12,
+    "norm_eps": 1e-05,
+    "num_heads": 12,
+    "out_dir": "outputs/climbmix110M_50B_100k",
+    "vocab_size": 32000,
+    "vocab_source": "llama2",
+    "wandb_log": false,
+    "wandb_project": "gdnc",
+    "wandb_run_name": "run2026_06_08_21_19_23",
+    "warmup_iters": 1000,
+    "weight_decay": 0.1
+  },
+  "training_step": 100000
+}

gdn-110m/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>"
+}

gdn-110m/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

gdn-110m/tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1024,
+  "pad_token": null,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>"
+}

gdn-15m/config.json

@@ -0,0 +1,37 @@
+{
+  "allow_neg_eigval": false,
+  "architectures": [
+    "GatedDeltaNetForCausalLM"
+  ],
+  "attn": null,
+  "attn_mode": "chunk",
+  "attnres_block_size": null,
+  "bos_token_id": 1,
+  "conv_size": 4,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "expand_v": 1.0,
+  "fuse_cross_entropy": true,
+  "fuse_linear_cross_entropy": false,
+  "fuse_norm": true,
+  "fuse_swiglu": true,
+  "head_dim": 72,
+  "hidden_act": "swish",
+  "hidden_ratio": 4,
+  "hidden_size": 288,
+  "initializer_range": 0.02,
+  "intermediate_size": 768,
+  "max_position_embeddings": 1024,
+  "model_type": "gated_deltanet",
+  "norm_eps": 1e-05,
+  "num_heads": 4,
+  "num_hidden_layers": 6,
+  "num_v_heads": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.3",
+  "use_cache": false,
+  "use_gate": true,
+  "use_l2warp": false,
+  "use_short_conv": true,
+  "vocab_size": 32000
+}

gdn-15m/eval/core_eval.csv

@@ -0,0 +1,24 @@
+Task,Accuracy,Centered
+hellaswag_zeroshot,0.281518,0.042024
+jeopardy,0.001417,0.001417
+bigbench_qa_wikidata,0.084199,0.084199
+arc_easy,0.385101,0.180135
+arc_challenge,0.226962,-0.030717
+copa,0.420000,-0.160000
+commonsense_qa,0.209664,0.012080
+piqa,0.600653,0.201306
+openbook_qa,0.278000,0.037333
+lambada_openai,0.130409,0.130409
+hellaswag,0.275443,0.033924
+winograd,0.501832,0.003663
+winogrande,0.490923,-0.018153
+bigbench_dyck_languages,0.011000,0.011000
+agi_eval_lsat_ar,0.256522,0.070652
+bigbench_cs_algorithms,0.394697,0.394697
+bigbench_operators,0.138095,0.138095
+bigbench_repeat_copy_logic,0.000000,0.000000
+squad,0.009934,0.009934
+coqa,0.044094,0.044094
+boolq,0.519572,-0.264285
+bigbench_language_identification,0.254300,0.179648
+CORE,,0.050066

gdn-15m/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.57.3",
+  "use_cache": false
+}

gdn-15m/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fad0580d4bc5d64785e2cd0217d2c40b95836e9d1ccfdf2b946a4bdac2826059
+size 62908992

gdn-15m/provenance.json

@@ -0,0 +1,59 @@
+{
+  "architecture": "gdn",
+  "best_validation_loss": 2.994854211807251,
+  "legacy_checkpoint": "climbmix15M.pt",
+  "legacy_sha256": "6d1035df306691380269ac595f72d87315120cf946127b1277d8a93310ffd628",
+  "model_args": {
+    "conv_size": 4,
+    "dim": 288,
+    "dropout": 0.0,
+    "head_k_dim": 72,
+    "head_v_dim": 72,
+    "hidden_dim": 768,
+    "max_seq_len": 1024,
+    "n_layers": 6,
+    "norm_eps": 1e-05,
+    "num_heads": 4,
+    "vocab_size": 32000
+  },
+  "source_repo": "ruhai-lin/gdn.c",
+  "source_revision": "667dd7d6592152f65ea35e9e0c497f1ecec050ff",
+  "training_config": {
+    "always_save_checkpoint": false,
+    "batch_size": 64,
+    "beta1": 0.9,
+    "beta2": 0.95,
+    "compile": false,
+    "conv_size": 4,
+    "decay_lr": true,
+    "device": "cuda",
+    "dim": 288,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "eval_interval": 2000,
+    "eval_iters": 100,
+    "eval_only": false,
+    "grad_clip": 1.0,
+    "gradient_accumulation_steps": 8,
+    "head_k_dim": 72,
+    "head_v_dim": 72,
+    "hidden_dim": 768,
+    "init_from": "scratch",
+    "learning_rate": 0.0005,
+    "log_interval": 1,
+    "max_iters": 100000,
+    "max_seq_len": 1024,
+    "n_layers": 6,
+    "norm_eps": 1e-05,
+    "num_heads": 4,
+    "out_dir": "outputs/climbmix15M_50B_100k",
+    "vocab_size": 32000,
+    "vocab_source": "llama2",
+    "wandb_log": false,
+    "wandb_project": "gdnc",
+    "wandb_run_name": "run2026_06_13_13_54_10",
+    "warmup_iters": 1000,
+    "weight_decay": 0.1
+  },
+  "training_step": 100000
+}

gdn-15m/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>"
+}

gdn-15m/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

gdn-15m/tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1024,
+  "pad_token": null,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>"
+}

gdn-42m/config.json

@@ -0,0 +1,37 @@
+{
+  "allow_neg_eigval": false,
+  "architectures": [
+    "GatedDeltaNetForCausalLM"
+  ],
+  "attn": null,
+  "attn_mode": "chunk",
+  "attnres_block_size": null,
+  "bos_token_id": 1,
+  "conv_size": 4,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "expand_v": 1.0,
+  "fuse_cross_entropy": true,
+  "fuse_linear_cross_entropy": false,
+  "fuse_norm": true,
+  "fuse_swiglu": true,
+  "head_dim": 64,
+  "hidden_act": "swish",
+  "hidden_ratio": 4,
+  "hidden_size": 512,
+  "initializer_range": 0.02,
+  "intermediate_size": 1376,
+  "max_position_embeddings": 1024,
+  "model_type": "gated_deltanet",
+  "norm_eps": 1e-05,
+  "num_heads": 8,
+  "num_hidden_layers": 8,
+  "num_v_heads": null,
+  "tie_word_embeddings": true,
+  "transformers_version": "4.57.3",
+  "use_cache": false,
+  "use_gate": true,
+  "use_l2warp": false,
+  "use_short_conv": true,
+  "vocab_size": 32000
+}

gdn-42m/eval/core_eval.csv

@@ -0,0 +1,24 @@
+Task,Accuracy,Centered
+hellaswag_zeroshot,0.322944,0.097258
+jeopardy,0.004251,0.004251
+bigbench_qa_wikidata,0.200827,0.200827
+arc_easy,0.453283,0.271044
+arc_challenge,0.244881,-0.006826
+copa,0.490000,-0.020000
+commonsense_qa,0.316953,0.146192
+piqa,0.627312,0.254625
+openbook_qa,0.294000,0.058667
+lambada_openai,0.214050,0.214050
+hellaswag,0.318861,0.091814
+winograd,0.564103,0.128205
+winogrande,0.498816,-0.002368
+bigbench_dyck_languages,0.032000,0.032000
+agi_eval_lsat_ar,0.221739,0.027174
+bigbench_cs_algorithms,0.419697,0.419697
+bigbench_operators,0.128571,0.128571
+bigbench_repeat_copy_logic,0.000000,0.000000
+squad,0.034626,0.034626
+coqa,0.086434,0.086434
+boolq,0.497554,-0.322228
+bigbench_language_identification,0.251400,0.176458
+CORE,,0.091840

gdn-42m/generation_config.json

@@ -0,0 +1,7 @@
+{
+  "_from_model_config": true,
+  "bos_token_id": 1,
+  "eos_token_id": 2,
+  "transformers_version": "4.57.3",
+  "use_cache": false
+}

gdn-42m/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:4688b5ff2cde237ca54e439920eea8678abc8a05b8bd0b2e39be8cf47725f8d5
+size 175623568

gdn-42m/provenance.json

@@ -0,0 +1,59 @@
+{
+  "architecture": "gdn",
+  "best_validation_loss": 2.652017116546631,
+  "legacy_checkpoint": "climbmix42M.pt",
+  "legacy_sha256": "2de4cbfe4b2a421813bc7cbf0bfc78c1e4f84627951c358b103f952dd12dfd70",
+  "model_args": {
+    "conv_size": 4,
+    "dim": 512,
+    "dropout": 0.0,
+    "head_k_dim": 64,
+    "head_v_dim": 64,
+    "hidden_dim": 1376,
+    "max_seq_len": 1024,
+    "n_layers": 8,
+    "norm_eps": 1e-05,
+    "num_heads": 8,
+    "vocab_size": 32000
+  },
+  "source_repo": "ruhai-lin/gdn.c",
+  "source_revision": "667dd7d6592152f65ea35e9e0c497f1ecec050ff",
+  "training_config": {
+    "always_save_checkpoint": false,
+    "batch_size": 64,
+    "beta1": 0.9,
+    "beta2": 0.95,
+    "compile": false,
+    "conv_size": 4,
+    "decay_lr": true,
+    "device": "cuda",
+    "dim": 512,
+    "dropout": 0.0,
+    "dtype": "bfloat16",
+    "eval_interval": 2000,
+    "eval_iters": 100,
+    "eval_only": false,
+    "grad_clip": 1.0,
+    "gradient_accumulation_steps": 8,
+    "head_k_dim": 64,
+    "head_v_dim": 64,
+    "hidden_dim": 1376,
+    "init_from": "scratch",
+    "learning_rate": 0.0005,
+    "log_interval": 1,
+    "max_iters": 100000,
+    "max_seq_len": 1024,
+    "n_layers": 8,
+    "norm_eps": 1e-05,
+    "num_heads": 8,
+    "out_dir": "outputs/climbmix42M_50B_100k",
+    "vocab_size": 32000,
+    "vocab_source": "llama2",
+    "wandb_log": false,
+    "wandb_project": "gdnc",
+    "wandb_run_name": "run2026_06_14_07_31_51",
+    "warmup_iters": 1000,
+    "weight_decay": 0.1
+  },
+  "training_step": 100000
+}

gdn-42m/special_tokens_map.json

@@ -0,0 +1,5 @@
+{
+  "bos_token": "<s>",
+  "eos_token": "</s>",
+  "unk_token": "<unk>"
+}

gdn-42m/tokenizer.model

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347
+size 499723

gdn-42m/tokenizer_config.json

@@ -0,0 +1,12 @@
+{
+  "add_bos_token": true,
+  "add_eos_token": false,
+  "bos_token": "<s>",
+  "clean_up_tokenization_spaces": false,
+  "eos_token": "</s>",
+  "legacy": true,
+  "model_max_length": 1024,
+  "pad_token": null,
+  "tokenizer_class": "LlamaTokenizer",
+  "unk_token": "<unk>"
+}

mixerloop-110m/config.json

@@ -0,0 +1,31 @@
+{
+  "architectures": [
+    "MixerLoopForCausalLM"
+  ],
+  "attn_mode": "chunk",
+  "bos_token_id": 1,
+  "conv_size": 4,
+  "dtype": "float32",
+  "eos_token_id": 2,
+  "expand_v": 1.0,
+  "fuse_cross_entropy": true,
+  "fuse_linear_cross_entropy": false,
+  "fuse_norm": true,
+  "fuse_swiglu": true,
+  "head_dim": 64,
+  "hidden_act": "swish",
+  "hidden_ratio": 4,
+  "hidden_size": 768,
+  "initializer_range": 0.02,
+  "intermediate_size": 2048,
+  "loop_count": 4,
+  "max_position_embeddings": 1024,
+  "model_type": "mixerloop",
+  "norm_eps": 1e-05,
+  "num_heads": 12,
+  "num_hidden_layers": 12,
+  "transformers_version": "4.57.3",
+  "use_cache": false,
+  "use_residual": true,
+  "vocab_size": 32000
+}

mixerloop-110m/eval/core_eval.csv

@@ -0,0 +1,24 @@
+Task,Accuracy,Centered
+hellaswag_zeroshot,0.408484,0.211312
+jeopardy,0.024091,0.024091
+bigbench_qa_wikidata,0.352985,0.352985
+arc_easy,0.563973,0.418631
+arc_challenge,0.302048,0.069397
+copa,0.540000,0.080000
+commonsense_qa,0.283374,0.104218
+piqa,0.688792,0.377584
+openbook_qa,0.338000,0.117333
+lambada_openai,0.306618,0.306618
+hellaswag,0.404800,0.206400
+winograd,0.582418,0.164835
+winogrande,0.513812,0.027624
+bigbench_dyck_languages,0.098000,0.098000
+agi_eval_lsat_ar,0.230435,0.038043
+bigbench_cs_algorithms,0.430303,0.430303
+bigbench_operators,0.123810,0.123810
+bigbench_repeat_copy_logic,0.000000,0.000000
+squad,0.080322,0.080322
+coqa,0.126018,0.126018
+boolq,0.574006,-0.121037
+bigbench_language_identification,0.260000,0.185919
+CORE,,0.155564