diff --git a/fullloop-110m/config.json b/fullloop-110m/config.json new file mode 100644 index 0000000000000000000000000000000000000000..033cc7b6524395b19d3a240868f4abaee9242fe2 --- /dev/null +++ b/fullloop-110m/config.json @@ -0,0 +1,31 @@ +{ + "architectures": [ + "FullLoopForCausalLM" + ], + "attn_mode": "chunk", + "bos_token_id": 1, + "conv_size": 4, + "dtype": "float32", + "eos_token_id": 2, + "expand_v": 1.0, + "fuse_cross_entropy": true, + "fuse_linear_cross_entropy": false, + "fuse_norm": true, + "fuse_swiglu": true, + "head_dim": 64, + "hidden_act": "swish", + "hidden_ratio": 4, + "hidden_size": 768, + "initializer_range": 0.02, + "intermediate_size": 2048, + "loop_count": 4, + "max_position_embeddings": 1024, + "model_type": "fullloop", + "norm_eps": 1e-05, + "num_heads": 12, + "num_hidden_layers": 12, + "transformers_version": "4.57.3", + "use_cache": false, + "use_residual": true, + "vocab_size": 32000 +} diff --git a/fullloop-110m/eval/core_eval.csv b/fullloop-110m/eval/core_eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..512d70f17e25401202a479421d8d1e81d752f88a --- /dev/null +++ b/fullloop-110m/eval/core_eval.csv @@ -0,0 +1,24 @@ +Task,Accuracy,Centered +hellaswag_zeroshot,0.433579,0.244772 +jeopardy,0.027870,0.027870 +bigbench_qa_wikidata,0.381477,0.381477 +arc_easy,0.587963,0.450617 +arc_challenge,0.308020,0.077361 +copa,0.560000,0.120000 +commonsense_qa,0.313677,0.142097 +piqa,0.691513,0.383025 +openbook_qa,0.360000,0.146667 +lambada_openai,0.323501,0.323501 +hellaswag,0.432185,0.242913 +winograd,0.593407,0.186813 +winogrande,0.522494,0.044988 +bigbench_dyck_languages,0.199000,0.199000 +agi_eval_lsat_ar,0.256522,0.070652 +bigbench_cs_algorithms,0.431818,0.431818 +bigbench_operators,0.152381,0.152381 +bigbench_repeat_copy_logic,0.031250,0.031250 +squad,0.097919,0.097919 +coqa,0.141927,0.141927 +boolq,0.537920,-0.215999 +bigbench_language_identification,0.249200,0.174037 +CORE,,0.175231 diff --git a/fullloop-110m/generation_config.json b/fullloop-110m/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9032a723674e6d0bdbef80891901e5200e077a48 --- /dev/null +++ b/fullloop-110m/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "transformers_version": "4.57.3", + "use_cache": false +} diff --git a/fullloop-110m/model.safetensors b/fullloop-110m/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..8920fb9bb0d16cf3084b28fb37cd827fcb11312c --- /dev/null +++ b/fullloop-110m/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9200a356411ca5c7b9dacecb3aca79d80e47fd94edfa0014e59630a304b7b22d +size 467797832 diff --git a/fullloop-110m/provenance.json b/fullloop-110m/provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..6e9244b3ba430b3be84dad9533acb079cc50b945 --- /dev/null +++ b/fullloop-110m/provenance.json @@ -0,0 +1,63 @@ +{ + "architecture": "fullloop", + "best_validation_loss": 2.3416037559509277, + "legacy_checkpoint": "climbmix110M.pt", + "legacy_sha256": "3420e2f20ea0f3f48db2a25c9e7cd8a82c7e7183dc3502faa60d24f3eb7d9c6e", + "model_args": { + "conv_size": 4, + "dim": 768, + "dropout": 0.0, + "head_k_dim": 64, + "head_v_dim": 64, + "hidden_dim": 2048, + "loop_count": 4, + "max_seq_len": 1024, + "n_layers": 12, + "norm_eps": 1e-05, + "num_heads": 12, + "use_residual": true, + "vocab_size": 32000 + }, + "source_repo": "ruhai-lin/LT2.c", + "source_revision": "82cdc6174c9b0b1501a0b1e8aaeaa2a436171b37", + "training_config": { + "always_save_checkpoint": false, + "batch_size": 16, + "beta1": 0.9, + "beta2": 0.95, + "compile": false, + "conv_size": 4, + "decay_lr": true, + "device": "cuda", + "dim": 768, + "dropout": 0.0, + "dtype": "bfloat16", + "eval_interval": 2000, + "eval_iters": 100, + "eval_only": false, + "grad_clip": 1.0, + "gradient_accumulation_steps": 32, + "head_k_dim": 64, + "head_v_dim": 64, + "hidden_dim": 2048, + "init_from": "resume", + "learning_rate": 0.0005, + "log_interval": 1, + "loop_count": 4, + "max_iters": 100000, + "max_seq_len": 1024, + "n_layers": 12, + "norm_eps": 1e-05, + "num_heads": 12, + "out_dir": "outputs/climbmix110M_50B_100k", + "use_residual": true, + "vocab_size": 32000, + "vocab_source": "llama2", + "wandb_log": false, + "wandb_project": "lt2c", + "wandb_run_name": "run2026_06_24_23_42_15", + "warmup_iters": 1000, + "weight_decay": 0.1 + }, + "training_step": 100000 +} diff --git a/fullloop-110m/special_tokens_map.json b/fullloop-110m/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..a52c50a199269393cd1548c7e6a77a654bd2001b --- /dev/null +++ b/fullloop-110m/special_tokens_map.json @@ -0,0 +1,5 @@ +{ + "bos_token": "", + "eos_token": "", + "unk_token": "" +} diff --git a/fullloop-110m/tokenizer.model b/fullloop-110m/tokenizer.model new file mode 100644 index 0000000000000000000000000000000000000000..6c00c742ce03c627d6cd5b795984876fa49fa899 --- /dev/null +++ b/fullloop-110m/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 +size 499723 diff --git a/fullloop-110m/tokenizer_config.json b/fullloop-110m/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f8a69e9757d4423e3ee4becf59bc6cc584163226 --- /dev/null +++ b/fullloop-110m/tokenizer_config.json @@ -0,0 +1,12 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": true, + "model_max_length": 1024, + "pad_token": null, + "tokenizer_class": "LlamaTokenizer", + "unk_token": "" +} diff --git a/fullloop-15m/config.json b/fullloop-15m/config.json new file mode 100644 index 0000000000000000000000000000000000000000..307f6a410066d5a9455a37c81eac0ff9ae246458 --- /dev/null +++ b/fullloop-15m/config.json @@ -0,0 +1,31 @@ +{ + "architectures": [ + "FullLoopForCausalLM" + ], + "attn_mode": "chunk", + "bos_token_id": 1, + "conv_size": 4, + "dtype": "float32", + "eos_token_id": 2, + "expand_v": 1.0, + "fuse_cross_entropy": true, + "fuse_linear_cross_entropy": false, + "fuse_norm": true, + "fuse_swiglu": true, + "head_dim": 72, + "hidden_act": "swish", + "hidden_ratio": 4, + "hidden_size": 288, + "initializer_range": 0.02, + "intermediate_size": 768, + "loop_count": 4, + "max_position_embeddings": 1024, + "model_type": "fullloop", + "norm_eps": 1e-05, + "num_heads": 4, + "num_hidden_layers": 6, + "transformers_version": "4.57.3", + "use_cache": false, + "use_residual": true, + "vocab_size": 32000 +} diff --git a/fullloop-15m/eval/core_eval.csv b/fullloop-15m/eval/core_eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..804520e40cb63ed15261269fdc9051947e2d7d70 --- /dev/null +++ b/fullloop-15m/eval/core_eval.csv @@ -0,0 +1,24 @@ +Task,Accuracy,Centered +hellaswag_zeroshot,0.284605,0.046140 +jeopardy,0.000472,0.000472 +bigbench_qa_wikidata,0.063826,0.063826 +arc_easy,0.393098,0.190797 +arc_challenge,0.230375,-0.026166 +copa,0.460000,-0.080000 +commonsense_qa,0.199017,-0.001229 +piqa,0.603917,0.207835 +openbook_qa,0.286000,0.048000 +lambada_openai,0.146711,0.146711 +hellaswag,0.281418,0.041891 +winograd,0.534799,0.069597 +winogrande,0.482242,-0.035517 +bigbench_dyck_languages,0.007000,0.007000 +agi_eval_lsat_ar,0.234783,0.043478 +bigbench_cs_algorithms,0.384848,0.384848 +bigbench_operators,0.090476,0.090476 +bigbench_repeat_copy_logic,0.000000,0.000000 +squad,0.016840,0.016840 +coqa,0.055618,0.055618 +boolq,0.528746,-0.240142 +bigbench_language_identification,0.258100,0.183828 +CORE,,0.055196 diff --git a/fullloop-15m/generation_config.json b/fullloop-15m/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9032a723674e6d0bdbef80891901e5200e077a48 --- /dev/null +++ b/fullloop-15m/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "transformers_version": "4.57.3", + "use_cache": false +} diff --git a/fullloop-15m/model.safetensors b/fullloop-15m/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..aa410c74bcc8de3b0f44b1178dd211e0003b7838 --- /dev/null +++ b/fullloop-15m/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9cb78b14c6f12a5796700efddaf34cf464ac26b353fcc7dc8488976331d4c62b +size 62913768 diff --git a/fullloop-15m/provenance.json b/fullloop-15m/provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..d513dc0e11028921df94928560e8d7659ba73fc4 --- /dev/null +++ b/fullloop-15m/provenance.json @@ -0,0 +1,63 @@ +{ + "architecture": "fullloop", + "best_validation_loss": 2.9359543323516846, + "legacy_checkpoint": "climbmix15M.pt", + "legacy_sha256": "927dfa31009123ab146bb69b7134f67f180c14d1bb3477d025b8a7d92d41cca5", + "model_args": { + "conv_size": 4, + "dim": 288, + "dropout": 0.0, + "head_k_dim": 72, + "head_v_dim": 72, + "hidden_dim": 768, + "loop_count": 4, + "max_seq_len": 1024, + "n_layers": 6, + "norm_eps": 1e-05, + "num_heads": 4, + "use_residual": true, + "vocab_size": 32000 + }, + "source_repo": "ruhai-lin/LT2.c", + "source_revision": "82cdc6174c9b0b1501a0b1e8aaeaa2a436171b37", + "training_config": { + "always_save_checkpoint": false, + "batch_size": 64, + "beta1": 0.9, + "beta2": 0.95, + "compile": false, + "conv_size": 4, + "decay_lr": true, + "device": "cuda", + "dim": 288, + "dropout": 0.0, + "dtype": "bfloat16", + "eval_interval": 2000, + "eval_iters": 100, + "eval_only": false, + "grad_clip": 1.0, + "gradient_accumulation_steps": 8, + "head_k_dim": 72, + "head_v_dim": 72, + "hidden_dim": 768, + "init_from": "scratch", + "learning_rate": 0.0005, + "log_interval": 1, + "loop_count": 4, + "max_iters": 100000, + "max_seq_len": 1024, + "n_layers": 6, + "norm_eps": 1e-05, + "num_heads": 4, + "out_dir": "outputs/climbmix15M_50B_100k", + "use_residual": true, + "vocab_size": 32000, + "vocab_source": "llama2", + "wandb_log": false, + "wandb_project": "lt2c", + "wandb_run_name": "run2026_06_17_02_08_27", + "warmup_iters": 1000, + "weight_decay": 0.1 + }, + "training_step": 100000 +} diff --git a/fullloop-15m/special_tokens_map.json b/fullloop-15m/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..a52c50a199269393cd1548c7e6a77a654bd2001b --- /dev/null +++ b/fullloop-15m/special_tokens_map.json @@ -0,0 +1,5 @@ +{ + "bos_token": "", + "eos_token": "", + "unk_token": "" +} diff --git a/fullloop-15m/tokenizer.model b/fullloop-15m/tokenizer.model new file mode 100644 index 0000000000000000000000000000000000000000..6c00c742ce03c627d6cd5b795984876fa49fa899 --- /dev/null +++ b/fullloop-15m/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 +size 499723 diff --git a/fullloop-15m/tokenizer_config.json b/fullloop-15m/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f8a69e9757d4423e3ee4becf59bc6cc584163226 --- /dev/null +++ b/fullloop-15m/tokenizer_config.json @@ -0,0 +1,12 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": true, + "model_max_length": 1024, + "pad_token": null, + "tokenizer_class": "LlamaTokenizer", + "unk_token": "" +} diff --git a/fullloop-42m/config.json b/fullloop-42m/config.json new file mode 100644 index 0000000000000000000000000000000000000000..67452955670fc8ae2860978712afb1480e0591d3 --- /dev/null +++ b/fullloop-42m/config.json @@ -0,0 +1,31 @@ +{ + "architectures": [ + "FullLoopForCausalLM" + ], + "attn_mode": "chunk", + "bos_token_id": 1, + "conv_size": 4, + "dtype": "float32", + "eos_token_id": 2, + "expand_v": 1.0, + "fuse_cross_entropy": true, + "fuse_linear_cross_entropy": false, + "fuse_norm": true, + "fuse_swiglu": true, + "head_dim": 64, + "hidden_act": "swish", + "hidden_ratio": 4, + "hidden_size": 512, + "initializer_range": 0.02, + "intermediate_size": 1376, + "loop_count": 4, + "max_position_embeddings": 1024, + "model_type": "fullloop", + "norm_eps": 1e-05, + "num_heads": 8, + "num_hidden_layers": 8, + "transformers_version": "4.57.3", + "use_cache": false, + "use_residual": true, + "vocab_size": 32000 +} diff --git a/fullloop-42m/eval/core_eval.csv b/fullloop-42m/eval/core_eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..45481be343e8b12377a739607cdeb72a6e49faa8 --- /dev/null +++ b/fullloop-42m/eval/core_eval.csv @@ -0,0 +1,24 @@ +Task,Accuracy,Centered +hellaswag_zeroshot,0.340669,0.120892 +jeopardy,0.007558,0.007558 +bigbench_qa_wikidata,0.179962,0.179962 +arc_easy,0.480640,0.307520 +arc_challenge,0.248294,-0.002275 +copa,0.450000,-0.100000 +commonsense_qa,0.212121,0.015152 +piqa,0.651251,0.302503 +openbook_qa,0.312000,0.082667 +lambada_openai,0.234232,0.234232 +hellaswag,0.336288,0.115050 +winograd,0.597070,0.194139 +winogrande,0.528019,0.056038 +bigbench_dyck_languages,0.067000,0.067000 +agi_eval_lsat_ar,0.234783,0.043478 +bigbench_cs_algorithms,0.361364,0.361364 +bigbench_operators,0.123810,0.123810 +bigbench_repeat_copy_logic,0.000000,0.000000 +squad,0.059224,0.059224 +coqa,0.099712,0.099712 +boolq,0.586239,-0.088846 +bigbench_language_identification,0.254900,0.180308 +CORE,,0.107249 diff --git a/fullloop-42m/generation_config.json b/fullloop-42m/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9032a723674e6d0bdbef80891901e5200e077a48 --- /dev/null +++ b/fullloop-42m/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "transformers_version": "4.57.3", + "use_cache": false +} diff --git a/fullloop-42m/model.safetensors b/fullloop-42m/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..db8bebf883c40dcc131db467b6bae06e8d2ef878 --- /dev/null +++ b/fullloop-42m/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:8e6c9e6dbc3548bae0c3d20127dcfbac40ed67684b366f6627f7dc73cfb3d8f3 +size 175631976 diff --git a/fullloop-42m/provenance.json b/fullloop-42m/provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..0f031f7702dfc8e14be8b12d8c47cf0b4bd90aa8 --- /dev/null +++ b/fullloop-42m/provenance.json @@ -0,0 +1,63 @@ +{ + "architecture": "fullloop", + "best_validation_loss": 2.5824697017669678, + "legacy_checkpoint": "climbmix42M.pt", + "legacy_sha256": "f95bea3bcfa16be0dfe04ed5260d53f5e3a777b4a467cb5937d63c6cdede5250", + "model_args": { + "conv_size": 4, + "dim": 512, + "dropout": 0.0, + "head_k_dim": 64, + "head_v_dim": 64, + "hidden_dim": 1376, + "loop_count": 4, + "max_seq_len": 1024, + "n_layers": 8, + "norm_eps": 1e-05, + "num_heads": 8, + "use_residual": true, + "vocab_size": 32000 + }, + "source_repo": "ruhai-lin/LT2.c", + "source_revision": "82cdc6174c9b0b1501a0b1e8aaeaa2a436171b37", + "training_config": { + "always_save_checkpoint": false, + "batch_size": 32, + "beta1": 0.9, + "beta2": 0.95, + "compile": false, + "conv_size": 4, + "decay_lr": true, + "device": "cuda", + "dim": 512, + "dropout": 0.0, + "dtype": "bfloat16", + "eval_interval": 2000, + "eval_iters": 100, + "eval_only": false, + "grad_clip": 1.0, + "gradient_accumulation_steps": 16, + "head_k_dim": 64, + "head_v_dim": 64, + "hidden_dim": 1376, + "init_from": "scratch", + "learning_rate": 0.0005, + "log_interval": 1, + "loop_count": 4, + "max_iters": 100000, + "max_seq_len": 1024, + "n_layers": 8, + "norm_eps": 1e-05, + "num_heads": 8, + "out_dir": "outputs/climbmix42M_50B_100k", + "use_residual": true, + "vocab_size": 32000, + "vocab_source": "llama2", + "wandb_log": false, + "wandb_project": "lt2c", + "wandb_run_name": "run2026_06_25_09_41_13", + "warmup_iters": 1000, + "weight_decay": 0.1 + }, + "training_step": 100000 +} diff --git a/fullloop-42m/special_tokens_map.json b/fullloop-42m/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..a52c50a199269393cd1548c7e6a77a654bd2001b --- /dev/null +++ b/fullloop-42m/special_tokens_map.json @@ -0,0 +1,5 @@ +{ + "bos_token": "", + "eos_token": "", + "unk_token": "" +} diff --git a/fullloop-42m/tokenizer.model b/fullloop-42m/tokenizer.model new file mode 100644 index 0000000000000000000000000000000000000000..6c00c742ce03c627d6cd5b795984876fa49fa899 --- /dev/null +++ b/fullloop-42m/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 +size 499723 diff --git a/fullloop-42m/tokenizer_config.json b/fullloop-42m/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f8a69e9757d4423e3ee4becf59bc6cc584163226 --- /dev/null +++ b/fullloop-42m/tokenizer_config.json @@ -0,0 +1,12 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": true, + "model_max_length": 1024, + "pad_token": null, + "tokenizer_class": "LlamaTokenizer", + "unk_token": "" +} diff --git a/gdn-110m/config.json b/gdn-110m/config.json new file mode 100644 index 0000000000000000000000000000000000000000..c0984616bf4087466fa998d4e666df3fc00f6041 --- /dev/null +++ b/gdn-110m/config.json @@ -0,0 +1,37 @@ +{ + "allow_neg_eigval": false, + "architectures": [ + "GatedDeltaNetForCausalLM" + ], + "attn": null, + "attn_mode": "chunk", + "attnres_block_size": null, + "bos_token_id": 1, + "conv_size": 4, + "dtype": "float32", + "eos_token_id": 2, + "expand_v": 1.0, + "fuse_cross_entropy": true, + "fuse_linear_cross_entropy": false, + "fuse_norm": true, + "fuse_swiglu": true, + "head_dim": 64, + "hidden_act": "swish", + "hidden_ratio": 4, + "hidden_size": 768, + "initializer_range": 0.02, + "intermediate_size": 2048, + "max_position_embeddings": 1024, + "model_type": "gated_deltanet", + "norm_eps": 1e-05, + "num_heads": 12, + "num_hidden_layers": 12, + "num_v_heads": null, + "tie_word_embeddings": true, + "transformers_version": "4.57.3", + "use_cache": false, + "use_gate": true, + "use_l2warp": false, + "use_short_conv": true, + "vocab_size": 32000 +} diff --git a/gdn-110m/eval/core_eval.csv b/gdn-110m/eval/core_eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..394e271899b12bf518816d4266eaacf9bd16bda7 --- /dev/null +++ b/gdn-110m/eval/core_eval.csv @@ -0,0 +1,24 @@ +Task,Accuracy,Centered +hellaswag_zeroshot,0.397132,0.196176 +jeopardy,0.013226,0.013226 +bigbench_qa_wikidata,0.314798,0.314798 +arc_easy,0.543350,0.391134 +arc_challenge,0.275597,0.034130 +copa,0.510000,0.020000 +commonsense_qa,0.205569,0.006962 +piqa,0.688248,0.376496 +openbook_qa,0.330000,0.106667 +lambada_openai,0.306035,0.306035 +hellaswag,0.391655,0.188873 +winograd,0.560440,0.120879 +winogrande,0.520916,0.041831 +bigbench_dyck_languages,0.150000,0.150000 +agi_eval_lsat_ar,0.269565,0.086957 +bigbench_cs_algorithms,0.450758,0.450758 +bigbench_operators,0.147619,0.147619 +bigbench_repeat_copy_logic,0.000000,0.000000 +squad,0.064522,0.064522 +coqa,0.115621,0.115621 +boolq,0.544954,-0.197489 +bigbench_language_identification,0.254700,0.180088 +CORE,,0.141604 diff --git a/gdn-110m/generation_config.json b/gdn-110m/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9032a723674e6d0bdbef80891901e5200e077a48 --- /dev/null +++ b/gdn-110m/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "transformers_version": "4.57.3", + "use_cache": false +} diff --git a/gdn-110m/model.safetensors b/gdn-110m/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..af343e5d2adf684a87cc4cb40edf7af58afbebd0 --- /dev/null +++ b/gdn-110m/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a9f2d623eb6bde918ebb45cacfeab5848b8d72ffb8812e9435540a33be315fdb +size 467785288 diff --git a/gdn-110m/provenance.json b/gdn-110m/provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..043adb99dcfc1e46df856e446834b80f773dd8db --- /dev/null +++ b/gdn-110m/provenance.json @@ -0,0 +1,59 @@ +{ + "architecture": "gdn", + "best_validation_loss": 2.4006850719451904, + "legacy_checkpoint": "climbmix110M.pt", + "legacy_sha256": "2394dee54eb4e87d5038bc45287d70606dad16a0578c0f7b5fa66e150dd2f024", + "model_args": { + "conv_size": 4, + "dim": 768, + "dropout": 0.0, + "head_k_dim": 64, + "head_v_dim": 64, + "hidden_dim": 2048, + "max_seq_len": 1024, + "n_layers": 12, + "norm_eps": 1e-05, + "num_heads": 12, + "vocab_size": 32000 + }, + "source_repo": "ruhai-lin/gdn.c", + "source_revision": "667dd7d6592152f65ea35e9e0c497f1ecec050ff", + "training_config": { + "always_save_checkpoint": false, + "batch_size": 64, + "beta1": 0.9, + "beta2": 0.95, + "compile": false, + "conv_size": 4, + "decay_lr": true, + "device": "cuda", + "dim": 768, + "dropout": 0.0, + "dtype": "bfloat16", + "eval_interval": 2000, + "eval_iters": 100, + "eval_only": false, + "grad_clip": 1.0, + "gradient_accumulation_steps": 8, + "head_k_dim": 64, + "head_v_dim": 64, + "hidden_dim": 2048, + "init_from": "resume", + "learning_rate": 0.0005, + "log_interval": 1, + "max_iters": 100000, + "max_seq_len": 1024, + "n_layers": 12, + "norm_eps": 1e-05, + "num_heads": 12, + "out_dir": "outputs/climbmix110M_50B_100k", + "vocab_size": 32000, + "vocab_source": "llama2", + "wandb_log": false, + "wandb_project": "gdnc", + "wandb_run_name": "run2026_06_08_21_19_23", + "warmup_iters": 1000, + "weight_decay": 0.1 + }, + "training_step": 100000 +} diff --git a/gdn-110m/special_tokens_map.json b/gdn-110m/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..a52c50a199269393cd1548c7e6a77a654bd2001b --- /dev/null +++ b/gdn-110m/special_tokens_map.json @@ -0,0 +1,5 @@ +{ + "bos_token": "", + "eos_token": "", + "unk_token": "" +} diff --git a/gdn-110m/tokenizer.model b/gdn-110m/tokenizer.model new file mode 100644 index 0000000000000000000000000000000000000000..6c00c742ce03c627d6cd5b795984876fa49fa899 --- /dev/null +++ b/gdn-110m/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 +size 499723 diff --git a/gdn-110m/tokenizer_config.json b/gdn-110m/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f8a69e9757d4423e3ee4becf59bc6cc584163226 --- /dev/null +++ b/gdn-110m/tokenizer_config.json @@ -0,0 +1,12 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": true, + "model_max_length": 1024, + "pad_token": null, + "tokenizer_class": "LlamaTokenizer", + "unk_token": "" +} diff --git a/gdn-15m/config.json b/gdn-15m/config.json new file mode 100644 index 0000000000000000000000000000000000000000..93fc7d4edf8f9ebf25b818f72cbc0aef6c97aa73 --- /dev/null +++ b/gdn-15m/config.json @@ -0,0 +1,37 @@ +{ + "allow_neg_eigval": false, + "architectures": [ + "GatedDeltaNetForCausalLM" + ], + "attn": null, + "attn_mode": "chunk", + "attnres_block_size": null, + "bos_token_id": 1, + "conv_size": 4, + "dtype": "float32", + "eos_token_id": 2, + "expand_v": 1.0, + "fuse_cross_entropy": true, + "fuse_linear_cross_entropy": false, + "fuse_norm": true, + "fuse_swiglu": true, + "head_dim": 72, + "hidden_act": "swish", + "hidden_ratio": 4, + "hidden_size": 288, + "initializer_range": 0.02, + "intermediate_size": 768, + "max_position_embeddings": 1024, + "model_type": "gated_deltanet", + "norm_eps": 1e-05, + "num_heads": 4, + "num_hidden_layers": 6, + "num_v_heads": null, + "tie_word_embeddings": true, + "transformers_version": "4.57.3", + "use_cache": false, + "use_gate": true, + "use_l2warp": false, + "use_short_conv": true, + "vocab_size": 32000 +} diff --git a/gdn-15m/eval/core_eval.csv b/gdn-15m/eval/core_eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..d258d1a115ce68279e53adbab8e4a9f1f1b92aca --- /dev/null +++ b/gdn-15m/eval/core_eval.csv @@ -0,0 +1,24 @@ +Task,Accuracy,Centered +hellaswag_zeroshot,0.281518,0.042024 +jeopardy,0.001417,0.001417 +bigbench_qa_wikidata,0.084199,0.084199 +arc_easy,0.385101,0.180135 +arc_challenge,0.226962,-0.030717 +copa,0.420000,-0.160000 +commonsense_qa,0.209664,0.012080 +piqa,0.600653,0.201306 +openbook_qa,0.278000,0.037333 +lambada_openai,0.130409,0.130409 +hellaswag,0.275443,0.033924 +winograd,0.501832,0.003663 +winogrande,0.490923,-0.018153 +bigbench_dyck_languages,0.011000,0.011000 +agi_eval_lsat_ar,0.256522,0.070652 +bigbench_cs_algorithms,0.394697,0.394697 +bigbench_operators,0.138095,0.138095 +bigbench_repeat_copy_logic,0.000000,0.000000 +squad,0.009934,0.009934 +coqa,0.044094,0.044094 +boolq,0.519572,-0.264285 +bigbench_language_identification,0.254300,0.179648 +CORE,,0.050066 diff --git a/gdn-15m/generation_config.json b/gdn-15m/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9032a723674e6d0bdbef80891901e5200e077a48 --- /dev/null +++ b/gdn-15m/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "transformers_version": "4.57.3", + "use_cache": false +} diff --git a/gdn-15m/model.safetensors b/gdn-15m/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..6b74876d9223a85a57c1beede463138010977c6d --- /dev/null +++ b/gdn-15m/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:fad0580d4bc5d64785e2cd0217d2c40b95836e9d1ccfdf2b946a4bdac2826059 +size 62908992 diff --git a/gdn-15m/provenance.json b/gdn-15m/provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..dbc943959c5a910922db3f90088c4ab1d8a25d02 --- /dev/null +++ b/gdn-15m/provenance.json @@ -0,0 +1,59 @@ +{ + "architecture": "gdn", + "best_validation_loss": 2.994854211807251, + "legacy_checkpoint": "climbmix15M.pt", + "legacy_sha256": "6d1035df306691380269ac595f72d87315120cf946127b1277d8a93310ffd628", + "model_args": { + "conv_size": 4, + "dim": 288, + "dropout": 0.0, + "head_k_dim": 72, + "head_v_dim": 72, + "hidden_dim": 768, + "max_seq_len": 1024, + "n_layers": 6, + "norm_eps": 1e-05, + "num_heads": 4, + "vocab_size": 32000 + }, + "source_repo": "ruhai-lin/gdn.c", + "source_revision": "667dd7d6592152f65ea35e9e0c497f1ecec050ff", + "training_config": { + "always_save_checkpoint": false, + "batch_size": 64, + "beta1": 0.9, + "beta2": 0.95, + "compile": false, + "conv_size": 4, + "decay_lr": true, + "device": "cuda", + "dim": 288, + "dropout": 0.0, + "dtype": "bfloat16", + "eval_interval": 2000, + "eval_iters": 100, + "eval_only": false, + "grad_clip": 1.0, + "gradient_accumulation_steps": 8, + "head_k_dim": 72, + "head_v_dim": 72, + "hidden_dim": 768, + "init_from": "scratch", + "learning_rate": 0.0005, + "log_interval": 1, + "max_iters": 100000, + "max_seq_len": 1024, + "n_layers": 6, + "norm_eps": 1e-05, + "num_heads": 4, + "out_dir": "outputs/climbmix15M_50B_100k", + "vocab_size": 32000, + "vocab_source": "llama2", + "wandb_log": false, + "wandb_project": "gdnc", + "wandb_run_name": "run2026_06_13_13_54_10", + "warmup_iters": 1000, + "weight_decay": 0.1 + }, + "training_step": 100000 +} diff --git a/gdn-15m/special_tokens_map.json b/gdn-15m/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..a52c50a199269393cd1548c7e6a77a654bd2001b --- /dev/null +++ b/gdn-15m/special_tokens_map.json @@ -0,0 +1,5 @@ +{ + "bos_token": "", + "eos_token": "", + "unk_token": "" +} diff --git a/gdn-15m/tokenizer.model b/gdn-15m/tokenizer.model new file mode 100644 index 0000000000000000000000000000000000000000..6c00c742ce03c627d6cd5b795984876fa49fa899 --- /dev/null +++ b/gdn-15m/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 +size 499723 diff --git a/gdn-15m/tokenizer_config.json b/gdn-15m/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f8a69e9757d4423e3ee4becf59bc6cc584163226 --- /dev/null +++ b/gdn-15m/tokenizer_config.json @@ -0,0 +1,12 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": true, + "model_max_length": 1024, + "pad_token": null, + "tokenizer_class": "LlamaTokenizer", + "unk_token": "" +} diff --git a/gdn-42m/config.json b/gdn-42m/config.json new file mode 100644 index 0000000000000000000000000000000000000000..f6cc98480733ce64c619c43bf7a64f6577f5bfa3 --- /dev/null +++ b/gdn-42m/config.json @@ -0,0 +1,37 @@ +{ + "allow_neg_eigval": false, + "architectures": [ + "GatedDeltaNetForCausalLM" + ], + "attn": null, + "attn_mode": "chunk", + "attnres_block_size": null, + "bos_token_id": 1, + "conv_size": 4, + "dtype": "float32", + "eos_token_id": 2, + "expand_v": 1.0, + "fuse_cross_entropy": true, + "fuse_linear_cross_entropy": false, + "fuse_norm": true, + "fuse_swiglu": true, + "head_dim": 64, + "hidden_act": "swish", + "hidden_ratio": 4, + "hidden_size": 512, + "initializer_range": 0.02, + "intermediate_size": 1376, + "max_position_embeddings": 1024, + "model_type": "gated_deltanet", + "norm_eps": 1e-05, + "num_heads": 8, + "num_hidden_layers": 8, + "num_v_heads": null, + "tie_word_embeddings": true, + "transformers_version": "4.57.3", + "use_cache": false, + "use_gate": true, + "use_l2warp": false, + "use_short_conv": true, + "vocab_size": 32000 +} diff --git a/gdn-42m/eval/core_eval.csv b/gdn-42m/eval/core_eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..8ca332d2141b662de24042d602077f9bbfa75ecf --- /dev/null +++ b/gdn-42m/eval/core_eval.csv @@ -0,0 +1,24 @@ +Task,Accuracy,Centered +hellaswag_zeroshot,0.322944,0.097258 +jeopardy,0.004251,0.004251 +bigbench_qa_wikidata,0.200827,0.200827 +arc_easy,0.453283,0.271044 +arc_challenge,0.244881,-0.006826 +copa,0.490000,-0.020000 +commonsense_qa,0.316953,0.146192 +piqa,0.627312,0.254625 +openbook_qa,0.294000,0.058667 +lambada_openai,0.214050,0.214050 +hellaswag,0.318861,0.091814 +winograd,0.564103,0.128205 +winogrande,0.498816,-0.002368 +bigbench_dyck_languages,0.032000,0.032000 +agi_eval_lsat_ar,0.221739,0.027174 +bigbench_cs_algorithms,0.419697,0.419697 +bigbench_operators,0.128571,0.128571 +bigbench_repeat_copy_logic,0.000000,0.000000 +squad,0.034626,0.034626 +coqa,0.086434,0.086434 +boolq,0.497554,-0.322228 +bigbench_language_identification,0.251400,0.176458 +CORE,,0.091840 diff --git a/gdn-42m/generation_config.json b/gdn-42m/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9032a723674e6d0bdbef80891901e5200e077a48 --- /dev/null +++ b/gdn-42m/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "transformers_version": "4.57.3", + "use_cache": false +} diff --git a/gdn-42m/model.safetensors b/gdn-42m/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a153c9763cc5744e8de26850fa9f6a043173ae61 --- /dev/null +++ b/gdn-42m/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4688b5ff2cde237ca54e439920eea8678abc8a05b8bd0b2e39be8cf47725f8d5 +size 175623568 diff --git a/gdn-42m/provenance.json b/gdn-42m/provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..3774035fdcf4ed3e5f609dc5ec30cd6a5d98e231 --- /dev/null +++ b/gdn-42m/provenance.json @@ -0,0 +1,59 @@ +{ + "architecture": "gdn", + "best_validation_loss": 2.652017116546631, + "legacy_checkpoint": "climbmix42M.pt", + "legacy_sha256": "2de4cbfe4b2a421813bc7cbf0bfc78c1e4f84627951c358b103f952dd12dfd70", + "model_args": { + "conv_size": 4, + "dim": 512, + "dropout": 0.0, + "head_k_dim": 64, + "head_v_dim": 64, + "hidden_dim": 1376, + "max_seq_len": 1024, + "n_layers": 8, + "norm_eps": 1e-05, + "num_heads": 8, + "vocab_size": 32000 + }, + "source_repo": "ruhai-lin/gdn.c", + "source_revision": "667dd7d6592152f65ea35e9e0c497f1ecec050ff", + "training_config": { + "always_save_checkpoint": false, + "batch_size": 64, + "beta1": 0.9, + "beta2": 0.95, + "compile": false, + "conv_size": 4, + "decay_lr": true, + "device": "cuda", + "dim": 512, + "dropout": 0.0, + "dtype": "bfloat16", + "eval_interval": 2000, + "eval_iters": 100, + "eval_only": false, + "grad_clip": 1.0, + "gradient_accumulation_steps": 8, + "head_k_dim": 64, + "head_v_dim": 64, + "hidden_dim": 1376, + "init_from": "scratch", + "learning_rate": 0.0005, + "log_interval": 1, + "max_iters": 100000, + "max_seq_len": 1024, + "n_layers": 8, + "norm_eps": 1e-05, + "num_heads": 8, + "out_dir": "outputs/climbmix42M_50B_100k", + "vocab_size": 32000, + "vocab_source": "llama2", + "wandb_log": false, + "wandb_project": "gdnc", + "wandb_run_name": "run2026_06_14_07_31_51", + "warmup_iters": 1000, + "weight_decay": 0.1 + }, + "training_step": 100000 +} diff --git a/gdn-42m/special_tokens_map.json b/gdn-42m/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..a52c50a199269393cd1548c7e6a77a654bd2001b --- /dev/null +++ b/gdn-42m/special_tokens_map.json @@ -0,0 +1,5 @@ +{ + "bos_token": "", + "eos_token": "", + "unk_token": "" +} diff --git a/gdn-42m/tokenizer.model b/gdn-42m/tokenizer.model new file mode 100644 index 0000000000000000000000000000000000000000..6c00c742ce03c627d6cd5b795984876fa49fa899 --- /dev/null +++ b/gdn-42m/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 +size 499723 diff --git a/gdn-42m/tokenizer_config.json b/gdn-42m/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f8a69e9757d4423e3ee4becf59bc6cc584163226 --- /dev/null +++ b/gdn-42m/tokenizer_config.json @@ -0,0 +1,12 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": true, + "model_max_length": 1024, + "pad_token": null, + "tokenizer_class": "LlamaTokenizer", + "unk_token": "" +} diff --git a/mixerloop-110m/config.json b/mixerloop-110m/config.json new file mode 100644 index 0000000000000000000000000000000000000000..032c0296aa11a8b90ce539a41a8b19700c99d7e1 --- /dev/null +++ b/mixerloop-110m/config.json @@ -0,0 +1,31 @@ +{ + "architectures": [ + "MixerLoopForCausalLM" + ], + "attn_mode": "chunk", + "bos_token_id": 1, + "conv_size": 4, + "dtype": "float32", + "eos_token_id": 2, + "expand_v": 1.0, + "fuse_cross_entropy": true, + "fuse_linear_cross_entropy": false, + "fuse_norm": true, + "fuse_swiglu": true, + "head_dim": 64, + "hidden_act": "swish", + "hidden_ratio": 4, + "hidden_size": 768, + "initializer_range": 0.02, + "intermediate_size": 2048, + "loop_count": 4, + "max_position_embeddings": 1024, + "model_type": "mixerloop", + "norm_eps": 1e-05, + "num_heads": 12, + "num_hidden_layers": 12, + "transformers_version": "4.57.3", + "use_cache": false, + "use_residual": true, + "vocab_size": 32000 +} diff --git a/mixerloop-110m/eval/core_eval.csv b/mixerloop-110m/eval/core_eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..02a4d3c316fd0d7f1bcdf50b46838728f8057e85 --- /dev/null +++ b/mixerloop-110m/eval/core_eval.csv @@ -0,0 +1,24 @@ +Task,Accuracy,Centered +hellaswag_zeroshot,0.408484,0.211312 +jeopardy,0.024091,0.024091 +bigbench_qa_wikidata,0.352985,0.352985 +arc_easy,0.563973,0.418631 +arc_challenge,0.302048,0.069397 +copa,0.540000,0.080000 +commonsense_qa,0.283374,0.104218 +piqa,0.688792,0.377584 +openbook_qa,0.338000,0.117333 +lambada_openai,0.306618,0.306618 +hellaswag,0.404800,0.206400 +winograd,0.582418,0.164835 +winogrande,0.513812,0.027624 +bigbench_dyck_languages,0.098000,0.098000 +agi_eval_lsat_ar,0.230435,0.038043 +bigbench_cs_algorithms,0.430303,0.430303 +bigbench_operators,0.123810,0.123810 +bigbench_repeat_copy_logic,0.000000,0.000000 +squad,0.080322,0.080322 +coqa,0.126018,0.126018 +boolq,0.574006,-0.121037 +bigbench_language_identification,0.260000,0.185919 +CORE,,0.155564 diff --git a/mixerloop-110m/generation_config.json b/mixerloop-110m/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9032a723674e6d0bdbef80891901e5200e077a48 --- /dev/null +++ b/mixerloop-110m/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "transformers_version": "4.57.3", + "use_cache": false +} diff --git a/mixerloop-110m/model.safetensors b/mixerloop-110m/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..facde77ec5dcfda8a69ab7f8e2d3481a6f121f41 --- /dev/null +++ b/mixerloop-110m/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:0476058a7f64f47d051bfd6a4120f97740c19e48881abc5b1110aeb6f0f7630c +size 467797832 diff --git a/mixerloop-110m/provenance.json b/mixerloop-110m/provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..fa95822a8d785be281190dce91c5ded0c21ef585 --- /dev/null +++ b/mixerloop-110m/provenance.json @@ -0,0 +1,63 @@ +{ + "architecture": "mixerloop", + "best_validation_loss": 2.3771042823791504, + "legacy_checkpoint": "climbmix110M.pt", + "legacy_sha256": "a2c2d113580386d54fdf12e1e7df67dad7c020d396fac823002ae3e53afed2dc", + "model_args": { + "conv_size": 4, + "dim": 768, + "dropout": 0.0, + "head_k_dim": 64, + "head_v_dim": 64, + "hidden_dim": 2048, + "loop_count": 4, + "max_seq_len": 1024, + "n_layers": 12, + "norm_eps": 1e-05, + "num_heads": 12, + "use_residual": true, + "vocab_size": 32000 + }, + "source_repo": "ruhai-lin/LT3.c", + "source_revision": "d004c7b74bdfb0fdc3bf84d679559c315318a2af", + "training_config": { + "always_save_checkpoint": false, + "batch_size": 32, + "beta1": 0.9, + "beta2": 0.95, + "compile": false, + "conv_size": 4, + "decay_lr": true, + "device": "cuda", + "dim": 768, + "dropout": 0.0, + "dtype": "bfloat16", + "eval_interval": 2000, + "eval_iters": 100, + "eval_only": false, + "grad_clip": 1.0, + "gradient_accumulation_steps": 16, + "head_k_dim": 64, + "head_v_dim": 64, + "hidden_dim": 2048, + "init_from": "resume", + "learning_rate": 0.0005, + "log_interval": 1, + "loop_count": 4, + "max_iters": 100000, + "max_seq_len": 1024, + "n_layers": 12, + "norm_eps": 1e-05, + "num_heads": 12, + "out_dir": "outputs/climbmix110M_50B_100k", + "use_residual": true, + "vocab_size": 32000, + "vocab_source": "llama2", + "wandb_log": false, + "wandb_project": "lt3c", + "wandb_run_name": "run2026_06_21_20_07_28", + "warmup_iters": 1000, + "weight_decay": 0.1 + }, + "training_step": 100000 +} diff --git a/mixerloop-110m/special_tokens_map.json b/mixerloop-110m/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..a52c50a199269393cd1548c7e6a77a654bd2001b --- /dev/null +++ b/mixerloop-110m/special_tokens_map.json @@ -0,0 +1,5 @@ +{ + "bos_token": "", + "eos_token": "", + "unk_token": "" +} diff --git a/mixerloop-110m/tokenizer.model b/mixerloop-110m/tokenizer.model new file mode 100644 index 0000000000000000000000000000000000000000..6c00c742ce03c627d6cd5b795984876fa49fa899 --- /dev/null +++ b/mixerloop-110m/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 +size 499723 diff --git a/mixerloop-110m/tokenizer_config.json b/mixerloop-110m/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f8a69e9757d4423e3ee4becf59bc6cc584163226 --- /dev/null +++ b/mixerloop-110m/tokenizer_config.json @@ -0,0 +1,12 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": true, + "model_max_length": 1024, + "pad_token": null, + "tokenizer_class": "LlamaTokenizer", + "unk_token": "" +} diff --git a/mixerloop-15m/config.json b/mixerloop-15m/config.json new file mode 100644 index 0000000000000000000000000000000000000000..6845a5292b458e3034ae7408a0e4d9ae52c7ba3f --- /dev/null +++ b/mixerloop-15m/config.json @@ -0,0 +1,31 @@ +{ + "architectures": [ + "MixerLoopForCausalLM" + ], + "attn_mode": "chunk", + "bos_token_id": 1, + "conv_size": 4, + "dtype": "float32", + "eos_token_id": 2, + "expand_v": 1.0, + "fuse_cross_entropy": true, + "fuse_linear_cross_entropy": false, + "fuse_norm": true, + "fuse_swiglu": true, + "head_dim": 72, + "hidden_act": "swish", + "hidden_ratio": 4, + "hidden_size": 288, + "initializer_range": 0.02, + "intermediate_size": 768, + "loop_count": 4, + "max_position_embeddings": 1024, + "model_type": "mixerloop", + "norm_eps": 1e-05, + "num_heads": 4, + "num_hidden_layers": 6, + "transformers_version": "4.57.3", + "use_cache": false, + "use_residual": true, + "vocab_size": 32000 +} diff --git a/mixerloop-15m/eval/core_eval.csv b/mixerloop-15m/eval/core_eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..51ebacd80a1cd13aff679cc29848b93e7fb6f792 --- /dev/null +++ b/mixerloop-15m/eval/core_eval.csv @@ -0,0 +1,24 @@ +Task,Accuracy,Centered +hellaswag_zeroshot,0.280522,0.040696 +jeopardy,0.001889,0.001889 +bigbench_qa_wikidata,0.015649,0.015649 +arc_easy,0.375000,0.166667 +arc_challenge,0.220137,-0.039818 +copa,0.420000,-0.160000 +commonsense_qa,0.217035,0.021294 +piqa,0.597388,0.194777 +openbook_qa,0.294000,0.058667 +lambada_openai,0.150204,0.150204 +hellaswag,0.278729,0.038306 +winograd,0.549451,0.098901 +winogrande,0.524073,0.048145 +bigbench_dyck_languages,0.089000,0.089000 +agi_eval_lsat_ar,0.243478,0.054348 +bigbench_cs_algorithms,0.370455,0.370455 +bigbench_operators,0.109524,0.109524 +bigbench_repeat_copy_logic,0.000000,0.000000 +squad,0.013245,0.013245 +coqa,0.055117,0.055117 +boolq,0.593884,-0.068727 +bigbench_language_identification,0.250600,0.175578 +CORE,,0.065178 diff --git a/mixerloop-15m/generation_config.json b/mixerloop-15m/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9032a723674e6d0bdbef80891901e5200e077a48 --- /dev/null +++ b/mixerloop-15m/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "transformers_version": "4.57.3", + "use_cache": false +} diff --git a/mixerloop-15m/model.safetensors b/mixerloop-15m/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..a92e61064ed492bafc65dfe17c15c8201831b57f --- /dev/null +++ b/mixerloop-15m/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:d1aa48cd77709a9eb1c2e52a4ad145a756923ba9cce4cd5827f58894e1bfba72 +size 62913768 diff --git a/mixerloop-15m/provenance.json b/mixerloop-15m/provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..80e2b4fcc146a3bdcc787e846620dfbe734dec06 --- /dev/null +++ b/mixerloop-15m/provenance.json @@ -0,0 +1,63 @@ +{ + "architecture": "mixerloop", + "best_validation_loss": 2.9463090896606445, + "legacy_checkpoint": "climbmix15M.pt", + "legacy_sha256": "646f10896fb6b8275f6b50ffb1b66215f97782526121c0c836869fb9086967b1", + "model_args": { + "conv_size": 4, + "dim": 288, + "dropout": 0.0, + "head_k_dim": 72, + "head_v_dim": 72, + "hidden_dim": 768, + "loop_count": 4, + "max_seq_len": 1024, + "n_layers": 6, + "norm_eps": 1e-05, + "num_heads": 4, + "use_residual": true, + "vocab_size": 32000 + }, + "source_repo": "ruhai-lin/LT3.c", + "source_revision": "d004c7b74bdfb0fdc3bf84d679559c315318a2af", + "training_config": { + "always_save_checkpoint": false, + "batch_size": 64, + "beta1": 0.9, + "beta2": 0.95, + "compile": false, + "conv_size": 4, + "decay_lr": true, + "device": "cuda", + "dim": 288, + "dropout": 0.0, + "dtype": "bfloat16", + "eval_interval": 2000, + "eval_iters": 100, + "eval_only": false, + "grad_clip": 1.0, + "gradient_accumulation_steps": 8, + "head_k_dim": 72, + "head_v_dim": 72, + "hidden_dim": 768, + "init_from": "scratch", + "learning_rate": 0.0005, + "log_interval": 1, + "loop_count": 4, + "max_iters": 100000, + "max_seq_len": 1024, + "n_layers": 6, + "norm_eps": 1e-05, + "num_heads": 4, + "out_dir": "outputs/climbmix15M_50B_100k", + "use_residual": true, + "vocab_size": 32000, + "vocab_source": "llama2", + "wandb_log": false, + "wandb_project": "lt3c", + "wandb_run_name": "run2026_06_20_11_56_03", + "warmup_iters": 1000, + "weight_decay": 0.1 + }, + "training_step": 100000 +} diff --git a/mixerloop-15m/special_tokens_map.json b/mixerloop-15m/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..a52c50a199269393cd1548c7e6a77a654bd2001b --- /dev/null +++ b/mixerloop-15m/special_tokens_map.json @@ -0,0 +1,5 @@ +{ + "bos_token": "", + "eos_token": "", + "unk_token": "" +} diff --git a/mixerloop-15m/tokenizer.model b/mixerloop-15m/tokenizer.model new file mode 100644 index 0000000000000000000000000000000000000000..6c00c742ce03c627d6cd5b795984876fa49fa899 --- /dev/null +++ b/mixerloop-15m/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 +size 499723 diff --git a/mixerloop-15m/tokenizer_config.json b/mixerloop-15m/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f8a69e9757d4423e3ee4becf59bc6cc584163226 --- /dev/null +++ b/mixerloop-15m/tokenizer_config.json @@ -0,0 +1,12 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": true, + "model_max_length": 1024, + "pad_token": null, + "tokenizer_class": "LlamaTokenizer", + "unk_token": "" +} diff --git a/mixerloop-42m/config.json b/mixerloop-42m/config.json new file mode 100644 index 0000000000000000000000000000000000000000..07a0bf96f9efd46f45cdde6a95d48cf4711892cc --- /dev/null +++ b/mixerloop-42m/config.json @@ -0,0 +1,31 @@ +{ + "architectures": [ + "MixerLoopForCausalLM" + ], + "attn_mode": "chunk", + "bos_token_id": 1, + "conv_size": 4, + "dtype": "float32", + "eos_token_id": 2, + "expand_v": 1.0, + "fuse_cross_entropy": true, + "fuse_linear_cross_entropy": false, + "fuse_norm": true, + "fuse_swiglu": true, + "head_dim": 64, + "hidden_act": "swish", + "hidden_ratio": 4, + "hidden_size": 512, + "initializer_range": 0.02, + "intermediate_size": 1376, + "loop_count": 4, + "max_position_embeddings": 1024, + "model_type": "mixerloop", + "norm_eps": 1e-05, + "num_heads": 8, + "num_hidden_layers": 8, + "transformers_version": "4.57.3", + "use_cache": false, + "use_residual": true, + "vocab_size": 32000 +} diff --git a/mixerloop-42m/eval/core_eval.csv b/mixerloop-42m/eval/core_eval.csv new file mode 100644 index 0000000000000000000000000000000000000000..492e6c0228cd4aedbb12f55527fb542faae6cde6 --- /dev/null +++ b/mixerloop-42m/eval/core_eval.csv @@ -0,0 +1,24 @@ +Task,Accuracy,Centered +hellaswag_zeroshot,0.327823,0.103764 +jeopardy,0.001889,0.001889 +bigbench_qa_wikidata,0.243443,0.243443 +arc_easy,0.453283,0.271044 +arc_challenge,0.242321,-0.010239 +copa,0.470000,-0.060000 +commonsense_qa,0.324324,0.155405 +piqa,0.638738,0.277476 +openbook_qa,0.322000,0.096000 +lambada_openai,0.231322,0.231322 +hellaswag,0.324736,0.099648 +winograd,0.593407,0.186813 +winogrande,0.529597,0.059195 +bigbench_dyck_languages,0.058000,0.058000 +agi_eval_lsat_ar,0.239130,0.048913 +bigbench_cs_algorithms,0.378030,0.378030 +bigbench_operators,0.104762,0.104762 +bigbench_repeat_copy_logic,0.000000,0.000000 +squad,0.047209,0.047209 +coqa,0.096831,0.096831 +boolq,0.579817,-0.105746 +bigbench_language_identification,0.258200,0.183938 +CORE,,0.112168 diff --git a/mixerloop-42m/generation_config.json b/mixerloop-42m/generation_config.json new file mode 100644 index 0000000000000000000000000000000000000000..9032a723674e6d0bdbef80891901e5200e077a48 --- /dev/null +++ b/mixerloop-42m/generation_config.json @@ -0,0 +1,7 @@ +{ + "_from_model_config": true, + "bos_token_id": 1, + "eos_token_id": 2, + "transformers_version": "4.57.3", + "use_cache": false +} diff --git a/mixerloop-42m/model.safetensors b/mixerloop-42m/model.safetensors new file mode 100644 index 0000000000000000000000000000000000000000..d486d88006adabfadd7e3530d052b7298517c3eb --- /dev/null +++ b/mixerloop-42m/model.safetensors @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4a86bcee5cf54df92524a199536adc90e8d3b3209b316f7e7582c108b70a9dcd +size 175631976 diff --git a/mixerloop-42m/provenance.json b/mixerloop-42m/provenance.json new file mode 100644 index 0000000000000000000000000000000000000000..92c40ffed1838ed60610d03effe468e7328a3e6b --- /dev/null +++ b/mixerloop-42m/provenance.json @@ -0,0 +1,63 @@ +{ + "architecture": "mixerloop", + "best_validation_loss": 2.610328435897827, + "legacy_checkpoint": "climbmix42M.pt", + "legacy_sha256": "30a30352dd6f63a9ef15f2a2fc94bebeffdccba6b0341797bd632dfc9bf572e2", + "model_args": { + "conv_size": 4, + "dim": 512, + "dropout": 0.0, + "head_k_dim": 64, + "head_v_dim": 64, + "hidden_dim": 1376, + "loop_count": 4, + "max_seq_len": 1024, + "n_layers": 8, + "norm_eps": 1e-05, + "num_heads": 8, + "use_residual": true, + "vocab_size": 32000 + }, + "source_repo": "ruhai-lin/LT3.c", + "source_revision": "d004c7b74bdfb0fdc3bf84d679559c315318a2af", + "training_config": { + "always_save_checkpoint": false, + "batch_size": 32, + "beta1": 0.9, + "beta2": 0.95, + "compile": false, + "conv_size": 4, + "decay_lr": true, + "device": "cuda", + "dim": 512, + "dropout": 0.0, + "dtype": "bfloat16", + "eval_interval": 2000, + "eval_iters": 100, + "eval_only": false, + "grad_clip": 1.0, + "gradient_accumulation_steps": 16, + "head_k_dim": 64, + "head_v_dim": 64, + "hidden_dim": 1376, + "init_from": "scratch", + "learning_rate": 0.0005, + "log_interval": 1, + "loop_count": 4, + "max_iters": 100000, + "max_seq_len": 1024, + "n_layers": 8, + "norm_eps": 1e-05, + "num_heads": 8, + "out_dir": "outputs/climbmix42M_50B_100k", + "use_residual": true, + "vocab_size": 32000, + "vocab_source": "llama2", + "wandb_log": false, + "wandb_project": "lt3c", + "wandb_run_name": "run2026_06_25_09_41_38", + "warmup_iters": 1000, + "weight_decay": 0.1 + }, + "training_step": 100000 +} diff --git a/mixerloop-42m/special_tokens_map.json b/mixerloop-42m/special_tokens_map.json new file mode 100644 index 0000000000000000000000000000000000000000..a52c50a199269393cd1548c7e6a77a654bd2001b --- /dev/null +++ b/mixerloop-42m/special_tokens_map.json @@ -0,0 +1,5 @@ +{ + "bos_token": "", + "eos_token": "", + "unk_token": "" +} diff --git a/mixerloop-42m/tokenizer.model b/mixerloop-42m/tokenizer.model new file mode 100644 index 0000000000000000000000000000000000000000..6c00c742ce03c627d6cd5b795984876fa49fa899 --- /dev/null +++ b/mixerloop-42m/tokenizer.model @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:9e556afd44213b6bd1be2b850ebbbd98f5481437a8021afaf58ee7fb1818d347 +size 499723 diff --git a/mixerloop-42m/tokenizer_config.json b/mixerloop-42m/tokenizer_config.json new file mode 100644 index 0000000000000000000000000000000000000000..f8a69e9757d4423e3ee4becf59bc6cc584163226 --- /dev/null +++ b/mixerloop-42m/tokenizer_config.json @@ -0,0 +1,12 @@ +{ + "add_bos_token": true, + "add_eos_token": false, + "bos_token": "", + "clean_up_tokenization_spaces": false, + "eos_token": "", + "legacy": true, + "model_max_length": 1024, + "pad_token": null, + "tokenizer_class": "LlamaTokenizer", + "unk_token": "" +}