Archive untouched NeurIPS submission GRT-20M-Base checkpoint and zero-shot SLM-Bench baseline
9f377d7 verified | { | |
| "model_id": "MatterWaveAI/grt-20m-base", | |
| "architecture": "GRTForCausalLM", | |
| "stored_parameters": 16730869, | |
| "non_embedding_parameters": 10280869, | |
| "dense_equiv_parameters": 22237440, | |
| "nn_linear_layers": 0, | |
| "pretraining": { | |
| "corpus": [ | |
| "OptimalScale/ClimbMix", | |
| "wikimedia/wikipedia", | |
| "tiiuae/falcon-refinedweb", | |
| "HuggingFaceFW/fineweb-edu", | |
| "HuggingFaceTB/cosmopedia", | |
| "nvidia/Nemotron-Pretraining-Dataset-sample", | |
| "HuggingFaceTB/finemath" | |
| ], | |
| "steps": 158000, | |
| "optimizer": "PairAdam", | |
| "train_loss": 2.114 | |
| }, | |
| "slm_bench_zero_shot": { | |
| "benchmark_suite": "SLM-Bench (9 Multiple-Choice Tasks + GSM8K Perplexity)", | |
| "evaluation_harness": "grt-slm-bench (training/slm_bench.py)", | |
| "mc_macro_average_accuracy": 0.3916, | |
| "pair_adam_mc_macro_average_accuracy": 0.3863, | |
| "random_macro_average": 0.3556, | |
| "tasks": { | |
| "boolq": { | |
| "split": "validation", | |
| "n_eval": 3270, | |
| "random": 0.5, | |
| "zero_shot_acc": 0.6217, | |
| "pair_adam_acc": 0.5343 | |
| }, | |
| "piqa": { | |
| "split": "validation", | |
| "n_eval": 1838, | |
| "random": 0.5, | |
| "zero_shot_acc": 0.5397, | |
| "pair_adam_acc": 0.5337 | |
| }, | |
| "winogrande": { | |
| "split": "validation", | |
| "n_eval": 1267, | |
| "random": 0.5, | |
| "zero_shot_acc": 0.5193, | |
| "pair_adam_acc": 0.5241 | |
| }, | |
| "qnli": { | |
| "split": "validation", | |
| "n_eval": 1000, | |
| "random": 0.5, | |
| "zero_shot_acc": 0.505, | |
| "pair_adam_acc": 0.557 | |
| }, | |
| "openbookqa": { | |
| "split": "test", | |
| "n_eval": 500, | |
| "random": 0.25, | |
| "zero_shot_acc": 0.342, | |
| "pair_adam_acc": 0.256 | |
| }, | |
| "hellaswag": { | |
| "split": "validation", | |
| "n_eval": 1000, | |
| "random": 0.25, | |
| "zero_shot_acc": 0.287, | |
| "pair_adam_acc": 0.318 | |
| }, | |
| "arc_easy": { | |
| "split": "test", | |
| "n_eval": 2376, | |
| "random": 0.25, | |
| "zero_shot_acc": 0.2643, | |
| "pair_adam_acc": 0.2685 | |
| }, | |
| "arc_challenge": { | |
| "split": "test", | |
| "n_eval": 1172, | |
| "random": 0.25, | |
| "zero_shot_acc": 0.2534, | |
| "pair_adam_acc": 0.2543 | |
| }, | |
| "commonsenseqa": { | |
| "split": "validation", | |
| "n_eval": 1221, | |
| "random": 0.2, | |
| "zero_shot_acc": 0.1916, | |
| "pair_adam_acc": 0.231 | |
| }, | |
| "gsm8k": { | |
| "split": "test", | |
| "n_eval": 500, | |
| "zero_shot_ppl": 60.12, | |
| "pair_adam_ppl": 12.52 | |
| } | |
| } | |
| } | |
| } | |