| { | |
| "model_id": "MatterWaveAI/grt-10m-base", | |
| "architecture": "GRTForCausalLM", | |
| "stored_parameters": 8684515, | |
| "non_embedding_parameters": 2234515, | |
| "dense_equiv_parameters": 17518848, | |
| "nn_linear_layers": 0, | |
| "pretraining": { | |
| "corpus": "nvidia/Nemotron-Pretraining-Dataset-sample", | |
| "steps": 336000, | |
| "optimizer": "PairAdam", | |
| "train_loss": 2.144, | |
| "val_loss": 4.033, | |
| "val_ppl": 56.4, | |
| "val_top1_acc": 0.4 | |
| }, | |
| "slm_bench_zero_shot": { | |
| "benchmark_suite": "SLM-Bench (9 Multiple-Choice Tasks + GSM8K Perplexity)", | |
| "evaluation_harness": "grt-slm-bench (training/slm_bench.py)", | |
| "mc_macro_average_accuracy": 0.3808, | |
| "random_macro_average": 0.3556, | |
| "tasks": { | |
| "boolq": { | |
| "split": "validation", | |
| "n_eval": 3270, | |
| "random": 0.5, | |
| "zero_shot_acc": 0.6214, | |
| "pair_adam_acc": 0.6113 | |
| }, | |
| "qnli": { | |
| "split": "validation", | |
| "n_eval": 1000, | |
| "random": 0.5, | |
| "zero_shot_acc": 0.527, | |
| "pair_adam_acc": 0.497 | |
| }, | |
| "piqa": { | |
| "split": "validation", | |
| "n_eval": 1838, | |
| "random": 0.5, | |
| "zero_shot_acc": 0.5218, | |
| "pair_adam_acc": 0.5082 | |
| }, | |
| "winogrande": { | |
| "split": "validation", | |
| "n_eval": 1267, | |
| "random": 0.5, | |
| "zero_shot_acc": 0.5051, | |
| "pair_adam_acc": 0.5075 | |
| }, | |
| "openbookqa": { | |
| "split": "test", | |
| "n_eval": 500, | |
| "random": 0.25, | |
| "zero_shot_acc": 0.3, | |
| "pair_adam_acc": 0.226 | |
| }, | |
| "arc_easy": { | |
| "split": "test", | |
| "n_eval": 2376, | |
| "random": 0.25, | |
| "zero_shot_acc": 0.2694, | |
| "pair_adam_acc": 0.266 | |
| }, | |
| "hellaswag": { | |
| "split": "validation", | |
| "n_eval": 1000, | |
| "random": 0.25, | |
| "zero_shot_acc": 0.256, | |
| "pair_adam_acc": 0.291 | |
| }, | |
| "arc_challenge": { | |
| "split": "test", | |
| "n_eval": 1172, | |
| "random": 0.25, | |
| "zero_shot_acc": 0.2253, | |
| "pair_adam_acc": 0.2491 | |
| }, | |
| "commonsenseqa": { | |
| "split": "validation", | |
| "n_eval": 1221, | |
| "random": 0.2, | |
| "zero_shot_acc": 0.2015, | |
| "pair_adam_acc": 0.2285 | |
| }, | |
| "gsm8k": { | |
| "split": "test", | |
| "n_eval": 500, | |
| "zero_shot_ppl": 145.85, | |
| "pair_adam_ppl": 37.27 | |
| } | |
| } | |
| } | |
| } | |