| { |
| "model_id": "MatterWaveAI/grt-10m", |
| "architecture": "GRTForCausalLM", |
| "stored_parameters": 8684515, |
| "non_embedding_parameters": 2234515, |
| "dense_equiv_parameters": 17518848, |
| "nn_linear_layers": 0, |
| "pretraining": { |
| "corpus": "nvidia/Nemotron-Pretraining-Dataset-sample", |
| "steps": 336000, |
| "optimizer": "PairAdam", |
| "train_loss": 2.144, |
| "val_loss": 4.033, |
| "val_ppl": 56.4, |
| "val_top1_acc": 0.4 |
| }, |
| "slm_bench": { |
| "benchmark_suite": "SLM-Bench (10-Task Core Suite)", |
| "evaluation_harness": "grt-slm-bench (training/slm_bench.py)", |
| "hardware": "AMD Ryzen AI Max+ 395 (Radeon 8060S / ROCm gfx1151)", |
| "macro_average_accuracy": { |
| "random_baseline": 0.345, |
| "zero_shot": 0.3647, |
| "pair_adam": 0.3603, |
| "symplectic_adam": 0.4463, |
| "huygens_symplectic_adam": 0.46 |
| }, |
| "tasks": { |
| "boolq": { |
| "split": "validation", |
| "n_eval": 3270, |
| "random_baseline": 0.5, |
| "zero_shot_acc": 0.6214, |
| "pair_adam_acc": 0.6113, |
| "symplectic_adam_acc": 0.6214, |
| "huygens_symplectic_adam_acc": 0.6217 |
| }, |
| "arc_easy": { |
| "split": "test", |
| "n_eval": 2376, |
| "random_baseline": 0.25, |
| "zero_shot_acc": 0.2694, |
| "pair_adam_acc": 0.266, |
| "symplectic_adam_acc": 0.3173, |
| "huygens_symplectic_adam_acc": 0.3228 |
| }, |
| "arc_challenge": { |
| "split": "test", |
| "n_eval": 1172, |
| "random_baseline": 0.25, |
| "zero_shot_acc": 0.2253, |
| "pair_adam_acc": 0.2491, |
| "symplectic_adam_acc": 0.2517, |
| "huygens_symplectic_adam_acc": 0.2713 |
| }, |
| "openbookqa": { |
| "split": "test", |
| "n_eval": 500, |
| "random_baseline": 0.25, |
| "zero_shot_acc": 0.3, |
| "pair_adam_acc": 0.226, |
| "symplectic_adam_acc": 0.334, |
| "huygens_symplectic_adam_acc": 0.352 |
| }, |
| "piqa": { |
| "split": "validation", |
| "n_eval": 1838, |
| "random_baseline": 0.5, |
| "zero_shot_acc": 0.5218, |
| "pair_adam_acc": 0.5082, |
| "symplectic_adam_acc": 0.5218, |
| "huygens_symplectic_adam_acc": 0.5218 |
| }, |
| "winogrande": { |
| "split": "validation", |
| "n_eval": 1267, |
| "random_baseline": 0.5, |
| "zero_shot_acc": 0.5051, |
| "pair_adam_acc": 0.5075, |
| "symplectic_adam_acc": 0.5185, |
| "huygens_symplectic_adam_acc": 0.5185 |
| }, |
| "hellaswag": { |
| "split": "validation", |
| "n_eval": 1000, |
| "random_baseline": 0.25, |
| "zero_shot_acc": 0.256, |
| "pair_adam_acc": 0.291, |
| "symplectic_adam_acc": 0.308, |
| "huygens_symplectic_adam_acc": 0.308 |
| }, |
| "qnli": { |
| "split": "validation", |
| "n_eval": 1000, |
| "random_baseline": 0.5, |
| "zero_shot_acc": 0.527, |
| "pair_adam_acc": 0.497, |
| "symplectic_adam_acc": 0.568, |
| "huygens_symplectic_adam_acc": 0.609 |
| }, |
| "commonsenseqa": { |
| "split": "validation", |
| "n_eval": 1221, |
| "random_baseline": 0.2, |
| "zero_shot_acc": 0.2015, |
| "pair_adam_acc": 0.2285, |
| "symplectic_adam_acc": 0.2285, |
| "huygens_symplectic_adam_acc": 0.2285 |
| }, |
| "gsm8k": { |
| "split": "test", |
| "n_eval": 500, |
| "random_baseline": 0.25, |
| "zero_shot_acc": 0.22, |
| "zero_shot_ppl": 145.85, |
| "pair_adam_acc": 0.218, |
| "pair_adam_ppl": 37.27, |
| "symplectic_adam_acc": 0.794, |
| "symplectic_adam_ppl": 43.47, |
| "huygens_symplectic_adam_acc": 0.846, |
| "huygens_symplectic_adam_ppl": 30.09 |
| } |
| } |
| } |
| } |
|
|
