Add 8-way classical geometric Mixture-of-Rotations optimizer ablation: eval_results.json
279645a verified | { | |
| "model_id": "MatterWaveAI/grt-30m-mor", | |
| "architecture": "GRTForCausalLM (Mixture-of-Rotations)", | |
| "base_model": "MatterWaveAI/grt-30m", | |
| "parameters": 31634086, | |
| "base_parameters": 29305498, | |
| "ffn_n_experts": 4, | |
| "mor_mode": "direct", | |
| "mor_router_mode": "rotational", | |
| "nn_linear_layers": 0, | |
| "before_upcycle_specialization": { | |
| "val_loss": 5.8467, | |
| "val_ppl": 346.102, | |
| "domain_val_loss": { | |
| "code_and_algorithms": 5.4092, | |
| "reasoning_and_language": 6.172, | |
| "rotational_geometry": 6.6421, | |
| "science_and_physics": 5.3337 | |
| } | |
| }, | |
| "after_upcycle_specialization": { | |
| "val_loss": 0.5115, | |
| "val_ppl": 1.6678, | |
| "domain_val_loss": { | |
| "code_and_algorithms": 0.4334, | |
| "reasoning_and_language": 0.6442, | |
| "rotational_geometry": 0.4991, | |
| "science_and_physics": 0.5293 | |
| }, | |
| "router_diagnostics": [ | |
| { | |
| "layer": 0, | |
| "mean_expert_probs": [ | |
| 0.2494, | |
| 0.2488, | |
| 0.253, | |
| 0.2487 | |
| ], | |
| "marginal_entropy": 1.3863, | |
| "mean_top1_confidence": 0.2545, | |
| "expert_angle_l2_divergence": 2.5326 | |
| }, | |
| { | |
| "layer": 1, | |
| "mean_expert_probs": [ | |
| 0.2492, | |
| 0.247, | |
| 0.2574, | |
| 0.2464 | |
| ], | |
| "marginal_entropy": 1.3861, | |
| "mean_top1_confidence": 0.2599, | |
| "expert_angle_l2_divergence": 2.5353 | |
| }, | |
| { | |
| "layer": 2, | |
| "mean_expert_probs": [ | |
| 0.2526, | |
| 0.2482, | |
| 0.2528, | |
| 0.2465 | |
| ], | |
| "marginal_entropy": 1.3862, | |
| "mean_top1_confidence": 0.2541, | |
| "expert_angle_l2_divergence": 2.5524 | |
| }, | |
| { | |
| "layer": 3, | |
| "mean_expert_probs": [ | |
| 0.251, | |
| 0.2486, | |
| 0.2548, | |
| 0.2457 | |
| ], | |
| "marginal_entropy": 1.3862, | |
| "mean_top1_confidence": 0.2559, | |
| "expert_angle_l2_divergence": 2.5425 | |
| }, | |
| { | |
| "layer": 4, | |
| "mean_expert_probs": [ | |
| 0.2511, | |
| 0.2486, | |
| 0.2562, | |
| 0.2441 | |
| ], | |
| "marginal_entropy": 1.3861, | |
| "mean_top1_confidence": 0.2586, | |
| "expert_angle_l2_divergence": 2.5487 | |
| }, | |
| { | |
| "layer": 5, | |
| "mean_expert_probs": [ | |
| 0.2508, | |
| 0.2485, | |
| 0.2572, | |
| 0.2436 | |
| ], | |
| "marginal_entropy": 1.3861, | |
| "mean_top1_confidence": 0.26, | |
| "expert_angle_l2_divergence": 2.5342 | |
| }, | |
| { | |
| "layer": 6, | |
| "mean_expert_probs": [ | |
| 0.2504, | |
| 0.2486, | |
| 0.2571, | |
| 0.2439 | |
| ], | |
| "marginal_entropy": 1.3861, | |
| "mean_top1_confidence": 0.2602, | |
| "expert_angle_l2_divergence": 2.5501 | |
| }, | |
| { | |
| "layer": 7, | |
| "mean_expert_probs": [ | |
| 0.251, | |
| 0.2488, | |
| 0.2567, | |
| 0.2435 | |
| ], | |
| "marginal_entropy": 1.3861, | |
| "mean_top1_confidence": 0.2592, | |
| "expert_angle_l2_divergence": 2.5291 | |
| }, | |
| { | |
| "layer": 8, | |
| "mean_expert_probs": [ | |
| 0.2506, | |
| 0.2461, | |
| 0.2605, | |
| 0.2427 | |
| ], | |
| "marginal_entropy": 1.3859, | |
| "mean_top1_confidence": 0.2658, | |
| "expert_angle_l2_divergence": 2.5455 | |
| }, | |
| { | |
| "layer": 9, | |
| "mean_expert_probs": [ | |
| 0.2502, | |
| 0.247, | |
| 0.2582, | |
| 0.2445 | |
| ], | |
| "marginal_entropy": 1.3861, | |
| "mean_top1_confidence": 0.2647, | |
| "expert_angle_l2_divergence": 2.5364 | |
| }, | |
| { | |
| "layer": 10, | |
| "mean_expert_probs": [ | |
| 0.2501, | |
| 0.2484, | |
| 0.2563, | |
| 0.2452 | |
| ], | |
| "marginal_entropy": 1.3862, | |
| "mean_top1_confidence": 0.2636, | |
| "expert_angle_l2_divergence": 2.5368 | |
| }, | |
| { | |
| "layer": 11, | |
| "mean_expert_probs": [ | |
| 0.2509, | |
| 0.2481, | |
| 0.256, | |
| 0.245 | |
| ], | |
| "marginal_entropy": 1.3862, | |
| "mean_top1_confidence": 0.2588, | |
| "expert_angle_l2_divergence": 2.524 | |
| } | |
| ] | |
| }, | |
| "optimizer_ablations_8way": { | |
| "pair_adam": { | |
| "variant": "PairAdam (Factored Toroidal Baseline)", | |
| "val_loss": 0.4829, | |
| "val_ppl": 1.6208, | |
| "domain_val_loss": { | |
| "code_and_algorithms": 0.3669, | |
| "reasoning_and_language": 0.6553, | |
| "rotational_geometry": 0.4772, | |
| "science_and_physics": 0.5155 | |
| }, | |
| "domain_val_ppl": { | |
| "code_and_algorithms": 1.4432, | |
| "reasoning_and_language": 1.9257, | |
| "rotational_geometry": 1.6115, | |
| "science_and_physics": 1.6745 | |
| }, | |
| "mean_routing_entropy_nats": 1.3862, | |
| "mean_routing_entropy_bits": 1.9999, | |
| "mean_top1_confidence": 0.2559, | |
| "expert_angle_l2_divergence": 2.5386, | |
| "top4_expert_angle_l2_divergence": 2.5347, | |
| "top4_full_expert_angle_l2_divergence": 3.5883, | |
| "elapsed_sec": 180.15 | |
| }, | |
| "hessian_pair_adam": { | |
| "variant": "HessianPairAdam (Sophia-Torus Cauchy-Riemann Curvature)", | |
| "val_loss": 0.5403, | |
| "val_ppl": 1.7165, | |
| "domain_val_loss": { | |
| "code_and_algorithms": 0.4082, | |
| "reasoning_and_language": 0.7201, | |
| "rotational_geometry": 0.533, | |
| "science_and_physics": 0.5916 | |
| }, | |
| "domain_val_ppl": { | |
| "code_and_algorithms": 1.5041, | |
| "reasoning_and_language": 2.0545, | |
| "rotational_geometry": 1.7041, | |
| "science_and_physics": 1.8068 | |
| }, | |
| "mean_routing_entropy_nats": 1.3862, | |
| "mean_routing_entropy_bits": 1.9999, | |
| "mean_top1_confidence": 0.2559, | |
| "expert_angle_l2_divergence": 2.5387, | |
| "top4_expert_angle_l2_divergence": 2.5349, | |
| "top4_full_expert_angle_l2_divergence": 3.5888, | |
| "elapsed_sec": 180.09 | |
| }, | |
| "polar_natural_adam": { | |
| "variant": "PolarNaturalAdam (Riemannian Polar Metric + Cauchy-Riemann Hessian)", | |
| "val_loss": 0.4848, | |
| "val_ppl": 1.6239, | |
| "domain_val_loss": { | |
| "code_and_algorithms": 0.3744, | |
| "reasoning_and_language": 0.6466, | |
| "rotational_geometry": 0.4747, | |
| "science_and_physics": 0.5226 | |
| }, | |
| "domain_val_ppl": { | |
| "code_and_algorithms": 1.4542, | |
| "reasoning_and_language": 1.9091, | |
| "rotational_geometry": 1.6076, | |
| "science_and_physics": 1.6865 | |
| }, | |
| "mean_routing_entropy_nats": 1.3862, | |
| "mean_routing_entropy_bits": 1.9999, | |
| "mean_top1_confidence": 0.2559, | |
| "expert_angle_l2_divergence": 2.5387, | |
| "top4_expert_angle_l2_divergence": 2.5349, | |
| "top4_full_expert_angle_l2_divergence": 3.5887, | |
| "elapsed_sec": 180.75 | |
| }, | |
| "cyclide_adam": { | |
| "variant": "CyclideAdam (Dupin Cyclide Conic Metric + Deviatoric Twist)", | |
| "val_loss": 0.5006, | |
| "val_ppl": 1.6498, | |
| "domain_val_loss": { | |
| "code_and_algorithms": 0.3758, | |
| "reasoning_and_language": 0.6718, | |
| "rotational_geometry": 0.4986, | |
| "science_and_physics": 0.5426 | |
| }, | |
| "domain_val_ppl": { | |
| "code_and_algorithms": 1.4561, | |
| "reasoning_and_language": 1.9578, | |
| "rotational_geometry": 1.6464, | |
| "science_and_physics": 1.7205 | |
| }, | |
| "mean_routing_entropy_nats": 1.3862, | |
| "mean_routing_entropy_bits": 1.9999, | |
| "mean_top1_confidence": 0.2559, | |
| "expert_angle_l2_divergence": 2.5534, | |
| "top4_expert_angle_l2_divergence": 2.5789, | |
| "top4_full_expert_angle_l2_divergence": 3.6659, | |
| "elapsed_sec": 180.09 | |
| }, | |
| "symplectic_adam": { | |
| "variant": "SymplecticAdam (Gromov-Liouville Non-Squeezing Floor + Casimir Cavity)", | |
| "val_loss": 0.5567, | |
| "val_ppl": 1.7448, | |
| "domain_val_loss": { | |
| "code_and_algorithms": 0.4185, | |
| "reasoning_and_language": 0.7417, | |
| "rotational_geometry": 0.553, | |
| "science_and_physics": 0.6082 | |
| }, | |
| "domain_val_ppl": { | |
| "code_and_algorithms": 1.5197, | |
| "reasoning_and_language": 2.0996, | |
| "rotational_geometry": 1.7385, | |
| "science_and_physics": 1.8372 | |
| }, | |
| "mean_routing_entropy_nats": 1.3862, | |
| "mean_routing_entropy_bits": 1.9999, | |
| "mean_top1_confidence": 0.2559, | |
| "expert_angle_l2_divergence": 2.5386, | |
| "top4_expert_angle_l2_divergence": 2.5345, | |
| "top4_full_expert_angle_l2_divergence": 3.588, | |
| "elapsed_sec": 180.2 | |
| }, | |
| "gyric_adam": { | |
| "variant": "GyricAdam (Poinsot-Lande Gyroscopic Projection + Thomas Precession)", | |
| "val_loss": 0.5745, | |
| "val_ppl": 1.7763, | |
| "domain_val_loss": { | |
| "code_and_algorithms": 0.4279, | |
| "reasoning_and_language": 0.7609, | |
| "rotational_geometry": 0.5672, | |
| "science_and_physics": 0.6412 | |
| }, | |
| "domain_val_ppl": { | |
| "code_and_algorithms": 1.5341, | |
| "reasoning_and_language": 2.1401, | |
| "rotational_geometry": 1.7633, | |
| "science_and_physics": 1.8987 | |
| }, | |
| "mean_routing_entropy_nats": 1.3862, | |
| "mean_routing_entropy_bits": 1.9999, | |
| "mean_top1_confidence": 0.2559, | |
| "expert_angle_l2_divergence": 2.5387, | |
| "top4_expert_angle_l2_divergence": 2.535, | |
| "top4_full_expert_angle_l2_divergence": 3.5888, | |
| "elapsed_sec": 177.29 | |
| }, | |
| "huygens_adam": { | |
| "variant": "HuygensAdam (Circular Order Parameter R_p Splay Torque)", | |
| "val_loss": 0.5225, | |
| "val_ppl": 1.6862, | |
| "domain_val_loss": { | |
| "code_and_algorithms": 0.3957, | |
| "reasoning_and_language": 0.7041, | |
| "rotational_geometry": 0.5107, | |
| "science_and_physics": 0.5696 | |
| }, | |
| "domain_val_ppl": { | |
| "code_and_algorithms": 1.4854, | |
| "reasoning_and_language": 2.022, | |
| "rotational_geometry": 1.6665, | |
| "science_and_physics": 1.7676 | |
| }, | |
| "mean_routing_entropy_nats": 1.3862, | |
| "mean_routing_entropy_bits": 1.9999, | |
| "mean_top1_confidence": 0.2559, | |
| "expert_angle_l2_divergence": 2.5676, | |
| "top4_expert_angle_l2_divergence": 2.6216, | |
| "top4_full_expert_angle_l2_divergence": 3.7258, | |
| "elapsed_sec": 176.07 | |
| }, | |
| "huygens_symplectic_adam": { | |
| "variant": "HuygensSymplecticAdam (Two-Stage Huygens -> Symplectic Curriculum)", | |
| "val_loss": 0.523, | |
| "val_ppl": 1.6871, | |
| "domain_val_loss": { | |
| "code_and_algorithms": 0.4091, | |
| "reasoning_and_language": 0.6859, | |
| "rotational_geometry": 0.5202, | |
| "science_and_physics": 0.557 | |
| }, | |
| "domain_val_ppl": { | |
| "code_and_algorithms": 1.5055, | |
| "reasoning_and_language": 1.9856, | |
| "rotational_geometry": 1.6823, | |
| "science_and_physics": 1.7454 | |
| }, | |
| "mean_routing_entropy_nats": 1.3862, | |
| "mean_routing_entropy_bits": 1.9999, | |
| "mean_top1_confidence": 0.2559, | |
| "expert_angle_l2_divergence": 2.5388, | |
| "top4_expert_angle_l2_divergence": 2.5351, | |
| "top4_full_expert_angle_l2_divergence": 3.5896, | |
| "elapsed_sec": 176.01 | |
| } | |
| } | |
| } | |