top-model: arch_a4a09054276917cb bpb=3.7711

METRICS.json

@@ -1,11 +1,11 @@
 {
-  "submission_id": "8fd2d2e091b427b88ddd66a20e0059f53a45d72fc8a7d2a5d7dc5a695b4921eb",
-  "arch_id": "arch_9be14d2887e68452",
-  "owner_hotkey": "94b4209b9ab016cfe0dae709e5331e1bb785979cb4f7238465373ab2e9e76053",
-  "bpb": 3.7815815533044894,
+  "submission_id": "3c73be2d7baf2191cf8ed9f6fb7a955208c5d980b07fdef9299b978617b88255",
+  "arch_id": "arch_a4a09054276917cb",
+  "owner_hotkey": "fa8cafcc9b3f5dbf4813825476ffb20c40cd73815b864c234b238e0fbbdd2277",
+  "bpb": 3.771072932322795,
   "n_params": 198147072,
   "tokens_seen": 2048,
-  "wall_clock_seconds": 18001.89401102066,
+  "wall_clock_seconds": 18002.58023238182,
   "battery": {
     "tier": "public",
     "groups": {
@@ -13,43 +13,43 @@
         "module": "g1_intrinsic",
         "status": "ok",
         "metrics": {
-          "g1.bpb.val": 3.781581564053405,
-          "g1.bpb.fresh": 4.56483704178391,
-          "g1.bpb.key_token": 5.9498663314135145,
-          "g1.posloss.p0_128": 4.027364303587954,
-          "g1.bpb.domain.code": 3.213470931508409,
-          "g1.bpb.domain.math": 4.1153188606926925,
-          "g1.bpb.domain.news": 5.5072218245968,
-          "g1.bpb.multidomain": 4.431966259454538,
-          "g1.bpb.domain.prose": 4.891853421020251,
-          "g1.posloss.p128_256": 3.736173988603456,
-          "g1.posloss.p256_384": 3.738122651941576,
-          "g1.posloss.p384_512": 3.732744499378431,
-          "g1.posloss.p512_end": 3.719130447742806,
-          "g1.bits_per_byte.val": 0.9492969237578822,
-          "g1.bits_per_byte.fresh": 1.22949692903927,
+          "g1.bpb.val": 3.771072981364723,
+          "g1.bpb.fresh": 4.600095753231995,
+          "g1.bpb.key_token": 5.913205079278005,
+          "g1.posloss.p0_128": 4.024760498346422,
+          "g1.bpb.domain.code": 3.2740145591586325,
+          "g1.bpb.domain.math": 4.186553902697417,
+          "g1.bpb.domain.news": 5.5054120296639395,
+          "g1.bpb.multidomain": 4.466367262519637,
+          "g1.bpb.domain.prose": 4.899488558558558,
+          "g1.posloss.p128_256": 3.722991881642181,
+          "g1.posloss.p256_384": 3.698078880221957,
+          "g1.posloss.p384_512": 3.735194090421734,
+          "g1.posloss.p512_end": 3.711143622510656,
+          "g1.bits_per_byte.val": 0.946712161598888,
+          "g1.bits_per_byte.fresh": 1.2394426337172075,
           "g1.assets.domains_present": 4.0,
-          "g1.bits_per_byte.key_token": 0.1345007811665392,
-          "g1.bits_per_byte.domain.code": 1.6003652201521366,
-          "g1.bits_per_byte.domain.math": 1.462186208654599,
-          "g1.bits_per_byte.domain.news": 1.5306066767927444,
-          "g1.bits_per_byte.multidomain": 1.4721819039849735,
-          "g1.bits_per_byte.domain.prose": 1.2955695103404137
+          "g1.bits_per_byte.key_token": 0.133794373345178,
+          "g1.bits_per_byte.domain.code": 1.6285906072820029,
+          "g1.bits_per_byte.domain.math": 1.489109197606233,
+          "g1.bits_per_byte.domain.news": 1.528726918392759,
+          "g1.bits_per_byte.multidomain": 1.4858481409958633,
+          "g1.bits_per_byte.domain.prose": 1.2969658407024582
         }
       },
       "g2": {
         "module": "g2_downstream",
         "status": "ok",
         "metrics": {
-          "g2.core.mean_nll": 4.182095974324691,
-          "g2.piqa.acc_norm": 0.64,
-          "g2.boolq.acc_norm": 0.64,
-          "g2.lambada.acc_norm": 0.965,
-          "g2.arc_easy.acc_norm": 0.285,
-          "g2.core.mean_acc_norm": 0.50125,
-          "g2.hellaswag.acc_norm": 0.375,
-          "g2.openbookqa.acc_norm": 0.305,
-          "g2.winogrande.acc_norm": 0.525,
+          "g2.core.mean_nll": 4.165985709400304,
+          "g2.piqa.acc_norm": 0.62,
+          "g2.boolq.acc_norm": 0.635,
+          "g2.lambada.acc_norm": 0.96,
+          "g2.arc_easy.acc_norm": 0.305,
+          "g2.core.mean_acc_norm": 0.503125,
+          "g2.hellaswag.acc_norm": 0.365,
+          "g2.openbookqa.acc_norm": 0.33,
+          "g2.winogrande.acc_norm": 0.535,
           "g2.assets.tasks_present": 8.0,
           "g2.arc_challenge.acc_norm": 0.275
         }
@@ -58,16 +58,16 @@
         "module": "g3_recall",
         "status": "ok",
         "metrics": {
-          "g3.copy.acc": 0.3888888888888889,
-          "g3.mqar.acc": 0.1875,
-          "g3.copy.g0.acc": 0.3333333333333333,
+          "g3.copy.acc": 0.3333333333333333,
+          "g3.mqar.acc": 0.20833333333333337,
+          "g3.copy.g0.acc": 0.4166666666666667,
           "g3.mqar.n4.acc": 0.3125,
-          "g3.mqar.n8.acc": 0.125,
-          "g3.copy.g16.acc": 0.4166666666666667,
-          "g3.copy.g64.acc": 0.4166666666666667,
-          "g3.mqar.n16.acc": 0.125,
-          "g3.recall.mean_acc": 0.6440972222222222,
-          "g3.recall.mean_nll": 5.6005342607683355,
+          "g3.mqar.n8.acc": 0.25,
+          "g3.copy.g16.acc": 0.25,
+          "g3.copy.g64.acc": 0.3333333333333333,
+          "g3.mqar.n16.acc": 0.0625,
+          "g3.recall.mean_acc": 0.6354166666666666,
+          "g3.recall.mean_nll": 5.251577494914333,
           "g3.passkey.base.acc": 1.0,
           "g3.induction.base.acc": 1.0
         }
@@ -78,110 +78,109 @@
         "metrics": {
           "g4.kk.acc": 0.5,
           "g4.s5.acc": 0.125,
-          "g4.mod.acc": 0.0,
-          "g4.bool.acc": 1.0,
-          "g4.dyck.acc": 0.7222222222222222,
+          "g4.mod.acc": 0.125,
+          "g4.bool.acc": 0.0,
+          "g4.dyck.acc": 0.8333333333333334,
           "g4.arith.acc": 0.1,
           "g4.kk.n3.acc": 0.5,
-          "g4.proof.acc": 0.375,
+          "g4.proof.acc": 0.6875,
           "g4.s5.k5.acc": 0.25,
-          "g4.mod.id.acc": 0.0,
+          "g4.mod.id.acc": 0.25,
           "g4.s5.k10.acc": 0.0,
-          "g4.dyck.k1.acc": 1.0,
-          "g4.dyck.k2.acc": 0.8333333333333334,
+          "g4.dyck.k1.acc": 0.9166666666666666,
+          "g4.dyck.k2.acc": 0.9166666666666666,
           "g4.mod.ood.acc": 0.0,
-          "g4.arith.t1.acc": 0.0,
+          "g4.arith.t1.acc": 0.25,
           "g4.arith.t2.acc": 0.0,
-          "g4.arith.t3.acc": 0.25,
-          "g4.proof.d0.acc": 0.25,
-          "g4.proof.d1.acc": 0.0,
+          "g4.arith.t3.acc": 0.0,
+          "g4.proof.d0.acc": 1.0,
+          "g4.proof.d1.acc": 1.0,
           "g4.proof.d2.acc": 0.5,
-          "g4.proof.d3.acc": 0.75,
-          "g4.bool.base.acc": 1.0,
-          "g4.arith.noop.acc": 0.0,
-          "g4.mod.id_ood_gap": 0.0,
-          "g4.dyck.k2long.acc": 0.3333333333333333,
-          "g4.arith.extrap.acc": 0.25,
-          "g4.reasoning.mean_acc": 0.4031746031746032,
-          "g4.reasoning.mean_nll": 6.624523202578227
+          "g4.proof.d3.acc": 0.25,
+          "g4.bool.base.acc": 0.0,
+          "g4.arith.noop.acc": 0.25,
+          "g4.mod.id_ood_gap": 0.25,
+          "g4.dyck.k2long.acc": 0.6666666666666666,
+          "g4.arith.extrap.acc": 0.0,
+          "g4.reasoning.mean_acc": 0.3386904761904762,
+          "g4.reasoning.mean_nll": 5.840880783895652
         }
       },
       "g5": {
         "module": "g5_longctx",
         "status": "ok",
         "metrics": {
-          "g5.lstar": 0.0,
-          "g5.retention": 0.5333333333333333,
-          "g5.ruler.acc": 0.14350000000000002,
-          "g5.babilong.acc": 0.13333333333333333,
+          "g5.lstar": 65536.0,
+          "g5.retention": 0.9448818897637796,
+          "g5.ruler.acc": 0.25425,
+          "g5.babilong.acc": 0.1,
           "g5.helmet_rag.n": 20.0,
           "g5.ruler.qa.acc": 0.125,
-          "g5.ruler.vt.acc": 0.18,
+          "g5.ruler.vt.acc": 0.24,
           "g5.natural_mcq.n": 40.0,
           "g5.ruler.qa.pack": 1.0,
-          "g5.helmet_rag.acc": 0.0,
-          "g5.mean.L4096.acc": 0.1875,
-          "g5.mean.L8192.acc": 0.1775,
+          "g5.helmet_rag.acc": 0.05,
+          "g5.mean.L4096.acc": 0.3175,
+          "g5.mean.L8192.acc": 0.075,
           "g5.natural.staged": 1.0,
-          "g5.ruler.mean_nll": 4.174695594443215,
-          "g5.mean.L16384.acc": 0.05500000000000001,
-          "g5.mean.L32768.acc": 0.15,
-          "g5.mean.L65536.acc": 0.1,
-          "g5.natural_mcq.acc": 0.25,
-          "g5.ruler.L4096.acc": 0.175,
-          "g5.ruler.L8192.acc": 0.155,
-          "g5.ruler.shot_rate": 0.8409090909090909,
+          "g5.ruler.mean_nll": 3.9016778855956824,
+          "g5.mean.L16384.acc": 0.14750000000000002,
+          "g5.mean.L32768.acc": 0.265,
+          "g5.mean.L65536.acc": 0.3,
+          "g5.natural_mcq.acc": 0.225,
+          "g5.ruler.L4096.acc": 0.43499999999999994,
+          "g5.ruler.L8192.acc": 0.15,
+          "g5.ruler.shot_rate": 0.8181818181818182,
           "g5.babilong.qa1.acc": 0.16666666666666666,
           "g5.babilong.qa2.acc": 0.16666666666666666,
-          "g5.babilong.qa3.acc": 0.16666666666666666,
-          "g5.babilong.qa4.acc": 0.16666666666666666,
-          "g5.babilong.qa5.acc": 0.0,
-          "g5.ruler.L16384.acc": 0.11,
-          "g5.ruler.L32768.acc": 0.15,
-          "g5.ruler.L65536.acc": 0.1,
-          "g5.ruler.mean_depth": 0.4641918252373103,
-          "g5.babilong.mean_nll": 7.515850392977397,
+          "g5.babilong.qa3.acc": 0.0,
+          "g5.babilong.qa4.acc": 0.0,
+          "g5.babilong.qa5.acc": 0.16666666666666666,
+          "g5.ruler.L16384.acc": 0.195,
+          "g5.ruler.L32768.acc": 0.265,
+          "g5.ruler.L65536.acc": 0.3,
+          "g5.ruler.mean_depth": 0.4283540504499409,
+          "g5.babilong.mean_nll": 8.110466469658745,
           "g5.helmet_rag.rank_n": 20.0,
-          "g5.ruler.niah_mk.acc": 0.1,
-          "g5.ruler.niah_mq.acc": 0.0625,
-          "g5.ruler.niah_mv.acc": 0.25,
+          "g5.ruler.niah_mk.acc": 0.5,
+          "g5.ruler.niah_mq.acc": 0.09375,
+          "g5.ruler.niah_mv.acc": 0.3125,
           "g5.babilong.L4096.acc": 0.2,
-          "g5.babilong.L8192.acc": 0.2,
+          "g5.babilong.L8192.acc": 0.0,
           "g5.ruler.qa.L4096.acc": 0.5,
           "g5.ruler.qa.L8192.acc": 0.0,
-          "g5.ruler.vt.L4096.acc": 0.0,
-          "g5.ruler.vt.L8192.acc": 0.4,
-          "g5.babilong.L16384.acc": 0.0,
-          "g5.babilong.mean_depth": 0.6042475217182637,
+          "g5.ruler.vt.L4096.acc": 0.3,
+          "g5.ruler.vt.L8192.acc": 0.5,
+          "g5.babilong.L16384.acc": 0.1,
+          "g5.babilong.mean_depth": 0.6391818811970181,
           "g5.babilong.mean_shots": 2.0,
-          "g5.helmet_rag.gold_nll": 4.411067061997302,
-          "g5.helmet_rag.rank_acc": 0.55,
-          "g5.ruler.L4096.len_err": 0.002294921875,
+          "g5.helmet_rag.gold_nll": 4.613458111201014,
+          "g5.helmet_rag.rank_acc": 0.5,
+          "g5.ruler.L4096.len_err": 0.0,
           "g5.ruler.L8192.len_err": 0.0,
           "g5.ruler.evidence_miss": 0.0,
           "g5.ruler.qa.L16384.acc": 0.0,
           "g5.ruler.qa.L32768.acc": 0.0,
-          "g5.ruler.vt.L16384.acc": 0.3,
-          "g5.ruler.vt.L32768.acc": 0.0,
-          "g5.ruler.vt.L65536.acc": 0.2,
+          "g5.ruler.vt.L16384.acc": 0.1,
+          "g5.ruler.vt.L32768.acc": 0.2,
+          "g5.ruler.vt.L65536.acc": 0.1,
           "g5.babilong.filler_pack": 1.0,
-          "g5.helmet_rag.L4096.acc": 0.0,
-          "g5.helmet_rag.L8192.acc": 0.0,
-          "g5.natural_mcq.mean_nll": 3.573467033900914,
+          "g5.helmet_rag.L8192.acc": 0.05,
+          "g5.natural_mcq.mean_nll": 3.686222618328003,
           "g5.ruler.L16384.len_err": 0.0,
           "g5.ruler.L32768.len_err": 0.0,
           "g5.ruler.L65536.len_err": 0.0,
-          "g5.natural_mcq.L8192.acc": 0.25,
+          "g5.natural_mcq.L8192.acc": 0.225,
           "g5.babilong.L4096.len_err": 0.0,
           "g5.babilong.L8192.len_err": 0.0,
           "g5.babilong.evidence_miss": 0.0,
-          "g5.babilong.qa1.L4096.acc": 0.0,
-          "g5.babilong.qa1.L8192.acc": 0.5,
+          "g5.babilong.qa1.L4096.acc": 0.5,
+          "g5.babilong.qa1.L8192.acc": 0.0,
           "g5.babilong.qa2.L4096.acc": 0.5,
           "g5.babilong.qa2.L8192.acc": 0.0,
           "g5.babilong.qa3.L4096.acc": 0.0,
-          "g5.babilong.qa3.L8192.acc": 0.5,
-          "g5.babilong.qa4.L4096.acc": 0.5,
+          "g5.babilong.qa3.L8192.acc": 0.0,
+          "g5.babilong.qa4.L4096.acc": 0.0,
           "g5.babilong.qa4.L8192.acc": 0.0,
           "g5.babilong.qa5.L4096.acc": 0.0,
           "g5.babilong.qa5.L8192.acc": 0.0,
@@ -190,20 +189,20 @@
           "g5.babilong.qa2.L16384.acc": 0.0,
           "g5.babilong.qa3.L16384.acc": 0.0,
           "g5.babilong.qa4.L16384.acc": 0.0,
-          "g5.babilong.qa5.L16384.acc": 0.0,
-          "g5.ruler.niah_mk.L4096.acc": 0.0,
+          "g5.babilong.qa5.L16384.acc": 0.5,
+          "g5.ruler.niah_mk.L4096.acc": 1.0,
           "g5.ruler.niah_mk.L8192.acc": 0.0,
-          "g5.ruler.niah_mq.L4096.acc": 0.125,
-          "g5.ruler.niah_mq.L8192.acc": 0.125,
-          "g5.ruler.niah_mv.L4096.acc": 0.25,
+          "g5.ruler.niah_mq.L4096.acc": 0.25,
+          "g5.ruler.niah_mq.L8192.acc": 0.0,
+          "g5.ruler.niah_mv.L4096.acc": 0.125,
           "g5.ruler.niah_mv.L8192.acc": 0.25,
-          "g5.ruler.niah_mk.L16384.acc": 0.0,
+          "g5.ruler.niah_mk.L16384.acc": 0.5,
           "g5.ruler.niah_mk.L32768.acc": 0.5,
-          "g5.ruler.niah_mk.L65536.acc": 0.0,
-          "g5.ruler.niah_mq.L16384.acc": 0.0,
+          "g5.ruler.niah_mk.L65536.acc": 0.5,
+          "g5.ruler.niah_mq.L16384.acc": 0.125,
           "g5.ruler.niah_mq.L32768.acc": 0.0,
           "g5.ruler.niah_mv.L16384.acc": 0.25,
-          "g5.ruler.niah_mv.L32768.acc": 0.25,
+          "g5.ruler.niah_mv.L32768.acc": 0.625,
           "g5.natural.pool_rows.helmet_rag": 96.0,
           "g5.natural.pool_rows.natural_mcq": 64.0
         }
@@ -221,19 +220,19 @@
         "status": "ok",
         "metrics": {
           "g7.vram.peak_gb": 2.311983616,
-          "g7.tpot.L1024.ms": 14.588175232347568,
-          "g7.tpot.L4096.ms": 14.590046521334443,
-          "g7.ttft.L1024.ms": 14.604408061131837,
-          "g7.ttft.L4096.ms": 14.57422086969018,
-          "g7.energy.skipped": 1.0,
-          "g7.tpot.L16384.ms": 14.59561431693146,
-          "g7.tpot.L32768.ms": 14.603247444028966,
-          "g7.ttft.L16384.ms": 14.574081171303988,
-          "g7.ttft.L32768.ms": 14.584711054340003,
-          "g7.quant.w8a8.dbpb": 0.0020096602709944733,
-          "g7.quant.w4a16.dbpb": 0.12105512679148234,
-          "g7.throughput.b1.toks": 68.6395717288809,
-          "g7.throughput.b32.toks": 85.30747321040076,
+          "g7.tpot.L1024.ms": 14.381302578840405,
+          "g7.tpot.L4096.ms": 14.391133765457198,
+          "g7.ttft.L1024.ms": 14.364506117999554,
+          "g7.ttft.L4096.ms": 14.32378590106964,
+          "g7.tpot.L16384.ms": 14.380262204213068,
+          "g7.tpot.L32768.ms": 14.386560505954549,
+          "g7.ttft.L16384.ms": 14.357506297528744,
+          "g7.ttft.L32768.ms": 14.330116100609304,
+          "g7.quant.w8a8.dbpb": 0.004147318629893704,
+          "g7.quant.w4a16.dbpb": 0.10966907988460406,
+          "g7.energy.j_per_token": 0.00359039564751617,
+          "g7.throughput.b1.toks": 69.58314974899956,
+          "g7.throughput.b32.toks": 94.74272171069568,
           "g7.state.analytic_unavailable": 1.0,
           "g7.state.bytes_per_token.measured": 8.0
         }
@@ -253,80 +252,80 @@
       }
     },
     "metrics": {
-      "org.g5.lstar": 0.0,
+      "org.g5.lstar": 65536.0,
       "org.g2.obqa_acc": {
-        "value": 0.305,
+        "value": 0.33,
         "clusters": {
-          "g2/openbookqa": 0.305
+          "g2/openbookqa": 0.33
         }
       },
       "org.g2.piqa_acc": {
-        "value": 0.64,
+        "value": 0.62,
         "clusters": {
-          "g2/piqa": 0.64
+          "g2/piqa": 0.62
         }
       },
       "org.g3.mqar_acc": {
-        "value": 0.1875,
+        "value": 0.20833333333333337,
         "clusters": {
           "mqar/n4": 0.3125,
-          "mqar/n8": 0.125,
-          "mqar/n16": 0.125
+          "mqar/n8": 0.25,
+          "mqar/n16": 0.0625
         }
       },
       "org.g4.dyck_acc": {
-        "value": 0.7222222222222222,
+        "value": 0.8333333333333334,
         "clusters": {
-          "dyck/k1/len10": 1.0,
-          "dyck/k2/len10": 0.8333333333333334,
-          "dyck/k2/len28": 0.3333333333333333
+          "dyck/k1/len10": 0.9166666666666666,
+          "dyck/k2/len10": 0.9166666666666666,
+          "dyck/k2/len28": 0.6666666666666666
         }
       },
       "org.g2.boolq_acc": {
-        "value": 0.64,
+        "value": 0.635,
         "clusters": {
-          "g2/boolq": 0.64
+          "g2/boolq": 0.635
         }
       },
       "org.g5.ruler_acc": {
-        "value": 0.14350000000000002,
+        "value": 0.25425,
         "clusters": {
           "qa@4096": 0.5,
           "qa@8192": 0.0,
-          "vt@4096": 0.0,
-          "vt@8192": 0.4,
+          "vt@4096": 0.3,
+          "vt@8192": 0.5,
           "qa@16384": 0.0,
           "qa@32768": 0.0,
-          "vt@16384": 0.30000000000000004,
-          "vt@32768": 0.0,
-          "vt@65536": 0.2,
-          "niah_mk@4096": 0.0,
+          "vt@16384": 0.1,
+          "vt@32768": 0.2,
+          "vt@65536": 0.1,
+          "niah_mk@4096": 1.0,
           "niah_mk@8192": 0.0,
-          "niah_mq@4096": 0.125,
-          "niah_mq@8192": 0.125,
-          "niah_mv@4096": 0.25,
+          "niah_mq@4096": 0.25,
+          "niah_mq@8192": 0.0,
+          "niah_mv@4096": 0.125,
           "niah_mv@8192": 0.25,
-          "niah_mk@16384": 0.0,
+          "niah_mk@16384": 0.5,
           "niah_mk@32768": 0.5,
-          "niah_mk@65536": 0.0,
-          "niah_mq@16384": 0.0,
+          "niah_mk@65536": 0.5,
+          "niah_mq@16384": 0.125,
           "niah_mq@32768": 0.0,
           "niah_mv@16384": 0.25,
-          "niah_mv@32768": 0.25
+          "niah_mv@32768": 0.625
         }
       },
       "org.g2.lambada_acc": {
-        "value": 0.965,
+        "value": 0.96,
         "clusters": {
-          "g2/lambada": 0.965
+          "g2/lambada": 0.96
         }
       },
       "org.g3.copying_acc": {
-        "value": 0.3888888888888889,
+        "value": 0.3333333333333333,
         "clusters": {
-          "copy/len16/gap0": 0.3333333333333333,
-          "copy/len16/gap16": 0.4166666666666667,
-          "copy/len16/gap64": 0.4166666666666667
+          "copy/len16/gap0": 0.4166666666666667,
+          "copy/len16/gap16": 0.25,
+          "copy/len16/gap64": 0.3333333333333333
         }
       },
       "org.g3.passkey_acc": {
@@ -336,35 +335,35 @@
         }
       },
       "org.g4.modular_acc": {
-        "value": 0.0,
+        "value": 0.125,
         "clusters": {
-          "mod/p97/id": 0.0,
+          "mod/p97/id": 0.25,
           "mod/p97/ood": 0.0
         }
       },
-      "org.g7.tpot_ms_32k": 14.603247444028966,
+      "org.g7.tpot_ms_32k": 14.386560505954549,
       "org.g7.ttft_ms_32k": {
-        "value": 14.584711054340003,
+        "value": 14.330116100609304,
         "clusters": {
-          "L32768": 14.584711054340003
+          "L32768": 14.330116100609304
         }
       },
       "org.g2.arc_easy_acc": {
-        "value": 0.285,
+        "value": 0.305,
         "clusters": {
-          "g2/arc_easy": 0.285
+          "g2/arc_easy": 0.305
         }
       },
       "org.g5.babilong_acc": {
-        "value": 0.13333333333333333,
+        "value": 0.1,
         "clusters": {
-          "qa1@4096": 0.0,
-          "qa1@8192": 0.5,
+          "qa1@4096": 0.5,
+          "qa1@8192": 0.0,
           "qa2@4096": 0.5,
           "qa2@8192": 0.0,
           "qa3@4096": 0.0,
-          "qa3@8192": 0.5,
-          "qa4@4096": 0.5,
+          "qa3@8192": 0.0,
+          "qa4@4096": 0.0,
           "qa4@8192": 0.0,
           "qa5@4096": 0.0,
           "qa5@8192": 0.0,
@@ -372,13 +371,13 @@
           "qa2@16384": 0.0,
           "qa3@16384": 0.0,
           "qa4@16384": 0.0,
-          "qa5@16384": 0.0
+          "qa5@16384": 0.5
         }
       },
       "org.g2.hellaswag_acc": {
-        "value": 0.375,
+        "value": 0.365,
         "clusters": {
-          "g2/hellaswag": 0.375
+          "g2/hellaswag": 0.365
         }
       },
       "org.g3.induction_acc": {
@@ -388,36 +387,35 @@
         }
       },
       "org.g2.winogrande_acc": {
-        "value": 0.525,
+        "value": 0.535,
         "clusters": {
-          "g2/winogrande": 0.525
+          "g2/winogrande": 0.535
         }
       },
       "org.g4.arithmetic_acc": {
         "value": 0.1,
         "clusters": {
-          "arith/t1": 0.0,
+          "arith/t1": 0.25,
           "arith/t2": 0.0,
-          "arith/t3": 0.25,
-          "arith/t2/noop": 0.0,
-          "arith/t2/extrap": 0.25
+          "arith/t3": 0.0,
+          "arith/t2/noop": 0.25,
+          "arith/t2/extrap": 0.0
         }
       },
       "org.g5.helmet_rag_acc": {
-        "value": 0.0,
+        "value": 0.05,
         "clusters": {
           "helmet_rag@nq": 0.0,
           "helmet_rag@popqa": 0.0,
           "helmet_rag@hotpotqa": 0.0,
-          "helmet_rag@triviaqa": 0.0
+          "helmet_rag@triviaqa": 0.16666666666666666
         }
       },
       "org.g4.proofwriter_acc": {
-        "value": 0.375,
+        "value": 0.6875,
         "clusters": {
-          "proof/d0/neg": 0.0,
           "proof/d0/pos": 1.0,
-          "proof/d1/neg": 0.0,
+          "proof/d1/pos": 1.0,
           "proof/d2/neg": 0.0,
           "proof/d2/pos": 1.0,
           "proof/d3/neg": 0.0,
@@ -425,16 +423,16 @@
         }
       },
       "org.g5.natural_mcq_acc": {
-        "value": 0.25,
+        "value": 0.225,
         "clusters": {
-          "natural_mcq@Multi-Document QA": 0.2727272727272727,
-          "natural_mcq@Single-Document QA": 0.26666666666666666,
-          "natural_mcq@Long In-context Learning": 0.5,
+          "natural_mcq@Multi-Document QA": 0.15384615384615383,
+          "natural_mcq@Single-Document QA": 0.3333333333333333,
           "natural_mcq@Code Repository Understanding": 0.3333333333333333,
           "natural_mcq@Long Structured Data Understanding": 0.0,
           "natural_mcq@Long-dialogue History Understanding": 0.0
         }
       },
+      "org.g7.joules_per_token": 0.00359039564751617,
       "org.g8.loss_spike_score": 1.0,
       "org.g8.mup_lr_stability": 1.0,
       "org.g2.arc_challenge_acc": {
@@ -444,27 +442,27 @@
         }
       },
       "org.g4.boolean_logic_acc": {
-        "value": 1.0,
+        "value": 0.0,
         "clusters": {
-          "bool/d2": 1.0
+          "bool/d2": 0.0
         }
       },
       "org.g7.throughput_toks_s": {
-        "value": 85.30747321040076,
+        "value": 94.74272171069568,
         "clusters": {
-          "b32": 85.30747321040076
+          "b32": 94.74272171069568
         }
       },
       "org.g1.bits_per_byte_code": {
-        "value": 1.6003652201521366,
+        "value": 1.6285906072820029,
         "clusters": {
-          "domain/code": 1.6003652201521366
+          "domain/code": 1.6285906072820029
         }
       },
       "org.g1.bits_per_byte_math": {
-        "value": 1.462186208654599,
+        "value": 1.489109197606233,
         "clusters": {
-          "domain/math": 1.462186208654599
+          "domain/math": 1.489109197606233
         }
       },
       "org.g4.knights_knaves_acc": {
@@ -474,16 +472,16 @@
         }
       },
       "org.g1.bits_per_byte_prose": {
-        "value": 1.2955695103404137,
+        "value": 1.2969658407024582,
         "clusters": {
-          "domain/prose": 1.2955695103404137
+          "domain/prose": 1.2969658407024582
         }
       },
-      "org.g1.bits_per_byte_key_token": 0.1345007811665392,
+      "org.g1.bits_per_byte_key_token": 0.133794373345178,
       "org.g1.bits_per_byte_fresh_crawl": {
-        "value": 1.22949692903927,
+        "value": 1.2394426337172075,
         "clusters": {
-          "fresh": 1.22949692903927
+          "fresh": 1.2394426337172075
         }
       },
       "org.g7.state_bytes_per_token_32k": 8.0
@@ -493,11 +491,11 @@
         "group": "g4",
         "metric": "org.g4.arithmetic_acc",
         "mirror": {
-          "value": 0.0,
+          "value": 0.25,
           "clusters": {
             "arith#0": 0.0,
             "arith#1": 0.0,
-            "arith#2": 0.0,
+            "arith#2": 1.0,
             "arith#3": 0.0
           }
         },
@@ -515,11 +513,11 @@
         "group": "g4",
         "metric": "org.g4.boolean_logic_acc",
         "mirror": {
-          "value": 0.25,
+          "value": 0.5,
           "clusters": {
-            "boolean_expr#0": 1.0,
-            "boolean_expr#1": 0.0,
-            "boolean_expr#2": 0.0,
+            "boolean_expr#0": 0.0,
+            "boolean_expr#1": 1.0,
+            "boolean_expr#2": 1.0,
             "boolean_expr#3": 0.0
           }
         },
@@ -541,8 +539,8 @@
           "clusters": {
             "dyck#0": 0.3333333333333333,
             "dyck#1": 1.0,
-            "dyck#2": 0.6666666666666666,
-            "dyck#3": 0.6666666666666666
+            "dyck#2": 0.3333333333333333,
+            "dyck#3": 1.0
           }
         },
         "public": {
@@ -559,20 +557,20 @@
         "group": "g4",
         "metric": "org.g4.modular_acc",
         "mirror": {
-          "value": 0.25,
+          "value": 0.0,
           "clusters": {
             "modular#0": 0.0,
             "modular#1": 0.0,
             "modular#2": 0.0,
-            "modular#3": 1.0
+            "modular#3": 0.0
           }
         },
         "public": {
-          "value": 0.5,
+          "value": 0.75,
           "clusters": {
-            "modular#0": 1.0,
-            "modular#1": 0.0,
-            "modular#2": 0.0,
+            "modular#0": 0.0,
+            "modular#1": 1.0,
+            "modular#2": 1.0,
             "modular#3": 1.0
           }
         }
@@ -581,10 +579,10 @@
         "group": "g4",
         "metric": "org.g4.knights_knaves_acc",
         "mirror": {
-          "value": 0.75,
+          "value": 0.25,
           "clusters": {
-            "knights_knaves#0": 1.0,
-            "knights_knaves#1": 1.0,
+            "knights_knaves#0": 0.0,
+            "knights_knaves#1": 0.0,
             "knights_knaves#2": 1.0,
             "knights_knaves#3": 0.0
           }
@@ -603,12 +601,12 @@
         "group": "g4",
         "metric": "org.g4.proofwriter_acc",
         "mirror": {
-          "value": 0.25,
+          "value": 0.5,
           "clusters": {
-            "proofwriter#0": 1.0,
-            "proofwriter#1": 0.0,
+            "proofwriter#0": 0.0,
+            "proofwriter#1": 1.0,
             "proofwriter#2": 0.0,
-            "proofwriter#3": 0.0
+            "proofwriter#3": 1.0
           }
         },
         "public": {
@@ -669,19 +667,19 @@
         "group": "g2",
         "metric": "org.g2.piqa_acc",
         "mirror": {
-          "value": 0.5,
+          "value": 0.25,
           "clusters": {
             "piqa#0": 1.0,
             "piqa#1": 0.0,
             "piqa#2": 0.0,
-            "piqa#3": 1.0
+            "piqa#3": 0.0
           }
         },
         "public": {
-          "value": 0.5,
+          "value": 1.0,
           "clusters": {
-            "piqa#0": 0.0,
-            "piqa#1": 0.0,
+            "piqa#0": 1.0,
+            "piqa#1": 1.0,
             "piqa#2": 1.0,
             "piqa#3": 1.0
           }
@@ -691,11 +689,11 @@
         "group": "g2",
         "metric": "org.g2.arc_easy_acc",
         "mirror": {
-          "value": 0.75,
+          "value": 0.5,
           "clusters": {
             "arc_easy#0": 1.0,
             "arc_easy#1": 1.0,
-            "arc_easy#2": 1.0,
+            "arc_easy#2": 0.0,
             "arc_easy#3": 0.0
           }
         },
@@ -713,9 +711,9 @@
         "group": "g2",
         "metric": "org.g2.arc_challenge_acc",
         "mirror": {
-          "value": 0.75,
+          "value": 0.5,
           "clusters": {
-            "arc_challenge#0": 1.0,
+            "arc_challenge#0": 0.0,
             "arc_challenge#1": 1.0,
             "arc_challenge#2": 0.0,
             "arc_challenge#3": 1.0
@@ -744,12 +742,12 @@
           }
         },
         "public": {
-          "value": 0.5,
+          "value": 0.75,
           "clusters": {
-            "winogrande#0": 0.0,
-            "winogrande#1": 1.0,
+            "winogrande#0": 1.0,
+            "winogrande#1": 0.0,
             "winogrande#2": 1.0,
-            "winogrande#3": 0.0
+            "winogrande#3": 1.0
           }
         }
       },
@@ -788,10 +786,10 @@
           }
         },
         "public": {
-          "value": 0.25,
+          "value": 0.5,
           "clusters": {
             "openbookqa#0": 0.0,
-            "openbookqa#1": 0.0,
+            "openbookqa#1": 1.0,
             "openbookqa#2": 1.0,
             "openbookqa#3": 0.0
           }
@@ -801,20 +799,19 @@
         "group": "g5",
         "metric": "org.g5.natural_mcq_acc",
         "mirror": {
-          "value": 0.0,
+          "value": 0.25,
           "clusters": {
+            "Multi-Document QA": 0.5,
             "Single-Document QA": 0.0,
-            "Long Structured Data Understanding": 0.0,
-            "Long-dialogue History Understanding": 0.0
+            "Code Repository Understanding": 0.0
           }
         },
         "public": {
           "value": 0.0,
           "clusters": {
             "Multi-Document QA": 0.0,
-            "Single-Document QA": 0.0,
             "Long In-context Learning": 0.0,
-            "Long-dialogue History Understanding": 0.0
+            "Code Repository Understanding": 0.0
           }
         }
       },
@@ -822,16 +819,16 @@
         "group": "g5",
         "metric": "org.g5.helmet_rag_acc",
         "mirror": {
-          "value": 0.0,
+          "value": 0.25,
           "clusters": {
-            "popqa": 0.0,
-            "hotpotqa": 0.0
+            "nq": 0.0,
+            "hotpotqa": 0.0,
+            "triviaqa": 1.0
           }
         },
         "public": {
           "value": 0.0,
           "clusters": {
-            "nq": 0.0,
             "popqa": 0.0,
             "triviaqa": 0.0
           }

README.md

@@ -27,15 +27,15 @@ Prism-protocol **public** eval pack (1×RTX 5090). Accuracy: **↑ higher better
 
 | Metric | This model | GPT-2 | GPT-2 Large | vs GPT-2 | vs GPT-2 Large |
 |---|---:|---:|---:|:---|:---|
-| Val BPB (G1) | 3.7816 | 4.7595 | 4.1639 | ↓ -0.9779 ✓ better | ↓ -0.3823 ✓ better |
-| HellaSwag | 0.375 | 0.355 | 0.395 | ↑ +0.020 ✓ better | ↓ -0.020 worse |
-| ARC-Easy | 0.285 | 0.245 | 0.280 | ↑ +0.040 ✓ better | ↑ +0.005 ✓ better |
+| Val BPB (G1) | 3.7711 | 4.7595 | 4.1639 | ↓ -0.9884 ✓ better | ↓ -0.3928 ✓ better |
+| HellaSwag | 0.365 | 0.355 | 0.395 | ↑ +0.010 ✓ better | ↓ -0.030 worse |
+| ARC-Easy | 0.305 | 0.245 | 0.280 | ↑ +0.060 ✓ better | ↑ +0.025 ✓ better |
 | ARC-Challenge | 0.275 | 0.240 | 0.280 | ↑ +0.035 ✓ better | ↓ -0.005 worse |
-| PIQA | 0.640 | 0.585 | 0.690 | ↑ +0.055 ✓ better | ↓ -0.050 worse |
-| WinoGrande | 0.525 | 0.515 | 0.545 | ↑ +0.010 ✓ better | ↓ -0.020 worse |
-| BoolQ | 0.640 | 0.575 | 0.640 | ↑ +0.065 ✓ better | = +0.000 tie |
-| LAMBADA | 0.965 | 0.970 | 0.985 | ↓ -0.005 worse | ↓ -0.020 worse |
-| OpenBookQA | 0.305 | 0.320 | 0.335 | ↓ -0.015 worse | ↓ -0.030 worse |
+| PIQA | 0.620 | 0.585 | 0.690 | ↑ +0.035 ✓ better | ↓ -0.070 worse |
+| WinoGrande | 0.535 | 0.515 | 0.545 | ↑ +0.020 ✓ better | ↓ -0.010 worse |
+| BoolQ | 0.635 | 0.575 | 0.640 | ↑ +0.060 ✓ better | ↓ -0.005 worse |
+| LAMBADA | 0.960 | 0.970 | 0.985 | ↓ -0.010 worse | ↓ -0.025 worse |
+| OpenBookQA | 0.330 | 0.320 | 0.335 | ↑ +0.010 ✓ better | ↓ -0.005 worse |
 
 ### Compute notes
 
@@ -43,10 +43,10 @@ Prism-protocol **public** eval pack (1×RTX 5090). Accuracy: **↑ higher better
 |---|---|---|---|
 | Parameters | 198.1M | 124M | 774M |
 | Size vs Large | 3.91× vs GPT-2 Large (774M) | 6.22× | 1× |
-| Train tokens | 1992818688 | _(eval-only)_ | _(eval-only)_ |
-| Wall clock | 18002s | _(eval-only)_ | _(eval-only)_ |
-| Sustained train throughput | 131.6 TFLOPS (est.) | n/a | n/a |
-| GPU (harness) | `GPU 0: NVIDIA GeForce RTX 5090 (UUID: GPU-52642165-c475-c281-9482-c20561376ccd)` | 1×RTX 5090 (eval) | 1×RTX 5090 (eval) |
+| Train tokens | 2127298560 | _(eval-only)_ | _(eval-only)_ |
+| Wall clock | 18003s | _(eval-only)_ | _(eval-only)_ |
+| Sustained train throughput | 140.5 TFLOPS (est.) | n/a | n/a |
+| GPU (harness) | `GPU 0: NVIDIA GeForce RTX 5090 (UUID: GPU-615bc7f0-025b-a1f7-ecb5-03099c82025a)` | 1×RTX 5090 (eval) | 1×RTX 5090 (eval) |
 
 Throughput ≈ `6 × N × D / wall` TFLOPS (dense transformer train FLOPs rule of thumb).
 
@@ -54,10 +54,10 @@ Throughput ≈ `6 × N × D / wall` TFLOPS (dense transformer train FLOPs rule o
 
 | field | value |
 |---|---|
-| arch_id | `arch_9be14d2887e68452` |
-| bpb | `3.781582` |
-| submission | `8fd2d2e091b427b88ddd66a20e0059f53a45d72fc8a7d2a5d7dc5a695b4921eb` |
-| owner_hotkey | `94b4209b9ab0…` |
+| arch_id | `arch_a4a09054276917cb` |
+| bpb | `3.771073` |
+| submission | `3c73be2d7baf2191cf8ed9f6fb7a955208c5d980b07fdef9299b978617b88255` |
+| owner_hotkey | `fa8cafcc9b3f…` |
 | hub repo | `BaseIntelligence/top-prism-architecture` |
 
 ## Load (trust_remote_code)

architecture.py

@@ -1,4 +1,4 @@
-# prism-automodel-copy patch_sha256=906c794a58d8bf272738e2aa61b875cebfb77c8e5ac11010f335134adf67ad3b
+# prism-automodel-copy patch_sha256=700ecf920252ddd61ae2e0fbaaf9a2db62fb634235510948d9f2f6756df22f8d
 # --- nemo_automodel/components/models/prism/__init__.py (arch) ---
 
 # --- nemo_automodel/components/models/prism/prism_entry.py (arch) ---
@@ -87,18 +87,10 @@ ADAM_LR = 0.003
 ADAM_BETAS = (0.9, 0.95)
 ADAM_WD = 0.0
 WARMUP_STEPS = 250
-DECAY_FRAC = 0.15              # доля бюджета под затухание, косинус до минимума
-#: Было 0.35: затухание начиналось на 65% бюджета, множитель падал до 0.03
-#: к середине прогона — последнюю треть модель почти не училась. Замерено
-#: парно на одном железе, одинаковые семена, 112.5M / 300M токенов:
-#:   семя 1: 485625 против 473125   (+12500)
-#:   семя 2: 488750 против 475625   (+13125)
-#: Прибавка идёт из piqa (+5.0) и winogrande (+5.2). Величина ОТНОСИТЕЛЬНАЯ
-#: (доля бюджета), поэтому форма расписания переносится на другой масштаб,
-#: в отличие от абсолютных величин.
+DECAY_FRAC = 0.35              # final share of the wall budget, cosine to min
 MIN_LR_FRAC = 0.0
 DOCSTART_FRAC = 0.5
-DATA_SEED = 1234
+DATA_SEED = 1303
 REPORT_EVERY = 25
 STEP_CEILING = 19000           # hard stop below the recipe's 20000
 VAL_LO, VAL_HI = 2048, 2304    # frozen val cut, removed from training texts
@@ -201,15 +193,25 @@ class RoPEDecoderLM(nn.Module):
         self.head.weight = self.tok_emb.weight      # tied
         self._cos = None
         self._sin = None
+        # muP (пункт 1): базовая ширина = D_MODEL, поэтому при боевой
+        # конфигурации оба множителя равны 1.0 и рецепт не меняется ни на бит.
+        self._mup_mult = math.sqrt(float(D_MODEL) / float(d_model))
+        self._mup_out = float(D_MODEL) / float(d_model)
         self.apply(self._init)
         for blk in self.layers:                     # clean residual stream at t=0
             nn.init.zeros_(blk.att.wo.weight)
             nn.init.zeros_(blk.mlp.w2.weight)
 
-    @staticmethod
-    def _init(m):
-        if isinstance(m, (nn.Linear, nn.Embedding)):
+    def _init(self, m):
+        # Скрытые веса масштабируем по ширине (muP), входной эмбеддинг и
+        # привязанный к нему readout — нет: в muP это "входные" веса.
+        # Порядок обхода тот же, что был у apply(staticmethod), число обращений
+        # к генератору то же — значит при _mup_mult == 1.0 результат тот же.
+        if isinstance(m, nn.Embedding):
             nn.init.normal_(m.weight, std=0.02)
+        elif isinstance(m, nn.Linear):
+            std = 0.02 if m is self.head else 0.02 * self._mup_mult
+            nn.init.normal_(m.weight, std=std)
 
     def _rope(self, device, dtype):
         if self._cos is None or self._cos.device != device or self._cos.dtype != dtype:
@@ -226,7 +228,10 @@ class RoPEDecoderLM(nn.Module):
         cos, sin = self._rope(ids.device, x.dtype)
         for blk in self.layers:
             x = blk(x, cos, sin)
-        return self.head(self.norm_f(x))
+        logits = self.head(self.norm_f(x))
+        # muP readout: при базовой ширине множитель ровно 1.0 — умножение
+        # пропускаем, чтобы боевой путь остался нетронутым и в графе тоже.
+        return logits if self._mup_out == 1.0 else logits * self._mup_out
 
 
 # ------------------------------ tokenizer --------------------------------

checkpoint.pt

@@ -1,3 +1,3 @@
 version https://git-lfs.github.com/spec/v1
-oid sha256:8dafa6667f69044e4b6e68ab4ba01550b7da170d0c4d30e228b761d6836ebc3c
+oid sha256:c872b87c736313b280655d21105b14b2998f65dfb2a5b26a50571954638fd8b7
 size 826185691

config.json

@@ -8,9 +8,9 @@
     "AutoModel": "modeling_prism.PrismCustomModel",
     "AutoModelForCausalLM": "modeling_prism.PrismCustomModel"
   },
-  "prism_arch_id": "arch_9be14d2887e68452",
-  "prism_submission_id": "8fd2d2e091b427b88ddd66a20e0059f53a45d72fc8a7d2a5d7dc5a695b4921eb",
-  "prism_bpb": 3.7815815533044894,
+  "prism_arch_id": "arch_a4a09054276917cb",
+  "prism_submission_id": "3c73be2d7baf2191cf8ed9f6fb7a955208c5d980b07fdef9299b978617b88255",
+  "prism_bpb": 3.771072932322795,
   "torch_dtype": "float32",
   "checkpoint_file": "checkpoint.pt"
 }

sources/.prism/automodel.patch

@@ -2,11 +2,11 @@ diff --git a/nemo_automodel/components/models/prism/__init__.py b/nemo_automodel
 new file mode 100644
 index 0000000..e69de29
 diff --git a/nemo_automodel/components/models/prism/prism_entry.py b/nemo_automodel/components/models/prism/prism_entry.py
-new file mode 100755
-index 0000000..7e52d24
+new file mode 100644
+index 0000000..4ebefa0
 --- /dev/null
 +++ b/nemo_automodel/components/models/prism/prism_entry.py
-@@ -0,0 +1,693 @@
+@@ -0,0 +1,698 @@
 +"""Prism recipe-2.0 train entry — self-contained, Prism-shaped.
 +
 +The harness (`prismlib.automodel.invoke_entry`) loads this file by path and,
@@ -92,18 +92,10 @@ index 0000000..7e52d24
 +ADAM_BETAS = (0.9, 0.95)
 +ADAM_WD = 0.0
 +WARMUP_STEPS = 250
-+DECAY_FRAC = 0.15              # доля бюджета под затухание, косинус до минимума
-+#: Было 0.35: затухание начиналось на 65% бюджета, множитель падал до 0.03
-+#: к середине прогона — последнюю треть модель почти не училась. Замерено
-+#: парно на одном железе, одинаковые семена, 112.5M / 300M токенов:
-+#:   семя 1: 485625 против 473125   (+12500)
-+#:   семя 2: 488750 против 475625   (+13125)
-+#: Прибавка идёт из piqa (+5.0) и winogrande (+5.2). Величина ОТНОСИТЕЛЬНАЯ
-+#: (доля бюджета), поэтому форма расписания переносится на другой масштаб,
-+#: в отличие от абсолютных величин.
++DECAY_FRAC = 0.35              # final share of the wall budget, cosine to min
 +MIN_LR_FRAC = 0.0
 +DOCSTART_FRAC = 0.5
-+DATA_SEED = 1234
++DATA_SEED = 1303
 +REPORT_EVERY = 25
 +STEP_CEILING = 19000           # hard stop below the recipe's 20000
 +VAL_LO, VAL_HI = 2048, 2304    # frozen val cut, removed from training texts
@@ -206,15 +198,25 @@ index 0000000..7e52d24
 +        self.head.weight = self.tok_emb.weight      # tied
 +        self._cos = None
 +        self._sin = None
++        # muP (пункт 1): базовая ширина = D_MODEL, поэтому при боевой
++        # конфигурации оба множителя равны 1.0 и рецепт не меняется ни на бит.
++        self._mup_mult = math.sqrt(float(D_MODEL) / float(d_model))
++        self._mup_out = float(D_MODEL) / float(d_model)
 +        self.apply(self._init)
 +        for blk in self.layers:                     # clean residual stream at t=0
 +            nn.init.zeros_(blk.att.wo.weight)
 +            nn.init.zeros_(blk.mlp.w2.weight)
 +
-+    @staticmethod
-+    def _init(m):
-+        if isinstance(m, (nn.Linear, nn.Embedding)):
++    def _init(self, m):
++        # Скрытые веса масштабируем по ширине (muP), входной эмбеддинг и
++        # привязанный к нему readout — нет: в muP это "входные" веса.
++        # Порядок обхода тот же, что был у apply(staticmethod), число обращений
++        # к генератору то же — значит при _mup_mult == 1.0 результат тот же.
++        if isinstance(m, nn.Embedding):
 +            nn.init.normal_(m.weight, std=0.02)
++        elif isinstance(m, nn.Linear):
++            std = 0.02 if m is self.head else 0.02 * self._mup_mult
++            nn.init.normal_(m.weight, std=std)
 +
 +    def _rope(self, device, dtype):
 +        if self._cos is None or self._cos.device != device or self._cos.dtype != dtype:
@@ -231,7 +233,10 @@ index 0000000..7e52d24
 +        cos, sin = self._rope(ids.device, x.dtype)
 +        for blk in self.layers:
 +            x = blk(x, cos, sin)
-+        return self.head(self.norm_f(x))
++        logits = self.head(self.norm_f(x))
++        # muP readout: при базовой ширине множитель ровно 1.0 — умножение
++        # пропускаем, чтобы боевой путь остался нетронутым и в графе тоже.
++        return logits if self._mup_out == 1.0 else logits * self._mup_out
 +
 +
 +# ------------------------------ tokenizer --------------------------------

sources/.prism/diffstat.json

@@ -1 +1 @@
-{"files":[{"path":"nemo_automodel/components/models/prism/__init__.py","added":0,"deleted":0,"class":"arch"},{"path":"nemo_automodel/components/models/prism/prism_entry.py","added":693,"deleted":0,"class":"arch"},{"path":"nemo_automodel/components/models/prism/tok8k/tokenizer.json","added":1,"deleted":0,"class":"arch"},{"path":"nemo_automodel/components/models/prism/tok8k/tokenizer_config.json","added":9,"deleted":0,"class":"arch"},{"path":"tokenizer/tokenizer.json","added":1,"deleted":0,"class":"other"},{"path":"tokenizer/tokenizer_config.json","added":9,"deleted":0,"class":"other"}],"total_added":713,"total_deleted":0}
+{"files":[{"path":"nemo_automodel/components/models/prism/__init__.py","added":0,"deleted":0,"class":"arch"},{"path":"nemo_automodel/components/models/prism/prism_entry.py","added":698,"deleted":0,"class":"arch"},{"path":"nemo_automodel/components/models/prism/tok8k/tokenizer.json","added":1,"deleted":0,"class":"arch"},{"path":"nemo_automodel/components/models/prism/tok8k/tokenizer_config.json","added":9,"deleted":0,"class":"arch"},{"path":"tokenizer/tokenizer.json","added":1,"deleted":0,"class":"other"},{"path":"tokenizer/tokenizer_config.json","added":9,"deleted":0,"class":"other"}],"total_added":718,"total_deleted":0}

sources/nemo_automodel/components/models/prism/prism_entry.py

@@ -83,18 +83,10 @@ ADAM_LR = 0.003
 ADAM_BETAS = (0.9, 0.95)
 ADAM_WD = 0.0
 WARMUP_STEPS = 250
-DECAY_FRAC = 0.15              # доля бюджета под затухание, косинус до минимума
-#: Было 0.35: затухание начиналось на 65% бюджета, множитель падал до 0.03
-#: к середине прогона — последнюю треть модель почти не училась. Замерено
-#: парно на одном железе, одинаковые семена, 112.5M / 300M токенов:
-#:   семя 1: 485625 против 473125   (+12500)
-#:   семя 2: 488750 против 475625   (+13125)
-#: Прибавка идёт из piqa (+5.0) и winogrande (+5.2). Величина ОТНОСИТЕЛЬНАЯ
-#: (доля бюджета), поэтому форма расписания переносится на другой масштаб,
-#: в отличие от абсолютных величин.
+DECAY_FRAC = 0.35              # final share of the wall budget, cosine to min
 MIN_LR_FRAC = 0.0
 DOCSTART_FRAC = 0.5
-DATA_SEED = 1234
+DATA_SEED = 1303
 REPORT_EVERY = 25
 STEP_CEILING = 19000           # hard stop below the recipe's 20000
 VAL_LO, VAL_HI = 2048, 2304    # frozen val cut, removed from training texts
@@ -197,15 +189,25 @@ class RoPEDecoderLM(nn.Module):
         self.head.weight = self.tok_emb.weight      # tied
         self._cos = None
         self._sin = None
+        # muP (пункт 1): базовая ширина = D_MODEL, поэтому при боевой
+        # конфигурации оба множителя равны 1.0 и рецепт не меняется ни на бит.
+        self._mup_mult = math.sqrt(float(D_MODEL) / float(d_model))
+        self._mup_out = float(D_MODEL) / float(d_model)
         self.apply(self._init)
         for blk in self.layers:                     # clean residual stream at t=0
             nn.init.zeros_(blk.att.wo.weight)
             nn.init.zeros_(blk.mlp.w2.weight)
 
-    @staticmethod
-    def _init(m):
-        if isinstance(m, (nn.Linear, nn.Embedding)):
+    def _init(self, m):
+        # Скрытые веса масштабируем по ширине (muP), входной эмбеддинг и
+        # привязанный к нему readout — нет: в muP это "входные" веса.
+        # Порядок обхода тот же, что был у apply(staticmethod), число обращений
+        # к генератору то же — значит при _mup_mult == 1.0 результат тот же.
+        if isinstance(m, nn.Embedding):
             nn.init.normal_(m.weight, std=0.02)
+        elif isinstance(m, nn.Linear):
+            std = 0.02 if m is self.head else 0.02 * self._mup_mult
+            nn.init.normal_(m.weight, std=std)
 
     def _rope(self, device, dtype):
         if self._cos is None or self._cos.device != device or self._cos.dtype != dtype:
@@ -222,7 +224,10 @@ class RoPEDecoderLM(nn.Module):
         cos, sin = self._rope(ids.device, x.dtype)
         for blk in self.layers:
             x = blk(x, cos, sin)
-        return self.head(self.norm_f(x))
+        logits = self.head(self.norm_f(x))
+        # muP readout: при базовой ширине множитель ровно 1.0 — умножение
+        # пропускаем, чтобы боевой путь остался нетронутым и в графе тоже.
+        return logits if self._mup_out == 1.0 else logits * self._mup_out
 
 
 # ------------------------------ tokenizer --------------------------------