upload specviz toy models

README.md

@@ -0,0 +1,94 @@
+---
+license: mit
+datasets: [roneneldan/TinyStories]
+tags: [speculative-decoding, toy-model, tinystories]
+---
+# specviz-tinystories
+
+Toy model family for visualizing speculative decoding. Shared 4k byte-level BPE tokenizer.
+
+| folder | what | notes |
+|---|---|---|
+| `target/` | 8-layer / 512-d GPT (RoPE, RMSNorm, tied emb) | trained with LayerSkip-style early-exit loss, so any prefix of layers is a usable self-drafter |
+| `draft/` | 2-layer / 256-d GPT | distilled from `target` (KL + CE) |
+| `medusa/` | 4 Medusa heads | on frozen `target` |
+| `hydra/` | 4 Hydra (sequentially dependent) heads | on frozen `target` |
+| `eagle/` | 1-layer EAGLE feature drafter | on frozen `target` |
+
+Each folder has `model.safetensors`, `config.json`, `metrics.json` (teacher-forced
+acceptance estimates on validation), and `tokenizer.json`.
+
+## Metrics (validation, teacher-forced)
+```json
+{
+  "draft": {
+    "val_ce": 1.647251844406128,
+    "kl_to_target": 0.4337791875004769,
+    "alpha": 0.7311673790216446,
+    "greedy": 0.7380981445312498,
+    "iter": 12000
+  },
+  "eagle": {
+    "reg": 0.115620232000947,
+    "alpha": 0.7858343034982681,
+    "greedy": 0.7855078756809234,
+    "iter": 10000
+  },
+  "hydra": {
+    "head1_alpha": 0.6419515252113344,
+    "head1_greedy": 0.6670437842607498,
+    "head2_alpha": 0.5645744204521179,
+    "head2_greedy": 0.5964047521352768,
+    "head3_alpha": 0.5303537666797639,
+    "head3_greedy": 0.5645859122276305,
+    "head4_alpha": 0.5123031288385391,
+    "head4_greedy": 0.5469441980123521,
+    "iter": 6000
+  },
+  "medusa": {
+    "head1_alpha": 0.4397877633571626,
+    "head1_greedy": 0.49165851026773455,
+    "head2_alpha": 0.2549418851733207,
+    "head2_greedy": 0.31615198105573644,
+    "head3_alpha": 0.17144695296883586,
+    "head3_greedy": 0.22621102407574656,
+    "head4_alpha": 0.13068613037467003,
+    "head4_greedy": 0.17579355239868166,
+    "iter": 6000
+  },
+  "target": {
+    "val_ce": 1.2464708268642428,
+    "exit1_alpha": 0.4845910012722016,
+    "exit1_greedy": 0.5295845031738281,
+    "exit2_alpha": 0.6358921855688096,
+    "exit2_greedy": 0.6550628662109373,
+    "exit3_alpha": 0.7252050876617432,
+    "exit3_greedy": 0.7312286376953127,
+    "exit4_alpha": 0.7719815969467161,
+    "exit4_greedy": 0.772979736328125,
+    "exit5_alpha": 0.8381113588809967,
+    "exit5_greedy": 0.8338546752929686,
+    "exit6_alpha": 0.8831344306468962,
+    "exit6_greedy": 0.8760437011718749,
+    "exit7_alpha": 0.9238532721996306,
+    "exit7_greedy": 0.9169792175292967,
+    "iter": 20000
+  },
+  "sanity": {
+    "draft_model": {
+      "greedy_lossless": true,
+      "mean_accepted": 2.0,
+      "tokens_per_target_call": 3.0
+    },
+    "layerskip_exit3": {
+      "greedy_lossless": true,
+      "mean_accepted": 2.4285714285714284,
+      "tokens_per_target_call": 3.4285714285714284
+    },
+    "spec_sample_T1": {
+      "mean_accepted": 1.813953488372093,
+      "tokens_per_target_call": 2.7906976744186047
+    }
+  }
+}
+```

draft/config.json

@@ -0,0 +1,16 @@
+{
+  "kind": "gpt",
+  "role": "draft",
+  "model": {
+    "vocab_size": 4096,
+    "block_size": 512,
+    "n_layer": 2,
+    "n_head": 4,
+    "n_embd": 256,
+    "rope_base": 10000.0
+  },
+  "train": {
+    "kd_alpha": 0.9,
+    "distilled_from": "target"
+  }
+}

draft/metrics.json

@@ -0,0 +1,95 @@
+{
+  "last": {
+    "val_ce": 1.647251844406128,
+    "kl_to_target": 0.4337791875004769,
+    "alpha": 0.7311673790216446,
+    "greedy": 0.7380981445312498,
+    "iter": 12000
+  },
+  "history": [
+    {
+      "val_ce": 2.0099524259567256,
+      "kl_to_target": 0.7797438204288483,
+      "alpha": 0.622846382856369,
+      "greedy": 0.6417129516601563,
+      "iter": 1000
+    },
+    {
+      "val_ce": 1.8566814422607423,
+      "kl_to_target": 0.6337778121232986,
+      "alpha": 0.6686296522617341,
+      "greedy": 0.6780601501464842,
+      "iter": 2000
+    },
+    {
+      "val_ce": 1.792548328638077,
+      "kl_to_target": 0.5724014222621918,
+      "alpha": 0.6858189970254898,
+      "greedy": 0.6960624694824219,
+      "iter": 3000
+    },
+    {
+      "val_ce": 1.7606663465499879,
+      "kl_to_target": 0.5405576616525649,
+      "alpha": 0.6973596751689911,
+      "greedy": 0.7049896240234375,
+      "iter": 4000
+    },
+    {
+      "val_ce": 1.746587711572647,
+      "kl_to_target": 0.5206510752439498,
+      "alpha": 0.7006595581769943,
+      "greedy": 0.7090461730957033,
+      "iter": 5000
+    },
+    {
+      "val_ce": 1.7139503300189969,
+      "kl_to_target": 0.5007144927978516,
+      "alpha": 0.7079429775476455,
+      "greedy": 0.7160675048828127,
+      "iter": 6000
+    },
+    {
+      "val_ce": 1.7016730666160584,
+      "kl_to_target": 0.4825879991054535,
+      "alpha": 0.7134584307670593,
+      "greedy": 0.7211517333984375,
+      "iter": 7000
+    },
+    {
+      "val_ce": 1.6721126437187195,
+      "kl_to_target": 0.46867714226245877,
+      "alpha": 0.7192983746528624,
+      "greedy": 0.7264602661132813,
+      "iter": 8000
+    },
+    {
+      "val_ce": 1.673528206348419,
+      "kl_to_target": 0.4556530773639678,
+      "alpha": 0.7239526838064192,
+      "greedy": 0.7305328369140626,
+      "iter": 9000
+    },
+    {
+      "val_ce": 1.6636894285678863,
+      "kl_to_target": 0.4467218339443207,
+      "alpha": 0.7272016823291778,
+      "greedy": 0.7334213256835938,
+      "iter": 10000
+    },
+    {
+      "val_ce": 1.6511635184288023,
+      "kl_to_target": 0.43592406362295144,
+      "alpha": 0.7305143564939498,
+      "greedy": 0.7367881774902344,
+      "iter": 11000
+    },
+    {
+      "val_ce": 1.647251844406128,
+      "kl_to_target": 0.4337791875004769,
+      "alpha": 0.7311673790216446,
+      "greedy": 0.7380981445312498,
+      "iter": 12000
+    }
+  ]
+}

draft/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b04a1aaf47be6790513bf2bda575a4130348fe42a82bf26507825ee81e7034b1
+size 10492208

eagle/config.json

@@ -0,0 +1,16 @@
+{
+  "kind": "eagle",
+  "model": {
+    "vocab_size": 4096,
+    "block_size": 512,
+    "n_layer": 8,
+    "n_head": 8,
+    "n_embd": 512,
+    "rope_base": 10000.0
+  },
+  "target": "target",
+  "train": {
+    "w_cls": 0.5,
+    "feat_noise": 0.1
+  }
+}

eagle/metrics.json

@@ -0,0 +1,70 @@
+{
+  "last": {
+    "reg": 0.115620232000947,
+    "alpha": 0.7858343034982681,
+    "greedy": 0.7855078756809234,
+    "iter": 10000
+  },
+  "history": [
+    {
+      "reg": 0.16248844116926192,
+      "alpha": 0.7045249938964844,
+      "greedy": 0.7131298869848252,
+      "iter": 1000
+    },
+    {
+      "reg": 0.1393620289862156,
+      "alpha": 0.7411754161119462,
+      "greedy": 0.7447376430034637,
+      "iter": 2000
+    },
+    {
+      "reg": 0.13170703276991844,
+      "alpha": 0.7544642329216005,
+      "greedy": 0.7579302132129669,
+      "iter": 3000
+    },
+    {
+      "reg": 0.1273528974503279,
+      "alpha": 0.7619078665971756,
+      "greedy": 0.7615092933177947,
+      "iter": 4000
+    },
+    {
+      "reg": 0.1248361334204674,
+      "alpha": 0.7684480398893356,
+      "greedy": 0.7694517374038696,
+      "iter": 5000
+    },
+    {
+      "reg": 0.12067152783274653,
+      "alpha": 0.7745981186628341,
+      "greedy": 0.7758852064609528,
+      "iter": 6000
+    },
+    {
+      "reg": 0.11911505199968815,
+      "alpha": 0.7785812288522721,
+      "greedy": 0.7784430116415023,
+      "iter": 7000
+    },
+    {
+      "reg": 0.11806977093219759,
+      "alpha": 0.7828734844923019,
+      "greedy": 0.7825571775436401,
+      "iter": 8000
+    },
+    {
+      "reg": 0.11849058754742146,
+      "alpha": 0.7845947831869126,
+      "greedy": 0.7849345535039902,
+      "iter": 9000
+    },
+    {
+      "reg": 0.115620232000947,
+      "alpha": 0.7858343034982681,
+      "greedy": 0.7855078756809234,
+      "iter": 10000
+    }
+  ]
+}

eagle/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0ba82601b36db042de1bbc469ace334de868e977cb746b29bb44e901669b0176
+size 14686912

hydra/config.json

@@ -0,0 +1,13 @@
+{
+  "kind": "hydra",
+  "K": 4,
+  "target": "target",
+  "target_model": {
+    "vocab_size": 4096,
+    "block_size": 512,
+    "n_layer": 8,
+    "n_head": 8,
+    "n_embd": 512,
+    "rope_base": 10000.0
+  }
+}

hydra/metrics.json

@@ -0,0 +1,81 @@
+{
+  "last": {
+    "head1_alpha": 0.6419515252113344,
+    "head1_greedy": 0.6670437842607498,
+    "head2_alpha": 0.5645744204521179,
+    "head2_greedy": 0.5964047521352768,
+    "head3_alpha": 0.5303537666797639,
+    "head3_greedy": 0.5645859122276305,
+    "head4_alpha": 0.5123031288385391,
+    "head4_greedy": 0.5469441980123521,
+    "iter": 6000
+  },
+  "history": [
+    {
+      "head1_alpha": 0.5846165388822556,
+      "head1_greedy": 0.6149431198835373,
+      "head2_alpha": 0.48963863402605057,
+      "head2_greedy": 0.524047213792801,
+      "head3_alpha": 0.4546733692288399,
+      "head3_greedy": 0.4870641157031059,
+      "head4_alpha": 0.4275311261415482,
+      "head4_greedy": 0.4656050026416778,
+      "iter": 1000
+    },
+    {
+      "head1_alpha": 0.6139427363872528,
+      "head1_greedy": 0.641572892665863,
+      "head2_alpha": 0.5227782577276229,
+      "head2_greedy": 0.5582904785871506,
+      "head3_alpha": 0.48853647857904436,
+      "head3_greedy": 0.5266699612140655,
+      "head4_alpha": 0.4622075036168098,
+      "head4_greedy": 0.501694755256176,
+      "iter": 2000
+    },
+    {
+      "head1_alpha": 0.629333007335663,
+      "head1_greedy": 0.6546905517578125,
+      "head2_alpha": 0.5431721359491348,
+      "head2_greedy": 0.5781771183013916,
+      "head3_alpha": 0.510604503750801,
+      "head3_greedy": 0.5455780595541,
+      "head4_alpha": 0.4853551983833313,
+      "head4_greedy": 0.5253983169794083,
+      "iter": 3000
+    },
+    {
+      "head1_alpha": 0.634756550192833,
+      "head1_greedy": 0.6599574863910674,
+      "head2_alpha": 0.5565346986055374,
+      "head2_greedy": 0.5882981330156325,
+      "head3_alpha": 0.5199958384037018,
+      "head3_greedy": 0.5554257839918137,
+      "head4_alpha": 0.5002639740705489,
+      "head4_greedy": 0.5371001482009888,
+      "iter": 4000
+    },
+    {
+      "head1_alpha": 0.6393299520015717,
+      "head1_greedy": 0.6642306625843047,
+      "head2_alpha": 0.5620603740215302,
+      "head2_greedy": 0.5925521194934846,
+      "head3_alpha": 0.5261775463819504,
+      "head3_greedy": 0.5606366783380508,
+      "head4_alpha": 0.5070496618747711,
+      "head4_greedy": 0.5432009696960449,
+      "iter": 5000
+    },
+    {
+      "head1_alpha": 0.6419515252113344,
+      "head1_greedy": 0.6670437842607498,
+      "head2_alpha": 0.5645744204521179,
+      "head2_greedy": 0.5964047521352768,
+      "head3_alpha": 0.5303537666797639,
+      "head3_greedy": 0.5645859122276305,
+      "head4_alpha": 0.5123031288385391,
+      "head4_greedy": 0.5469441980123521,
+      "iter": 6000
+    }
+  ]
+}

hydra/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:361c980c34a10c60090bc7d1d9622c2c92e435dfc9ce02515ca0a4218b987cb3
+size 18892080

medusa/config.json

@@ -0,0 +1,13 @@
+{
+  "kind": "medusa",
+  "K": 4,
+  "target": "target",
+  "target_model": {
+    "vocab_size": 4096,
+    "block_size": 512,
+    "n_layer": 8,
+    "n_head": 8,
+    "n_embd": 512,
+    "rope_base": 10000.0
+  }
+}

medusa/metrics.json

@@ -0,0 +1,81 @@
+{
+  "last": {
+    "head1_alpha": 0.4397877633571626,
+    "head1_greedy": 0.49165851026773455,
+    "head2_alpha": 0.2549418851733207,
+    "head2_greedy": 0.31615198105573644,
+    "head3_alpha": 0.17144695296883586,
+    "head3_greedy": 0.22621102407574656,
+    "head4_alpha": 0.13068613037467003,
+    "head4_greedy": 0.17579355239868166,
+    "iter": 6000
+  },
+  "history": [
+    {
+      "head1_alpha": 0.42103663235902783,
+      "head1_greedy": 0.471053996682167,
+      "head2_alpha": 0.24204960912466045,
+      "head2_greedy": 0.3009237289428711,
+      "head3_alpha": 0.16207220479846002,
+      "head3_greedy": 0.21406250670552254,
+      "head4_alpha": 0.1241915762424469,
+      "head4_greedy": 0.1651267223060131,
+      "iter": 1000
+    },
+    {
+      "head1_alpha": 0.4280305802822113,
+      "head1_greedy": 0.4776082381606102,
+      "head2_alpha": 0.24610844999551776,
+      "head2_greedy": 0.30440258979797363,
+      "head3_alpha": 0.16589450091123584,
+      "head3_greedy": 0.2168037891387939,
+      "head4_alpha": 0.1272031757980585,
+      "head4_greedy": 0.16875307410955426,
+      "iter": 2000
+    },
+    {
+      "head1_alpha": 0.43427005112171174,
+      "head1_greedy": 0.4846318483352661,
+      "head2_alpha": 0.25126655772328377,
+      "head2_greedy": 0.31229781210422514,
+      "head3_alpha": 0.16857100352644916,
+      "head3_greedy": 0.2231105804443359,
+      "head4_alpha": 0.12878014594316484,
+      "head4_greedy": 0.17323603406548505,
+      "iter": 3000
+    },
+    {
+      "head1_alpha": 0.4371277198195458,
+      "head1_greedy": 0.488535037636757,
+      "head2_alpha": 0.2534819707274437,
+      "head2_greedy": 0.3144929721951485,
+      "head3_alpha": 0.17099412232637404,
+      "head3_greedy": 0.22533614486455916,
+      "head4_alpha": 0.13038476184010506,
+      "head4_greedy": 0.17381120920181278,
+      "iter": 4000
+    },
+    {
+      "head1_alpha": 0.43966121226549154,
+      "head1_greedy": 0.49001803547143935,
+      "head2_alpha": 0.2547260284423828,
+      "head2_greedy": 0.31685357391834257,
+      "head3_alpha": 0.1713761068880558,
+      "head3_greedy": 0.226174183934927,
+      "head4_alpha": 0.1306981660425663,
+      "head4_greedy": 0.17601808533072472,
+      "iter": 5000
+    },
+    {
+      "head1_alpha": 0.4397877633571626,
+      "head1_greedy": 0.49165851026773455,
+      "head2_alpha": 0.2549418851733207,
+      "head2_greedy": 0.31615198105573644,
+      "head3_alpha": 0.17144695296883586,
+      "head3_greedy": 0.22621102407574656,
+      "head4_alpha": 0.13068613037467003,
+      "head4_greedy": 0.17579355239868166,
+      "iter": 6000
+    }
+  ]
+}

medusa/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:57670da9538e3bd0638da006ab1eaacd98da80889ba9242afa78deb46e09f6d8
+size 37757960

sanity.json

@@ -0,0 +1,16 @@
+{
+  "draft_model": {
+    "greedy_lossless": true,
+    "mean_accepted": 2.0,
+    "tokens_per_target_call": 3.0
+  },
+  "layerskip_exit3": {
+    "greedy_lossless": true,
+    "mean_accepted": 2.4285714285714284,
+    "tokens_per_target_call": 3.4285714285714284
+  },
+  "spec_sample_T1": {
+    "mean_accepted": 1.813953488372093,
+    "tokens_per_target_call": 2.7906976744186047
+  }
+}

target/config.json

@@ -0,0 +1,15 @@
+{
+  "kind": "gpt",
+  "role": "target",
+  "model": {
+    "vocab_size": 4096,
+    "block_size": 512,
+    "n_layer": 8,
+    "n_head": 8,
+    "n_embd": 512,
+    "rope_base": 10000.0
+  },
+  "train": {
+    "early_exit": 0.25
+  }
+}

target/metrics.json

@@ -0,0 +1,382 @@
+{
+  "last": {
+    "val_ce": 1.2464708268642428,
+    "exit1_alpha": 0.4845910012722016,
+    "exit1_greedy": 0.5295845031738281,
+    "exit2_alpha": 0.6358921855688096,
+    "exit2_greedy": 0.6550628662109373,
+    "exit3_alpha": 0.7252050876617432,
+    "exit3_greedy": 0.7312286376953127,
+    "exit4_alpha": 0.7719815969467161,
+    "exit4_greedy": 0.772979736328125,
+    "exit5_alpha": 0.8381113588809967,
+    "exit5_greedy": 0.8338546752929686,
+    "exit6_alpha": 0.8831344306468962,
+    "exit6_greedy": 0.8760437011718749,
+    "exit7_alpha": 0.9238532721996306,
+    "exit7_greedy": 0.9169792175292967,
+    "iter": 20000
+  },
+  "history": [
+    {
+      "val_ce": 1.771557992696762,
+      "exit1_alpha": 0.40818989574909204,
+      "exit1_greedy": 0.4042259216308594,
+      "exit2_alpha": 0.5836701780557633,
+      "exit2_greedy": 0.5889640808105467,
+      "exit3_alpha": 0.7103229671716691,
+      "exit3_greedy": 0.7000419616699218,
+      "exit4_alpha": 0.770584425330162,
+      "exit4_greedy": 0.7579452514648438,
+      "exit5_alpha": 0.8510826379060744,
+      "exit5_greedy": 0.8368385314941407,
+      "exit6_alpha": 0.897156646847725,
+      "exit6_greedy": 0.884052276611328,
+      "exit7_alpha": 0.9390101432800292,
+      "exit7_greedy": 0.9295768737792968,
+      "iter": 1000
+    },
+    {
+      "val_ce": 1.578459221124649,
+      "exit1_alpha": 0.40710014700889596,
+      "exit1_greedy": 0.42163848876953125,
+      "exit2_alpha": 0.5925680071115492,
+      "exit2_greedy": 0.6041282653808594,
+      "exit3_alpha": 0.7103522688150407,
+      "exit3_greedy": 0.7058708190917969,
+      "exit4_alpha": 0.7660878688097,
+      "exit4_greedy": 0.7575103759765625,
+      "exit5_alpha": 0.8463849425315857,
+      "exit5_greedy": 0.8351921081542968,
+      "exit6_alpha": 0.8927793353796003,
+      "exit6_greedy": 0.8805702209472657,
+      "exit7_alpha": 0.9349214017391205,
+      "exit7_greedy": 0.9257751464843752,
+      "iter": 2000
+    },
+    {
+      "val_ce": 1.5022810697555542,
+      "exit1_alpha": 0.4170039817690849,
+      "exit1_greedy": 0.44345245361328134,
+      "exit2_alpha": 0.5942453950643539,
+      "exit2_greedy": 0.6097404479980467,
+      "exit3_alpha": 0.708651602268219,
+      "exit3_greedy": 0.7068855285644531,
+      "exit4_alpha": 0.7627668380737305,
+      "exit4_greedy": 0.7567466735839844,
+      "exit5_alpha": 0.8417250126600265,
+      "exit5_greedy": 0.8329322814941403,
+      "exit6_alpha": 0.8888692647218703,
+      "exit6_greedy": 0.8786048889160155,
+      "exit7_alpha": 0.9317562252283097,
+      "exit7_greedy": 0.9234054565429688,
+      "iter": 3000
+    },
+    {
+      "val_ce": 1.4586917817592617,
+      "exit1_alpha": 0.433258655667305,
+      "exit1_greedy": 0.4643768310546875,
+      "exit2_alpha": 0.6047763884067535,
+      "exit2_greedy": 0.6220146179199219,
+      "exit3_alpha": 0.7129614770412446,
+      "exit3_greedy": 0.7139602661132812,
+      "exit4_alpha": 0.765247431397438,
+      "exit4_greedy": 0.7619926452636717,
+      "exit5_alpha": 0.8431548476219176,
+      "exit5_greedy": 0.8360275268554689,
+      "exit6_alpha": 0.8900604933500288,
+      "exit6_greedy": 0.8808296203613284,
+      "exit7_alpha": 0.9319884389638898,
+      "exit7_greedy": 0.9244018554687499,
+      "iter": 4000
+    },
+    {
+      "val_ce": 1.433526623249054,
+      "exit1_alpha": 0.44229043573141097,
+      "exit1_greedy": 0.4734825134277343,
+      "exit2_alpha": 0.6097689032554625,
+      "exit2_greedy": 0.6249656677246095,
+      "exit3_alpha": 0.7152164906263351,
+      "exit3_greedy": 0.7157157897949219,
+      "exit4_alpha": 0.7679181784391402,
+      "exit4_greedy": 0.7638214111328127,
+      "exit5_alpha": 0.842146846652031,
+      "exit5_greedy": 0.8342453002929687,
+      "exit6_alpha": 0.8888639360666275,
+      "exit6_greedy": 0.8797622680664063,
+      "exit7_alpha": 0.930886721611023,
+      "exit7_greedy": 0.9237228393554686,
+      "iter": 5000
+    },
+    {
+      "val_ce": 1.39659908413887,
+      "exit1_alpha": 0.4487991124391556,
+      "exit1_greedy": 0.4879180908203125,
+      "exit2_alpha": 0.6146820276975632,
+      "exit2_greedy": 0.6330322265624999,
+      "exit3_alpha": 0.717953586578369,
+      "exit3_greedy": 0.7202896118164063,
+      "exit4_alpha": 0.7694938749074937,
+      "exit4_greedy": 0.7674110412597656,
+      "exit5_alpha": 0.84256731569767,
+      "exit5_greedy": 0.8361511230468749,
+      "exit6_alpha": 0.8886137694120406,
+      "exit6_greedy": 0.8800559997558592,
+      "exit7_alpha": 0.9299631059169766,
+      "exit7_greedy": 0.9224426269531251,
+      "iter": 6000
+    },
+    {
+      "val_ce": 1.383489817380905,
+      "exit1_alpha": 0.45552693754434587,
+      "exit1_greedy": 0.4913726806640625,
+      "exit2_alpha": 0.6181112408638,
+      "exit2_greedy": 0.6341178894042968,
+      "exit3_alpha": 0.7176161557435988,
+      "exit3_greedy": 0.7191116333007813,
+      "exit4_alpha": 0.7689908534288407,
+      "exit4_greedy": 0.7660240173339844,
+      "exit5_alpha": 0.8409761995077134,
+      "exit5_greedy": 0.834588623046875,
+      "exit6_alpha": 0.8872842937707902,
+      "exit6_greedy": 0.8789833068847654,
+      "exit7_alpha": 0.9289374917745589,
+      "exit7_greedy": 0.9214012145996094,
+      "iter": 7000
+    },
+    {
+      "val_ce": 1.3478731572628022,
+      "exit1_alpha": 0.4623634889721871,
+      "exit1_greedy": 0.4993957519531251,
+      "exit2_alpha": 0.621457040309906,
+      "exit2_greedy": 0.6377197265625,
+      "exit3_alpha": 0.7188945442438125,
+      "exit3_greedy": 0.7204681396484375,
+      "exit4_alpha": 0.7694859355688096,
+      "exit4_greedy": 0.7671745300292967,
+      "exit5_alpha": 0.8412264496088029,
+      "exit5_greedy": 0.8346000671386719,
+      "exit6_alpha": 0.886994031071663,
+      "exit6_greedy": 0.8790596008300781,
+      "exit7_alpha": 0.9285726338624954,
+      "exit7_greedy": 0.9216156005859374,
+      "iter": 8000
+    },
+    {
+      "val_ce": 1.3483326613903046,
+      "exit1_alpha": 0.4615658625960351,
+      "exit1_greedy": 0.5015533447265625,
+      "exit2_alpha": 0.6214722603559494,
+      "exit2_greedy": 0.6372215270996094,
+      "exit3_alpha": 0.7183588057756425,
+      "exit3_greedy": 0.7196212768554687,
+      "exit4_alpha": 0.768870809674263,
+      "exit4_greedy": 0.7661483764648437,
+      "exit5_alpha": 0.839602640271187,
+      "exit5_greedy": 0.8330055236816406,
+      "exit6_alpha": 0.8852128148078918,
+      "exit6_greedy": 0.8765960693359375,
+      "exit7_alpha": 0.9268552124500274,
+      "exit7_greedy": 0.9186050415039063,
+      "iter": 9000
+    },
+    {
+      "val_ce": 1.3316114962100982,
+      "exit1_alpha": 0.4700911849737167,
+      "exit1_greedy": 0.5082527160644531,
+      "exit2_alpha": 0.6264044821262361,
+      "exit2_greedy": 0.6428421020507813,
+      "exit3_alpha": 0.7207205474376678,
+      "exit3_greedy": 0.7237045288085937,
+      "exit4_alpha": 0.7705703556537628,
+      "exit4_greedy": 0.7688064575195312,
+      "exit5_alpha": 0.8408509403467177,
+      "exit5_greedy": 0.8349418640136721,
+      "exit6_alpha": 0.886443716287613,
+      "exit6_greedy": 0.8784645080566407,
+      "exit7_alpha": 0.9278432577848436,
+      "exit7_greedy": 0.9202407836914063,
+      "iter": 10000
+    },
+    {
+      "val_ce": 1.3176176846027374,
+      "exit1_alpha": 0.4697788402438164,
+      "exit1_greedy": 0.5085655212402345,
+      "exit2_alpha": 0.6254494339227676,
+      "exit2_greedy": 0.64173583984375,
+      "exit3_alpha": 0.7195282131433487,
+      "exit3_greedy": 0.7223030090332031,
+      "exit4_alpha": 0.768567582964897,
+      "exit4_greedy": 0.7674957275390626,
+      "exit5_alpha": 0.8386009097099303,
+      "exit5_greedy": 0.8323188781738281,
+      "exit6_alpha": 0.8845931142568589,
+      "exit6_greedy": 0.8763595581054688,
+      "exit7_alpha": 0.9260558307170867,
+      "exit7_greedy": 0.9184646606445311,
+      "iter": 11000
+    },
+    {
+      "val_ce": 1.3059154570102693,
+      "exit1_alpha": 0.4728017628192901,
+      "exit1_greedy": 0.512060546875,
+      "exit2_alpha": 0.6275364607572556,
+      "exit2_greedy": 0.6449378967285156,
+      "exit3_alpha": 0.7219109445810318,
+      "exit3_greedy": 0.7250289916992186,
+      "exit4_alpha": 0.771248546242714,
+      "exit4_greedy": 0.7697929382324218,
+      "exit5_alpha": 0.8400995552539825,
+      "exit5_greedy": 0.8334976196289063,
+      "exit6_alpha": 0.8853256195783613,
+      "exit6_greedy": 0.8770217895507812,
+      "exit7_alpha": 0.9262120515108109,
+      "exit7_greedy": 0.9189743041992188,
+      "iter": 12000
+    },
+    {
+      "val_ce": 1.2948933601379393,
+      "exit1_alpha": 0.4731743648648262,
+      "exit1_greedy": 0.5167854309082032,
+      "exit2_alpha": 0.6282133609056473,
+      "exit2_greedy": 0.6462585449218751,
+      "exit3_alpha": 0.7214778035879135,
+      "exit3_greedy": 0.7259895324707033,
+      "exit4_alpha": 0.7699974924325942,
+      "exit4_greedy": 0.7699966430664062,
+      "exit5_alpha": 0.8384121149778363,
+      "exit5_greedy": 0.8336082458496096,
+      "exit6_alpha": 0.884169813990593,
+      "exit6_greedy": 0.8764663696289063,
+      "exit7_alpha": 0.9253626286983491,
+      "exit7_greedy": 0.9174041748046873,
+      "iter": 13000
+    },
+    {
+      "val_ce": 1.2827632665634154,
+      "exit1_alpha": 0.47814678400754934,
+      "exit1_greedy": 0.5198051452636718,
+      "exit2_alpha": 0.6312671095132828,
+      "exit2_greedy": 0.6491500854492187,
+      "exit3_alpha": 0.7226436764001848,
+      "exit3_greedy": 0.72613525390625,
+      "exit4_alpha": 0.7707418739795684,
+      "exit4_greedy": 0.7703193664550783,
+      "exit5_alpha": 0.8391160368919371,
+      "exit5_greedy": 0.8340530395507814,
+      "exit6_alpha": 0.8843333840370178,
+      "exit6_greedy": 0.8766525268554689,
+      "exit7_alpha": 0.9253863930702211,
+      "exit7_greedy": 0.9177177429199219,
+      "iter": 14000
+    },
+    {
+      "val_ce": 1.2767474710941316,
+      "exit1_alpha": 0.4784462839365005,
+      "exit1_greedy": 0.5214790344238283,
+      "exit2_alpha": 0.6313386857509614,
+      "exit2_greedy": 0.6499969482421876,
+      "exit3_alpha": 0.7227032780647278,
+      "exit3_greedy": 0.7276084899902343,
+      "exit4_alpha": 0.7713228404521941,
+      "exit4_greedy": 0.7715507507324219,
+      "exit5_alpha": 0.8385546714067458,
+      "exit5_greedy": 0.8338310241699219,
+      "exit6_alpha": 0.8839981228113175,
+      "exit6_greedy": 0.8766891479492187,
+      "exit7_alpha": 0.9249983906745911,
+      "exit7_greedy": 0.9178123474121093,
+      "iter": 15000
+    },
+    {
+      "val_ce": 1.2510948657989502,
+      "exit1_alpha": 0.4791820466518403,
+      "exit1_greedy": 0.522113037109375,
+      "exit2_alpha": 0.6319353550672531,
+      "exit2_greedy": 0.6504577636718749,
+      "exit3_alpha": 0.7221516221761702,
+      "exit3_greedy": 0.7268997192382812,
+      "exit4_alpha": 0.7699011594057082,
+      "exit4_greedy": 0.7703872680664061,
+      "exit5_alpha": 0.8373788088560106,
+      "exit5_greedy": 0.8321022033691405,
+      "exit6_alpha": 0.88275009393692,
+      "exit6_greedy": 0.8752052307128906,
+      "exit7_alpha": 0.9238883793354034,
+      "exit7_greedy": 0.9170501708984374,
+      "iter": 16000
+    },
+    {
+      "val_ce": 1.2593918740749364,
+      "exit1_alpha": 0.48289879858493817,
+      "exit1_greedy": 0.5271568298339844,
+      "exit2_alpha": 0.6351416736841201,
+      "exit2_greedy": 0.654430389404297,
+      "exit3_alpha": 0.7253278106451035,
+      "exit3_greedy": 0.7305152893066407,
+      "exit4_alpha": 0.7729058593511582,
+      "exit4_greedy": 0.7742111206054688,
+      "exit5_alpha": 0.8391105562448502,
+      "exit5_greedy": 0.8342247009277343,
+      "exit6_alpha": 0.8841602981090545,
+      "exit6_greedy": 0.8770401000976563,
+      "exit7_alpha": 0.9245910406112671,
+      "exit7_greedy": 0.9174201965332032,
+      "iter": 17000
+    },
+    {
+      "val_ce": 1.2452915370464326,
+      "exit1_alpha": 0.4827954232692719,
+      "exit1_greedy": 0.5270515441894532,
+      "exit2_alpha": 0.6341491907835006,
+      "exit2_greedy": 0.6547569274902343,
+      "exit3_alpha": 0.72407668530941,
+      "exit3_greedy": 0.7304679870605468,
+      "exit4_alpha": 0.7716502159833908,
+      "exit4_greedy": 0.773052978515625,
+      "exit5_alpha": 0.8385924726724626,
+      "exit5_greedy": 0.8343406677246094,
+      "exit6_alpha": 0.8831680685281753,
+      "exit6_greedy": 0.8763328552246094,
+      "exit7_alpha": 0.9239696323871613,
+      "exit7_greedy": 0.9172851562500002,
+      "iter": 18000
+    },
+    {
+      "val_ce": 1.2473996341228482,
+      "exit1_alpha": 0.484264649450779,
+      "exit1_greedy": 0.5282669067382812,
+      "exit2_alpha": 0.6363260746002196,
+      "exit2_greedy": 0.6554977416992187,
+      "exit3_alpha": 0.7256934702396393,
+      "exit3_greedy": 0.7315994262695313,
+      "exit4_alpha": 0.7724919855594633,
+      "exit4_greedy": 0.7737304687500001,
+      "exit5_alpha": 0.8385471880435944,
+      "exit5_greedy": 0.834428405761719,
+      "exit6_alpha": 0.8831394642591474,
+      "exit6_greedy": 0.8758834838867188,
+      "exit7_alpha": 0.9239964902400971,
+      "exit7_greedy": 0.9169631958007811,
+      "iter": 19000
+    },
+    {
+      "val_ce": 1.2464708268642428,
+      "exit1_alpha": 0.4845910012722016,
+      "exit1_greedy": 0.5295845031738281,
+      "exit2_alpha": 0.6358921855688096,
+      "exit2_greedy": 0.6550628662109373,
+      "exit3_alpha": 0.7252050876617432,
+      "exit3_greedy": 0.7312286376953127,
+      "exit4_alpha": 0.7719815969467161,
+      "exit4_greedy": 0.772979736328125,
+      "exit5_alpha": 0.8381113588809967,
+      "exit5_greedy": 0.8338546752929686,
+      "exit6_alpha": 0.8831344306468962,
+      "exit6_greedy": 0.8760437011718749,
+      "exit7_alpha": 0.9238532721996306,
+      "exit7_greedy": 0.9169792175292967,
+      "iter": 20000
+    }
+  ]
+}

target/model.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:92219629bfd8d9a35ac6139edd833c6b4400e1970558fc5cf0509bab6928fd0e
+size 109091456