specviz-tinystories

Toy model family for visualizing speculative decoding. Shared 4k byte-level BPE tokenizer.

folder	what	notes
target/	8-layer / 512-d GPT (RoPE, RMSNorm, tied emb)	trained with LayerSkip-style early-exit loss, so any prefix of layers is a usable self-drafter
draft/	2-layer / 256-d GPT	distilled from target (KL + CE)
medusa/	4 Medusa heads	on frozen target
hydra/	4 Hydra (sequentially dependent) heads	on frozen target
eagle/	1-layer EAGLE feature drafter	on frozen target

Each folder has model.safetensors, config.json, metrics.json (teacher-forced acceptance estimates on validation), and tokenizer.json.

Metrics (validation, teacher-forced)

{
  "draft": {
    "val_ce": 1.647251844406128,
    "kl_to_target": 0.4337791875004769,
    "alpha": 0.7311673790216446,
    "greedy": 0.7380981445312498,
    "iter": 12000
  },
  "eagle": {
    "reg": 0.115620232000947,
    "alpha": 0.7858343034982681,
    "greedy": 0.7855078756809234,
    "iter": 10000
  },
  "hydra": {
    "head1_alpha": 0.6419515252113344,
    "head1_greedy": 0.6670437842607498,
    "head2_alpha": 0.5645744204521179,
    "head2_greedy": 0.5964047521352768,
    "head3_alpha": 0.5303537666797639,
    "head3_greedy": 0.5645859122276305,
    "head4_alpha": 0.5123031288385391,
    "head4_greedy": 0.5469441980123521,
    "iter": 6000
  },
  "medusa": {
    "head1_alpha": 0.4397877633571626,
    "head1_greedy": 0.49165851026773455,
    "head2_alpha": 0.2549418851733207,
    "head2_greedy": 0.31615198105573644,
    "head3_alpha": 0.17144695296883586,
    "head3_greedy": 0.22621102407574656,
    "head4_alpha": 0.13068613037467003,
    "head4_greedy": 0.17579355239868166,
    "iter": 6000
  },
  "target": {
    "val_ce": 1.2464708268642428,
    "exit1_alpha": 0.4845910012722016,
    "exit1_greedy": 0.5295845031738281,
    "exit2_alpha": 0.6358921855688096,
    "exit2_greedy": 0.6550628662109373,
    "exit3_alpha": 0.7252050876617432,
    "exit3_greedy": 0.7312286376953127,
    "exit4_alpha": 0.7719815969467161,
    "exit4_greedy": 0.772979736328125,
    "exit5_alpha": 0.8381113588809967,
    "exit5_greedy": 0.8338546752929686,
    "exit6_alpha": 0.8831344306468962,
    "exit6_greedy": 0.8760437011718749,
    "exit7_alpha": 0.9238532721996306,
    "exit7_greedy": 0.9169792175292967,
    "iter": 20000
  },
  "sanity": {
    "draft_model": {
      "greedy_lossless": true,
      "mean_accepted": 2.0,
      "tokens_per_target_call": 3.0
    },
    "layerskip_exit3": {
      "greedy_lossless": true,
      "mean_accepted": 2.4285714285714284,
      "tokens_per_target_call": 3.4285714285714284
    },
    "spec_sample_T1": {
      "mean_accepted": 1.813953488372093,
      "tokens_per_target_call": 2.7906976744186047
    }
  }
}