tarekziade HF Staff commited on
Commit
ca2fe62
Β·
verified Β·
1 Parent(s): b36756e

Shrink pages: model title, linked pins, back link, source findings; add the refresh tools

Browse files
This view is limited to 50 files because it contains too many changes. Β  See raw diff
Files changed (50) hide show
  1. data.json +0 -0
  2. index.html +1 -1
  3. shrink/Apertus-v1.1-0.5B-Instruct-mic.html +4 -2
  4. shrink/Apertus-v1.1-0.5B-Instruct-mic.json +1 -0
  5. shrink/Cosmos3-Nano-mic.html +4 -2
  6. shrink/Cosmos3-Nano-mic.json +1 -0
  7. shrink/DeepSeek-Coder-V2-Lite-Instruct-mic.html +5 -3
  8. shrink/DeepSeek-Coder-V2-Lite-Instruct-mic.json +1 -0
  9. shrink/DeepSeek-R1-mic.html +4 -2
  10. shrink/DeepSeek-R1-mic.json +1 -0
  11. shrink/DeepSeek-V3.2-mic.html +4 -2
  12. shrink/DeepSeek-V3.2-mic.json +1 -0
  13. shrink/DeepSeek-V4-Flash-0731-mic.html +4 -2
  14. shrink/DeepSeek-V4-Flash-0731-mic.json +1 -0
  15. shrink/ESMC-300M-hf-mic.html +4 -2
  16. shrink/ESMC-300M-hf-mic.json +1 -0
  17. shrink/EuroBERT-210m-mic.html +4 -2
  18. shrink/EuroBERT-210m-mic.json +1 -0
  19. shrink/Falcon-H1-Tiny-90M-Instruct-mic.html +4 -2
  20. shrink/Falcon-H1-Tiny-90M-Instruct-mic.json +1 -0
  21. shrink/Florence-2-base-mic.html +4 -2
  22. shrink/Florence-2-base-mic.json +1 -0
  23. shrink/GLM-4-9B-0414-mic.html +4 -2
  24. shrink/GLM-4-9B-0414-mic.json +1 -0
  25. shrink/GLM-4.5-Air-mic.html +5 -3
  26. shrink/GLM-4.5-Air-mic.json +1 -0
  27. shrink/GLM-4.7-Flash-mic.html +4 -2
  28. shrink/GLM-4.7-Flash-mic.json +1 -0
  29. shrink/GLM-5.3-Flash-mic.html +4 -2
  30. shrink/GLM-5.3-Flash-mic.json +1 -0
  31. shrink/GLM-5.3-mic.html +5 -3
  32. shrink/GLM-5.3-mic.json +1 -0
  33. shrink/GLM-OCR-mic.html +4 -2
  34. shrink/GLM-OCR-mic.json +1 -0
  35. shrink/Hunyuan-0.5B-Instruct-mic.html +4 -2
  36. shrink/Hunyuan-0.5B-Instruct-mic.json +1 -0
  37. shrink/HunyuanOCR-mic.html +4 -2
  38. shrink/HunyuanOCR-mic.json +1 -0
  39. shrink/Hy-MT2-30B-A3B-mic.html +4 -2
  40. shrink/Hy-MT2-30B-A3B-mic.json +1 -0
  41. shrink/Inkling-mic.html +4 -2
  42. shrink/Inkling-mic.json +1 -0
  43. shrink/Jamba-tiny-dev-mic.html +4 -2
  44. shrink/Jamba-tiny-dev-mic.json +1 -0
  45. shrink/Janus-Pro-1B-mic.html +4 -2
  46. shrink/Janus-Pro-1B-mic.json +1 -0
  47. shrink/LCO-Embedding-Omni-3B-mic.html +4 -2
  48. shrink/LCO-Embedding-Omni-3B-mic.json +1 -0
  49. shrink/LFM2-8B-A1B-mic.html +4 -2
  50. shrink/LFM2-8B-A1B-mic.json +1 -0
data.json CHANGED
The diff for this file is too large to render. See raw diff
 
index.html CHANGED
@@ -174,7 +174,7 @@ a{color:var(--accent);text-decoration:none}a:hover{text-decoration:underline}
174
  const view = td('', 'top');
175
  if (r[1] === 'Done') for (const [page, findings] of shrink[r[0]] || []) {
176
  const a = document.createElement('a');
177
- a.href = `shrink/${page}.html`; a.target = '_blank'; a.rel = 'noopener';
178
  a.textContent = page.replace(/-mic$/, '') + (findings ? ` ⚠ ${findings}` : '');
179
  a.title = `${page}: ${findings ? findings + ' finding(s) to review' : 'same architecture, smaller'}`;
180
  if (view.childNodes.length) view.append(document.createElement('br'));
 
174
  const view = td('', 'top');
175
  if (r[1] === 'Done') for (const [page, findings] of shrink[r[0]] || []) {
176
  const a = document.createElement('a');
177
+ a.href = `shrink/${page}.html`;
178
  a.textContent = page.replace(/-mic$/, '') + (findings ? ` ⚠ ${findings}` : '');
179
  a.title = `${page}: ${findings ? findings + ' finding(s) to review' : 'same architecture, smaller'}`;
180
  if (view.childNodes.length) view.append(document.createElement('br'));
shrink/Apertus-v1.1-0.5B-Instruct-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/Apertus-v1.1-0.5B-Instruct-mic@55ae98cbccd9da534828bccc9b1f8bee4974731d ← swiss-ai/Apertus-v1.1-0.5B-Instruct@a140fd61fb57422c36a26301caea17edee799874 Β· ApertusForCausalLM Β· random weights</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">438.3M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">8.4M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 20</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i><i class="c0" title="layer 2: DecoderLayer"></i><i class="c0" title="layer 3: DecoderLayer"></i><i class="c0" title="layer 4: DecoderLayer"></i><i class="c0" title="layer 5: DecoderLayer"></i><i class="c0" title="layer 6: DecoderLayer"></i><i class="c0" title="layer 7: DecoderLayer"></i><i class="c0" title="layer 8: DecoderLayer"></i><i class="c0" title="layer 9: DecoderLayer"></i><i class="c0" title="layer 10: DecoderLayer"></i><i class="c0" title="layer 11: DecoderLayer"></i><i class="c0" title="layer 12: DecoderLayer"></i><i class="c0" title="layer 13: DecoderLayer"></i><i class="c0" title="layer 14: DecoderLayer"></i><i class="c0" title="layer 15: DecoderLayer"></i><i class="c0" title="layer 16: DecoderLayer"></i><i class="c0" title="layer 17: DecoderLayer"></i><i class="c0" title="layer 18: DecoderLayer"></i><i class="c0" title="layer 19: DecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>DecoderLayer</td><td>20</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.693809" title="model.layers: 304.1M (69.4%)"></i><i class="c1" style="flex:0.306189" title="model.embed_tokens: 134.2M (30.6%)"></i><i class="c2" style="flex:0.000002" title="model.norm: 1.0K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.006828" title="model.layers: 57.7K (0.7%)"></i><i class="c1" style="flex:0.993165" title="model.embed_tokens: 8.4M (99.3%)"></i><i class="c2" style="flex:0.000008" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>304.1M</td><td>69.4%</td><td>57.7K</td><td>0.7%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>134.2M</td><td>30.6%</td><td>8.4M</td><td>99.3%</td></tr><tr><td><i class="c2"></i>model.norm</td><td>1.0K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>GQA (16 heads, 4 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>linear</td><td>linear</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>apertus</td><td>apertus</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>131072</td><td>131072</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>xielu</td><td>xielu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>4096</td><td>4096</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Apertus omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>121</td><td>13</td></tr><tr class=""><td>RMSNorm</td><td>81</td><td>9</td></tr><tr class=""><td>Attention</td><td>20</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>20</td><td>2</td></tr><tr class=""><td>MLP</td><td>20</td><td>2</td></tr><tr class=""><td>XIELUActivation</td><td>20</td><td>2</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">5 fields changed, 26 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>hidden_size</td><td>1024</td><td>64</td></tr><tr><td>intermediate_size</td><td>6144</td><td>128</td></tr><tr><td>num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>num_hidden_layers</td><td>20</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>4</td><td>2</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>swiss-ai/Apertus-v1.1-0.5B-Instruct: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>swiss-ai/Apertus-v1.1-0.5B-Instruct</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/Apertus-v1.1-0.5B-Instruct-mic/tree/55ae98cbccd9da534828bccc9b1f8bee4974731d" title="huggingface-mic/Apertus-v1.1-0.5B-Instruct-mic@55ae98cbccd9da534828bccc9b1f8bee4974731d">huggingface-mic/Apertus-v1.1-0.5B-Instruct-mic@55ae98cbccd9</a> ← <a href="https://huggingface.co/swiss-ai/Apertus-v1.1-0.5B-Instruct/tree/a140fd61fb57422c36a26301caea17edee799874" title="swiss-ai/Apertus-v1.1-0.5B-Instruct@a140fd61fb57422c36a26301caea17edee799874">swiss-ai/Apertus-v1.1-0.5B-Instruct@a140fd61fb57</a> Β· ApertusForCausalLM Β· random weights</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">438.3M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">8.4M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 20</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i><i class="c0" title="layer 2: DecoderLayer"></i><i class="c0" title="layer 3: DecoderLayer"></i><i class="c0" title="layer 4: DecoderLayer"></i><i class="c0" title="layer 5: DecoderLayer"></i><i class="c0" title="layer 6: DecoderLayer"></i><i class="c0" title="layer 7: DecoderLayer"></i><i class="c0" title="layer 8: DecoderLayer"></i><i class="c0" title="layer 9: DecoderLayer"></i><i class="c0" title="layer 10: DecoderLayer"></i><i class="c0" title="layer 11: DecoderLayer"></i><i class="c0" title="layer 12: DecoderLayer"></i><i class="c0" title="layer 13: DecoderLayer"></i><i class="c0" title="layer 14: DecoderLayer"></i><i class="c0" title="layer 15: DecoderLayer"></i><i class="c0" title="layer 16: DecoderLayer"></i><i class="c0" title="layer 17: DecoderLayer"></i><i class="c0" title="layer 18: DecoderLayer"></i><i class="c0" title="layer 19: DecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>DecoderLayer</td><td>20</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.693809" title="model.layers: 304.1M (69.4%)"></i><i class="c1" style="flex:0.306189" title="model.embed_tokens: 134.2M (30.6%)"></i><i class="c2" style="flex:0.000002" title="model.norm: 1.0K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.006828" title="model.layers: 57.7K (0.7%)"></i><i class="c1" style="flex:0.993165" title="model.embed_tokens: 8.4M (99.3%)"></i><i class="c2" style="flex:0.000008" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>304.1M</td><td>69.4%</td><td>57.7K</td><td>0.7%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>134.2M</td><td>30.6%</td><td>8.4M</td><td>99.3%</td></tr><tr><td><i class="c2"></i>model.norm</td><td>1.0K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>GQA (16 heads, 4 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>linear</td><td>linear</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>apertus</td><td>apertus</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>131072</td><td>131072</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>xielu</td><td>xielu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>4096</td><td>4096</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Apertus omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>121</td><td>13</td></tr><tr class=""><td>RMSNorm</td><td>81</td><td>9</td></tr><tr class=""><td>Attention</td><td>20</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>20</td><td>2</td></tr><tr class=""><td>MLP</td><td>20</td><td>2</td></tr><tr class=""><td>XIELUActivation</td><td>20</td><td>2</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">5 fields changed, 26 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>hidden_size</td><td>1024</td><td>64</td></tr><tr><td>intermediate_size</td><td>6144</td><td>128</td></tr><tr><td>num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>num_hidden_layers</td><td>20</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>4</td><td>2</td></tr></table>
36
  </main></body></html>
shrink/Apertus-v1.1-0.5B-Instruct-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/Apertus-v1.1-0.5B-Instruct-mic@55ae98cbccd9da534828bccc9b1f8bee4974731d", "findings": []}
shrink/Cosmos3-Nano-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/Cosmos3-Nano-mic@4c4a117fd8d6b13993505f86a24b15580e8b25d6 ← nvidia/Cosmos3-Nano@e59a53c25979a090fa8706c9acc0c254a6e89b92 Β· Cosmos3OmniForConditionalGeneration Β· random weights</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">8.8B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">20.2M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">3</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.visual.blocks</h3><div class="row"><span class="lab">Original Β· 27</span><div class="cells"><i class="c0" title="layer 0: Qwen3VLVisionBlock"></i><i class="c0" title="layer 1: Qwen3VLVisionBlock"></i><i class="c0" title="layer 2: Qwen3VLVisionBlock"></i><i class="c0" title="layer 3: Qwen3VLVisionBlock"></i><i class="c0" title="layer 4: Qwen3VLVisionBlock"></i><i class="c0" title="layer 5: Qwen3VLVisionBlock"></i><i class="c0" title="layer 6: Qwen3VLVisionBlock"></i><i class="c0" title="layer 7: Qwen3VLVisionBlock"></i><i class="c0" title="layer 8: Qwen3VLVisionBlock"></i><i class="c0" title="layer 9: Qwen3VLVisionBlock"></i><i class="c0" title="layer 10: Qwen3VLVisionBlock"></i><i class="c0" title="layer 11: Qwen3VLVisionBlock"></i><i class="c0" title="layer 12: Qwen3VLVisionBlock"></i><i class="c0" title="layer 13: Qwen3VLVisionBlock"></i><i class="c0" title="layer 14: Qwen3VLVisionBlock"></i><i class="c0" title="layer 15: Qwen3VLVisionBlock"></i><i class="c0" title="layer 16: Qwen3VLVisionBlock"></i><i class="c0" title="layer 17: Qwen3VLVisionBlock"></i><i class="c0" title="layer 18: Qwen3VLVisionBlock"></i><i class="c0" title="layer 19: Qwen3VLVisionBlock"></i><i class="c0" title="layer 20: Qwen3VLVisionBlock"></i><i class="c0" title="layer 21: Qwen3VLVisionBlock"></i><i class="c0" title="layer 22: Qwen3VLVisionBlock"></i><i class="c0" title="layer 23: Qwen3VLVisionBlock"></i><i class="c0" title="layer 24: Qwen3VLVisionBlock"></i><i class="c0" title="layer 25: Qwen3VLVisionBlock"></i><i class="c0" title="layer 26: Qwen3VLVisionBlock"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: Qwen3VLVisionBlock"></i><i class="c0" title="layer 1: Qwen3VLVisionBlock"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>Qwen3VLVisionBlock</td><td>27</td><td>2</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.visual.deepstack_merger_list</h3><div class="row"><span class="lab">Original Β· 3</span><div class="cells"><i class="c0" title="layer 0: GELU"></i><i class="c0" title="layer 1: GELU"></i><i class="c0" title="layer 2: GELU"></i></div></div><div class="row"><span class="lab">Fixture Β· 3</span><div class="cells"><i class="c0" title="layer 0: GELU"></i><i class="c0" title="layer 1: GELU"></i><i class="c0" title="layer 2: GELU"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>GELU</td><td>3</td><td>3</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.language_model.layers</h3><div class="row"><span class="lab">Original Β· 36</span><div class="cells"><i class="c0" title="layer 0: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 1: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 2: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 3: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 4: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 5: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 6: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 7: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 8: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 9: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 10: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 11: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 12: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 13: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 14: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 15: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 16: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 17: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 18: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 19: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 20: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 21: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 22: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 23: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 24: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 25: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 26: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 27: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 28: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 29: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 30: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 31: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 32: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 33: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 34: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 35: Qwen3VLTextDecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 1: Qwen3VLTextDecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>Qwen3VLTextDecoderLayer</td><td>36</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.792286" title="model.language_model.layers: 6.9B (79.2%)"></i><i class="c1" style="flex:0.070985" title="model.language_model: 622.3M (7.1%)"></i><i class="c2" style="flex:0.070984" title="lm_head: 622.3M (7.1%)"></i><i class="c3" style="flex:0.046933" title="model.visual.blocks: 411.5M (4.7%)"></i><i class="c4" style="flex:0.013731" title="model.visual.deepstack_merger_list: 120.4M (1.4%)"></i><i class="c5" style="flex:0.005081" title="model.visual: 44.5M (0.5%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.003672" title="model.language_model.layers: 74.0K (0.4%)"></i><i class="c1" style="flex:0.482213" title="model.language_model: 9.7M (48.2%)"></i><i class="c2" style="flex:0.482209" title="lm_head: 9.7M (48.2%)"></i><i class="c3" style="flex:0.003320" title="model.visual.blocks: 66.9K (0.3%)"></i><i class="c4" style="flex:0.012311" title="model.visual.deepstack_merger_list: 248.3K (1.2%)"></i><i class="c5" style="flex:0.016275" title="model.visual: 328.2K (1.6%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.language_model.layers</td><td>6.9B</td><td>79.2%</td><td>74.0K</td><td>0.4%</td></tr><tr><td><i class="c1"></i>model.language_model</td><td>622.3M</td><td>7.1%</td><td>9.7M</td><td>48.2%</td></tr><tr><td><i class="c2"></i>lm_head</td><td>622.3M</td><td>7.1%</td><td>9.7M</td><td>48.2%</td></tr><tr><td><i class="c3"></i>model.visual.blocks</td><td>411.5M</td><td>4.7%</td><td>66.9K</td><td>0.3%</td></tr><tr><td><i class="c4"></i>model.visual.deepstack_merger_list</td><td>120.4M</td><td>1.4%</td><td>248.3K</td><td>1.2%</td></tr><tr><td><i class="c5"></i>model.visual</td><td>44.5M</td><td>0.5%</td><td>328.2K</td><td>1.6%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>model_type</td><td>cosmos3_omni</td><td>cosmos3_omni</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>attention style</td><td>GQA (32 heads, 8 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>model_type</td><td>qwen3_vl_text</td><td>qwen3_vl_text</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>vocab_size</td><td>151936</td><td>151936</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>max_position_embeddings</td><td>262144</td><td>262144</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>RoPE type</td><td>axial</td><td>axial</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>model_type</td><td>qwen3_vl_vision</td><td>qwen3_vl_vision</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>hidden_act</td><td>gelu_pytorch_tanh</td><td>gelu_pytorch_tanh</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>patch_size</td><td>16</td><td>16</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Cosmos3Omni omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>369</td><td>31</td></tr><tr class=""><td>Qwen3VLTextRMSNorm</td><td>145</td><td>9</td></tr><tr class=""><td>LayerNorm</td><td>58</td><td>8</td></tr><tr class=""><td>Qwen3VLTextAttention</td><td>36</td><td>2</td></tr><tr class=""><td>Qwen3VLTextDecoderLayer</td><td>36</td><td>2</td></tr><tr class=""><td>Qwen3VLTextMLP</td><td>36</td><td>2</td></tr><tr class=""><td>SiLUActivation</td><td>36</td><td>2</td></tr><tr class=""><td>GELUTanh</td><td>27</td><td>2</td></tr><tr class=""><td>Qwen3VLVisionAttention</td><td>27</td><td>2</td></tr><tr class=""><td>Qwen3VLVisionBlock</td><td>27</td><td>2</td></tr><tr class=""><td>Qwen3VLVisionMLP</td><td>27</td><td>2</td></tr><tr class=""><td>GELU</td><td>4</td><td>4</td></tr><tr class=""><td>Qwen3VLVisionPatchMerger</td><td>4</td><td>4</td></tr><tr class=""><td>ModuleList</td><td>3</td><td>3</td></tr><tr class=""><td>Embedding</td><td>2</td><td>2</td></tr><tr class=""><td>Conv3d</td><td>1</td><td>1</td></tr><tr class=""><td>ForConditionalGeneration</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen3VLTextModel</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen3VLTextRotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen3VLVisionModel</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen3VLVisionPatchEmbed</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen3VLVisionRotaryEmbedding</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">12 fields changed, 54 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>text_config.head_dim</td><td>128</td><td>16</td></tr><tr><td>text_config.hidden_size</td><td>4096</td><td>64</td></tr><tr><td>text_config.intermediate_size</td><td>12288</td><td>128</td></tr><tr><td>text_config.num_attention_heads</td><td>32</td><td>4</td></tr><tr><td>text_config.num_hidden_layers</td><td>36</td><td>2</td></tr><tr><td>text_config.num_key_value_heads</td><td>8</td><td>2</td></tr><tr><td>text_config.rope_parameters</td><td>{&quot;mrope_interleaved&quot;: true, &quot;mrope_section&quot;: [24, 20, 20], &quot;rope_type&quot;: &quot;defa…</td><td>{&quot;mrope_interleaved&quot;: true, &quot;mrope_section&quot;: [3, 3, 2], &quot;rope_theta&quot;: 5000000…</td></tr><tr><td>vision_config.depth</td><td>27</td><td>2</td></tr><tr><td>vision_config.hidden_size</td><td>1152</td><td>64</td></tr><tr><td>vision_config.intermediate_size</td><td>4304</td><td>128</td></tr><tr><td>vision_config.num_heads</td><td>16</td><td>4</td></tr><tr><td>vision_config.out_hidden_size</td><td>4096</td><td>64</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>nvidia/Cosmos3-Nano: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>nvidia/Cosmos3-Nano</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/Cosmos3-Nano-mic/tree/4c4a117fd8d6b13993505f86a24b15580e8b25d6" title="huggingface-mic/Cosmos3-Nano-mic@4c4a117fd8d6b13993505f86a24b15580e8b25d6">huggingface-mic/Cosmos3-Nano-mic@4c4a117fd8d6</a> ← <a href="https://huggingface.co/nvidia/Cosmos3-Nano/tree/e59a53c25979a090fa8706c9acc0c254a6e89b92" title="nvidia/Cosmos3-Nano@e59a53c25979a090fa8706c9acc0c254a6e89b92">nvidia/Cosmos3-Nano@e59a53c25979</a> Β· Cosmos3OmniForConditionalGeneration Β· random weights</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">8.8B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">20.2M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">3</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.visual.blocks</h3><div class="row"><span class="lab">Original Β· 27</span><div class="cells"><i class="c0" title="layer 0: Qwen3VLVisionBlock"></i><i class="c0" title="layer 1: Qwen3VLVisionBlock"></i><i class="c0" title="layer 2: Qwen3VLVisionBlock"></i><i class="c0" title="layer 3: Qwen3VLVisionBlock"></i><i class="c0" title="layer 4: Qwen3VLVisionBlock"></i><i class="c0" title="layer 5: Qwen3VLVisionBlock"></i><i class="c0" title="layer 6: Qwen3VLVisionBlock"></i><i class="c0" title="layer 7: Qwen3VLVisionBlock"></i><i class="c0" title="layer 8: Qwen3VLVisionBlock"></i><i class="c0" title="layer 9: Qwen3VLVisionBlock"></i><i class="c0" title="layer 10: Qwen3VLVisionBlock"></i><i class="c0" title="layer 11: Qwen3VLVisionBlock"></i><i class="c0" title="layer 12: Qwen3VLVisionBlock"></i><i class="c0" title="layer 13: Qwen3VLVisionBlock"></i><i class="c0" title="layer 14: Qwen3VLVisionBlock"></i><i class="c0" title="layer 15: Qwen3VLVisionBlock"></i><i class="c0" title="layer 16: Qwen3VLVisionBlock"></i><i class="c0" title="layer 17: Qwen3VLVisionBlock"></i><i class="c0" title="layer 18: Qwen3VLVisionBlock"></i><i class="c0" title="layer 19: Qwen3VLVisionBlock"></i><i class="c0" title="layer 20: Qwen3VLVisionBlock"></i><i class="c0" title="layer 21: Qwen3VLVisionBlock"></i><i class="c0" title="layer 22: Qwen3VLVisionBlock"></i><i class="c0" title="layer 23: Qwen3VLVisionBlock"></i><i class="c0" title="layer 24: Qwen3VLVisionBlock"></i><i class="c0" title="layer 25: Qwen3VLVisionBlock"></i><i class="c0" title="layer 26: Qwen3VLVisionBlock"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: Qwen3VLVisionBlock"></i><i class="c0" title="layer 1: Qwen3VLVisionBlock"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>Qwen3VLVisionBlock</td><td>27</td><td>2</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.visual.deepstack_merger_list</h3><div class="row"><span class="lab">Original Β· 3</span><div class="cells"><i class="c0" title="layer 0: GELU"></i><i class="c0" title="layer 1: GELU"></i><i class="c0" title="layer 2: GELU"></i></div></div><div class="row"><span class="lab">Fixture Β· 3</span><div class="cells"><i class="c0" title="layer 0: GELU"></i><i class="c0" title="layer 1: GELU"></i><i class="c0" title="layer 2: GELU"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>GELU</td><td>3</td><td>3</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.language_model.layers</h3><div class="row"><span class="lab">Original Β· 36</span><div class="cells"><i class="c0" title="layer 0: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 1: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 2: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 3: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 4: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 5: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 6: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 7: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 8: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 9: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 10: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 11: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 12: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 13: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 14: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 15: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 16: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 17: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 18: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 19: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 20: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 21: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 22: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 23: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 24: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 25: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 26: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 27: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 28: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 29: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 30: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 31: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 32: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 33: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 34: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 35: Qwen3VLTextDecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: Qwen3VLTextDecoderLayer"></i><i class="c0" title="layer 1: Qwen3VLTextDecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>Qwen3VLTextDecoderLayer</td><td>36</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.792286" title="model.language_model.layers: 6.9B (79.2%)"></i><i class="c1" style="flex:0.070985" title="model.language_model: 622.3M (7.1%)"></i><i class="c2" style="flex:0.070984" title="lm_head: 622.3M (7.1%)"></i><i class="c3" style="flex:0.046933" title="model.visual.blocks: 411.5M (4.7%)"></i><i class="c4" style="flex:0.013731" title="model.visual.deepstack_merger_list: 120.4M (1.4%)"></i><i class="c5" style="flex:0.005081" title="model.visual: 44.5M (0.5%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.003672" title="model.language_model.layers: 74.0K (0.4%)"></i><i class="c1" style="flex:0.482213" title="model.language_model: 9.7M (48.2%)"></i><i class="c2" style="flex:0.482209" title="lm_head: 9.7M (48.2%)"></i><i class="c3" style="flex:0.003320" title="model.visual.blocks: 66.9K (0.3%)"></i><i class="c4" style="flex:0.012311" title="model.visual.deepstack_merger_list: 248.3K (1.2%)"></i><i class="c5" style="flex:0.016275" title="model.visual: 328.2K (1.6%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.language_model.layers</td><td>6.9B</td><td>79.2%</td><td>74.0K</td><td>0.4%</td></tr><tr><td><i class="c1"></i>model.language_model</td><td>622.3M</td><td>7.1%</td><td>9.7M</td><td>48.2%</td></tr><tr><td><i class="c2"></i>lm_head</td><td>622.3M</td><td>7.1%</td><td>9.7M</td><td>48.2%</td></tr><tr><td><i class="c3"></i>model.visual.blocks</td><td>411.5M</td><td>4.7%</td><td>66.9K</td><td>0.3%</td></tr><tr><td><i class="c4"></i>model.visual.deepstack_merger_list</td><td>120.4M</td><td>1.4%</td><td>248.3K</td><td>1.2%</td></tr><tr><td><i class="c5"></i>model.visual</td><td>44.5M</td><td>0.5%</td><td>328.2K</td><td>1.6%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>model_type</td><td>cosmos3_omni</td><td>cosmos3_omni</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>attention style</td><td>GQA (32 heads, 8 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>model_type</td><td>qwen3_vl_text</td><td>qwen3_vl_text</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>vocab_size</td><td>151936</td><td>151936</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>max_position_embeddings</td><td>262144</td><td>262144</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>RoPE type</td><td>axial</td><td>axial</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>model_type</td><td>qwen3_vl_vision</td><td>qwen3_vl_vision</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>hidden_act</td><td>gelu_pytorch_tanh</td><td>gelu_pytorch_tanh</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>patch_size</td><td>16</td><td>16</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Cosmos3Omni omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>369</td><td>31</td></tr><tr class=""><td>Qwen3VLTextRMSNorm</td><td>145</td><td>9</td></tr><tr class=""><td>LayerNorm</td><td>58</td><td>8</td></tr><tr class=""><td>Qwen3VLTextAttention</td><td>36</td><td>2</td></tr><tr class=""><td>Qwen3VLTextDecoderLayer</td><td>36</td><td>2</td></tr><tr class=""><td>Qwen3VLTextMLP</td><td>36</td><td>2</td></tr><tr class=""><td>SiLUActivation</td><td>36</td><td>2</td></tr><tr class=""><td>GELUTanh</td><td>27</td><td>2</td></tr><tr class=""><td>Qwen3VLVisionAttention</td><td>27</td><td>2</td></tr><tr class=""><td>Qwen3VLVisionBlock</td><td>27</td><td>2</td></tr><tr class=""><td>Qwen3VLVisionMLP</td><td>27</td><td>2</td></tr><tr class=""><td>GELU</td><td>4</td><td>4</td></tr><tr class=""><td>Qwen3VLVisionPatchMerger</td><td>4</td><td>4</td></tr><tr class=""><td>ModuleList</td><td>3</td><td>3</td></tr><tr class=""><td>Embedding</td><td>2</td><td>2</td></tr><tr class=""><td>Conv3d</td><td>1</td><td>1</td></tr><tr class=""><td>ForConditionalGeneration</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen3VLTextModel</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen3VLTextRotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen3VLVisionModel</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen3VLVisionPatchEmbed</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen3VLVisionRotaryEmbedding</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">12 fields changed, 54 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>text_config.head_dim</td><td>128</td><td>16</td></tr><tr><td>text_config.hidden_size</td><td>4096</td><td>64</td></tr><tr><td>text_config.intermediate_size</td><td>12288</td><td>128</td></tr><tr><td>text_config.num_attention_heads</td><td>32</td><td>4</td></tr><tr><td>text_config.num_hidden_layers</td><td>36</td><td>2</td></tr><tr><td>text_config.num_key_value_heads</td><td>8</td><td>2</td></tr><tr><td>text_config.rope_parameters</td><td>{&quot;mrope_interleaved&quot;: true, &quot;mrope_section&quot;: [24, 20, 20], &quot;rope_type&quot;: &quot;defa…</td><td>{&quot;mrope_interleaved&quot;: true, &quot;mrope_section&quot;: [3, 3, 2], &quot;rope_theta&quot;: 5000000…</td></tr><tr><td>vision_config.depth</td><td>27</td><td>2</td></tr><tr><td>vision_config.hidden_size</td><td>1152</td><td>64</td></tr><tr><td>vision_config.intermediate_size</td><td>4304</td><td>128</td></tr><tr><td>vision_config.num_heads</td><td>16</td><td>4</td></tr><tr><td>vision_config.out_hidden_size</td><td>4096</td><td>64</td></tr></table>
36
  </main></body></html>
shrink/Cosmos3-Nano-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/Cosmos3-Nano-mic@4c4a117fd8d6b13993505f86a24b15580e8b25d6", "findings": []}
shrink/DeepSeek-Coder-V2-Lite-Instruct-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/DeepSeek-Coder-V2-Lite-Instruct-mic@90601ffd1320fd0db167a3ade1c6cd0ed35c2229 ← deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct@e434a23f91ba5b4923cf6c9d9a238eb4a08e3a11 Β· DeepseekV2ForCausalLM Β· random weights</p>
33
- <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">15.7B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">13.2M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 27</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 2: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 3: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 4: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 5: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 6: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 7: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 8: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 9: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 10: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 11: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 12: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 13: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 14: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 15: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 16: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 17: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 18: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 19: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 20: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 21: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 22: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 23: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 24: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 25: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 26: Experts, Moe, TopkRouter"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, Moe, TopkRouter"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>no Experts</td><td>1</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>Experts, Moe, TopkRouter</td><td>26</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.973296" title="model.layers: 15.3B (97.3%)"></i><i class="c1" style="flex:0.013352" title="lm_head: 209.7M (1.3%)"></i><i class="c2" style="flex:0.013352" title="model.embed_tokens: 209.7M (1.3%)"></i><i class="c3" style="flex:0.000000" title="model.norm: 2.0K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.006792" title="model.layers: 89.6K (0.7%)"></i><i class="c1" style="flex:0.496602" title="lm_head: 6.6M (49.7%)"></i><i class="c2" style="flex:0.496602" title="model.embed_tokens: 6.6M (49.7%)"></i><i class="c3" style="flex:0.000005" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>15.3B</td><td>97.3%</td><td>89.6K</td><td>0.7%</td></tr><tr><td><i class="c1"></i>lm_head</td><td>209.7M</td><td>1.3%</td><td>6.6M</td><td>49.7%</td></tr><tr><td><i class="c2"></i>model.embed_tokens</td><td>209.7M</td><td>1.3%</td><td>6.6M</td><td>49.7%</td></tr><tr><td><i class="c3"></i>model.norm</td><td>2.0K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>MHA (16 heads, 16 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-6 of 64 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>yarn</td><td>yarn</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>deepseek_v2</td><td>deepseek_v2</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>102400</td><td>102400</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>163840</td><td>163840</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix DeepseekV2 omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>190</td><td>15</td></tr><tr class=""><td>RMSNorm</td><td>82</td><td>7</td></tr><tr class=""><td>SiLUActivation</td><td>53</td><td>3</td></tr><tr class=""><td>Attention</td><td>27</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>27</td><td>2</td></tr><tr class=""><td>MLP</td><td>27</td><td>2</td></tr><tr class=""><td>Experts</td><td>26</td><td>1</td></tr><tr class=""><td>Moe</td><td>26</td><td>1</td></tr><tr class=""><td>TopkRouter</td><td>26</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">13 fields changed, 39 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>head_dim</td><td>64</td><td>8</td></tr><tr><td>hidden_size</td><td>2048</td><td>64</td></tr><tr><td>intermediate_size</td><td>10944</td><td>128</td></tr><tr><td>kv_lora_rank</td><td>512</td><td>16</td></tr><tr><td>moe_intermediate_size</td><td>1408</td><td>32</td></tr><tr><td>n_routed_experts</td><td>64</td><td>4</td></tr><tr><td>num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>num_experts_per_tok</td><td>6</td><td>2</td></tr><tr><td>num_hidden_layers</td><td>27</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>16</td><td>4</td></tr><tr><td>qk_nope_head_dim</td><td>128</td><td>16</td></tr><tr><td>qk_rope_head_dim</td><td>64</td><td>8</td></tr><tr><td>v_head_dim</td><td>128</td><td>16</td></tr></table>
 
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/DeepSeek-Coder-V2-Lite-Instruct-mic/tree/90601ffd1320fd0db167a3ade1c6cd0ed35c2229" title="huggingface-mic/DeepSeek-Coder-V2-Lite-Instruct-mic@90601ffd1320fd0db167a3ade1c6cd0ed35c2229">huggingface-mic/DeepSeek-Coder-V2-Lite-Instruct-mic@90601ffd1320</a> ← <a href="https://huggingface.co/deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct/tree/e434a23f91ba5b4923cf6c9d9a238eb4a08e3a11" title="deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct@e434a23f91ba5b4923cf6c9d9a238eb4a08e3a11">deepseek-ai/DeepSeek-Coder-V2-Lite-Instruct@e434a23f91ba</a> Β· DeepseekV2ForCausalLM Β· random weights</p>
35
+ <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">15.7B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">13.2M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 27</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 2: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 3: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 4: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 5: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 6: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 7: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 8: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 9: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 10: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 11: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 12: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 13: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 14: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 15: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 16: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 17: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 18: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 19: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 20: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 21: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 22: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 23: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 24: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 25: Experts, Moe, TopkRouter"></i><i class="c1" title="layer 26: Experts, Moe, TopkRouter"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, Moe, TopkRouter"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>no Experts</td><td>1</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>Experts, Moe, TopkRouter</td><td>26</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.973296" title="model.layers: 15.3B (97.3%)"></i><i class="c1" style="flex:0.013352" title="model.embed_tokens: 209.7M (1.3%)"></i><i class="c2" style="flex:0.013352" title="lm_head: 209.7M (1.3%)"></i><i class="c3" style="flex:0.000000" title="model.norm: 2.0K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.006792" title="model.layers: 89.6K (0.7%)"></i><i class="c1" style="flex:0.496602" title="model.embed_tokens: 6.6M (49.7%)"></i><i class="c2" style="flex:0.496602" title="lm_head: 6.6M (49.7%)"></i><i class="c3" style="flex:0.000005" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>15.3B</td><td>97.3%</td><td>89.6K</td><td>0.7%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>209.7M</td><td>1.3%</td><td>6.6M</td><td>49.7%</td></tr><tr><td><i class="c2"></i>lm_head</td><td>209.7M</td><td>1.3%</td><td>6.6M</td><td>49.7%</td></tr><tr><td><i class="c3"></i>model.norm</td><td>2.0K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>MHA (16 heads, 16 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-6 of 64 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>yarn</td><td>yarn</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>deepseek_v2</td><td>deepseek_v2</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>102400</td><td>102400</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>163840</td><td>163840</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix DeepseekV2 omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>190</td><td>15</td></tr><tr class=""><td>RMSNorm</td><td>82</td><td>7</td></tr><tr class=""><td>SiLUActivation</td><td>53</td><td>3</td></tr><tr class=""><td>Attention</td><td>27</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>27</td><td>2</td></tr><tr class=""><td>MLP</td><td>27</td><td>2</td></tr><tr class=""><td>Experts</td><td>26</td><td>1</td></tr><tr class=""><td>Moe</td><td>26</td><td>1</td></tr><tr class=""><td>TopkRouter</td><td>26</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">13 fields changed, 39 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>head_dim</td><td>64</td><td>8</td></tr><tr><td>hidden_size</td><td>2048</td><td>64</td></tr><tr><td>intermediate_size</td><td>10944</td><td>128</td></tr><tr><td>kv_lora_rank</td><td>512</td><td>16</td></tr><tr><td>moe_intermediate_size</td><td>1408</td><td>32</td></tr><tr><td>n_routed_experts</td><td>64</td><td>4</td></tr><tr><td>num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>num_experts_per_tok</td><td>6</td><td>2</td></tr><tr><td>num_hidden_layers</td><td>27</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>16</td><td>4</td></tr><tr><td>qk_nope_head_dim</td><td>128</td><td>16</td></tr><tr><td>qk_rope_head_dim</td><td>64</td><td>8</td></tr><tr><td>v_head_dim</td><td>128</td><td>16</td></tr></table>
36
  </main></body></html>
shrink/DeepSeek-Coder-V2-Lite-Instruct-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/DeepSeek-Coder-V2-Lite-Instruct-mic@90601ffd1320fd0db167a3ade1c6cd0ed35c2229", "findings": []}
shrink/DeepSeek-R1-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/DeepSeek-R1-mic@89eddb3d8f9419e63b667eb7d5244fec8c768ed9 ← deepseek-ai/DeepSeek-R1@56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad Β· DeepseekV3ForCausalLM Β· random weights Β· built before the shrink rule (no recorded recipe)</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">671.0B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">16.6M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 61</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c0" title="layer 1: no Experts"></i><i class="c0" title="layer 2: no Experts"></i><i class="c1" title="layer 3: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 4: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 5: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 6: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 7: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 8: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 9: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 10: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 11: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 12: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 13: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 14: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 15: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 16: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 17: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 18: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 19: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 20: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 21: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 22: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 23: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 24: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 25: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 26: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 27: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 28: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 29: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 30: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 31: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 32: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 33: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 34: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 35: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 36: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 37: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 38: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 39: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 40: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 41: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 42: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 43: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 44: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 45: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 46: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 47: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 48: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 49: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 50: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 51: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 52: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 53: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 54: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 55: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 56: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 57: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 58: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 59: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 60: Experts, MoE, TopkRouter"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, MoE, TopkRouter"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>no Experts</td><td>3</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>Experts, MoE, TopkRouter</td><td>58</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.997238" title="model.layers: 669.2B (99.7%)"></i><i class="c1" style="flex:0.001381" title="model.embed_tokens: 926.7M (0.1%)"></i><i class="c2" style="flex:0.001381" title="lm_head: 926.7M (0.1%)"></i><i class="c3" style="flex:0.000000" title="model.norm: 7.2K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.004593" title="model.layers: 76.4K (0.5%)"></i><i class="c1" style="flex:0.497702" title="model.embed_tokens: 8.3M (49.8%)"></i><i class="c2" style="flex:0.497702" title="lm_head: 8.3M (49.8%)"></i><i class="c3" style="flex:0.000004" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>669.2B</td><td>99.7%</td><td>76.4K</td><td>0.5%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>926.7M</td><td>0.1%</td><td>8.3M</td><td>49.8%</td></tr><tr><td><i class="c2"></i>lm_head</td><td>926.7M</td><td>0.1%</td><td>8.3M</td><td>49.8%</td></tr><tr><td><i class="c3"></i>model.norm</td><td>7.2K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>MHA (128 heads, 128 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-8 of 256 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>yarn</td><td>yarn</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>deepseek_v3</td><td>deepseek_v3</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>129280</td><td>129280</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>163840</td><td>163840</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix DeepseekV3 omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>489</td><td>17</td></tr><tr class=""><td>RMSNorm</td><td>245</td><td>9</td></tr><tr class=""><td>SiLUActivation</td><td>119</td><td>3</td></tr><tr class=""><td>Attention</td><td>61</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>61</td><td>2</td></tr><tr class=""><td>MLP</td><td>61</td><td>2</td></tr><tr class=""><td>Experts</td><td>58</td><td>1</td></tr><tr class=""><td>MoE</td><td>58</td><td>1</td></tr><tr class=""><td>TopkRouter</td><td>58</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">18 fields changed, 35 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>first_k_dense_replace</td><td>3</td><td>1</td></tr><tr><td>head_dim</td><td>64</td><td>8</td></tr><tr><td>hidden_size</td><td>7168</td><td>64</td></tr><tr><td>intermediate_size</td><td>18432</td><td>128</td></tr><tr><td>kv_lora_rank</td><td>512</td><td>16</td></tr><tr><td>moe_intermediate_size</td><td>2048</td><td>32</td></tr><tr><td>n_group</td><td>8</td><td>1</td></tr><tr><td>n_routed_experts</td><td>256</td><td>4</td></tr><tr><td>num_attention_heads</td><td>128</td><td>4</td></tr><tr><td>num_experts_per_tok</td><td>8</td><td>2</td></tr><tr><td>num_hidden_layers</td><td>61</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>128</td><td>4</td></tr><tr><td>q_lora_rank</td><td>1536</td><td>16</td></tr><tr><td>qk_nope_head_dim</td><td>128</td><td>16</td></tr><tr><td>qk_rope_head_dim</td><td>64</td><td>8</td></tr><tr><td>quantization_config</td><td>{&quot;activation_scheme&quot;: &quot;dynamic&quot;, &quot;fmt&quot;: &quot;e4m3&quot;, &quot;quant_method&quot;: &quot;fp8&quot;, &quot;weigh…</td><td>β€”</td></tr><tr><td>topk_group</td><td>4</td><td>1</td></tr><tr><td>v_head_dim</td><td>128</td><td>16</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>deepseek-ai/DeepSeek-R1: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>deepseek-ai/DeepSeek-R1</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/DeepSeek-R1-mic/tree/89eddb3d8f9419e63b667eb7d5244fec8c768ed9" title="huggingface-mic/DeepSeek-R1-mic@89eddb3d8f9419e63b667eb7d5244fec8c768ed9">huggingface-mic/DeepSeek-R1-mic@89eddb3d8f94</a> ← <a href="https://huggingface.co/deepseek-ai/DeepSeek-R1/tree/56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad" title="deepseek-ai/DeepSeek-R1@56d4cbbb4d29f4355bab4b9a39ccb717a14ad5ad">deepseek-ai/DeepSeek-R1@56d4cbbb4d29</a> Β· DeepseekV3ForCausalLM Β· random weights Β· built before the shrink rule (no recorded recipe)</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">671.0B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">16.6M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 61</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c0" title="layer 1: no Experts"></i><i class="c0" title="layer 2: no Experts"></i><i class="c1" title="layer 3: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 4: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 5: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 6: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 7: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 8: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 9: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 10: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 11: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 12: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 13: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 14: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 15: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 16: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 17: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 18: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 19: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 20: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 21: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 22: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 23: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 24: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 25: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 26: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 27: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 28: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 29: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 30: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 31: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 32: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 33: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 34: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 35: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 36: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 37: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 38: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 39: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 40: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 41: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 42: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 43: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 44: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 45: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 46: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 47: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 48: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 49: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 50: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 51: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 52: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 53: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 54: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 55: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 56: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 57: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 58: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 59: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 60: Experts, MoE, TopkRouter"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, MoE, TopkRouter"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>no Experts</td><td>3</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>Experts, MoE, TopkRouter</td><td>58</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.997238" title="model.layers: 669.2B (99.7%)"></i><i class="c1" style="flex:0.001381" title="model.embed_tokens: 926.7M (0.1%)"></i><i class="c2" style="flex:0.001381" title="lm_head: 926.7M (0.1%)"></i><i class="c3" style="flex:0.000000" title="model.norm: 7.2K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.004593" title="model.layers: 76.4K (0.5%)"></i><i class="c1" style="flex:0.497702" title="model.embed_tokens: 8.3M (49.8%)"></i><i class="c2" style="flex:0.497702" title="lm_head: 8.3M (49.8%)"></i><i class="c3" style="flex:0.000004" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>669.2B</td><td>99.7%</td><td>76.4K</td><td>0.5%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>926.7M</td><td>0.1%</td><td>8.3M</td><td>49.8%</td></tr><tr><td><i class="c2"></i>lm_head</td><td>926.7M</td><td>0.1%</td><td>8.3M</td><td>49.8%</td></tr><tr><td><i class="c3"></i>model.norm</td><td>7.2K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>MHA (128 heads, 128 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-8 of 256 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>yarn</td><td>yarn</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>deepseek_v3</td><td>deepseek_v3</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>129280</td><td>129280</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>163840</td><td>163840</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix DeepseekV3 omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>489</td><td>17</td></tr><tr class=""><td>RMSNorm</td><td>245</td><td>9</td></tr><tr class=""><td>SiLUActivation</td><td>119</td><td>3</td></tr><tr class=""><td>Attention</td><td>61</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>61</td><td>2</td></tr><tr class=""><td>MLP</td><td>61</td><td>2</td></tr><tr class=""><td>Experts</td><td>58</td><td>1</td></tr><tr class=""><td>MoE</td><td>58</td><td>1</td></tr><tr class=""><td>TopkRouter</td><td>58</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">18 fields changed, 35 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>first_k_dense_replace</td><td>3</td><td>1</td></tr><tr><td>head_dim</td><td>64</td><td>8</td></tr><tr><td>hidden_size</td><td>7168</td><td>64</td></tr><tr><td>intermediate_size</td><td>18432</td><td>128</td></tr><tr><td>kv_lora_rank</td><td>512</td><td>16</td></tr><tr><td>moe_intermediate_size</td><td>2048</td><td>32</td></tr><tr><td>n_group</td><td>8</td><td>1</td></tr><tr><td>n_routed_experts</td><td>256</td><td>4</td></tr><tr><td>num_attention_heads</td><td>128</td><td>4</td></tr><tr><td>num_experts_per_tok</td><td>8</td><td>2</td></tr><tr><td>num_hidden_layers</td><td>61</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>128</td><td>4</td></tr><tr><td>q_lora_rank</td><td>1536</td><td>16</td></tr><tr><td>qk_nope_head_dim</td><td>128</td><td>16</td></tr><tr><td>qk_rope_head_dim</td><td>64</td><td>8</td></tr><tr><td>quantization_config</td><td>{&quot;activation_scheme&quot;: &quot;dynamic&quot;, &quot;fmt&quot;: &quot;e4m3&quot;, &quot;quant_method&quot;: &quot;fp8&quot;, &quot;weigh…</td><td>β€”</td></tr><tr><td>topk_group</td><td>4</td><td>1</td></tr><tr><td>v_head_dim</td><td>128</td><td>16</td></tr></table>
36
  </main></body></html>
shrink/DeepSeek-R1-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/DeepSeek-R1-mic@89eddb3d8f9419e63b667eb7d5244fec8c768ed9", "findings": []}
shrink/DeepSeek-V3.2-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/DeepSeek-V3.2-mic@2156444e97b761624604df50de0ac677d402f289 ← deepseek-ai/DeepSeek-V3.2@a7e62ac04ecb2c0a54d736dc46601c5606cf10a6 Β· DeepseekV32ForCausalLM Β· random weights Β· built before the shrink rule (no recorded recipe)</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">671.9B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">16.9M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 61</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c0" title="layer 1: no Experts"></i><i class="c0" title="layer 2: no Experts"></i><i class="c1" title="layer 3: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 4: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 5: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 6: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 7: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 8: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 9: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 10: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 11: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 12: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 13: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 14: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 15: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 16: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 17: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 18: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 19: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 20: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 21: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 22: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 23: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 24: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 25: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 26: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 27: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 28: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 29: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 30: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 31: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 32: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 33: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 34: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 35: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 36: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 37: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 38: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 39: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 40: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 41: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 42: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 43: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 44: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 45: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 46: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 47: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 48: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 49: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 50: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 51: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 52: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 53: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 54: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 55: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 56: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 57: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 58: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 59: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 60: Experts, MoE, TopkRouter"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, MoE, TopkRouter"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>no Experts</td><td>3</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>Experts, MoE, TopkRouter</td><td>58</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.997242" title="model.layers: 670.0B (99.7%)"></i><i class="c1" style="flex:0.001379" title="model.embed_tokens: 926.7M (0.1%)"></i><i class="c2" style="flex:0.001379" title="lm_head: 926.7M (0.1%)"></i><i class="c3" style="flex:0.000000" title="model.norm: 7.2K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.021499" title="model.layers: 363.6K (2.1%)"></i><i class="c1" style="flex:0.489248" title="model.embed_tokens: 8.3M (48.9%)"></i><i class="c2" style="flex:0.489248" title="lm_head: 8.3M (48.9%)"></i><i class="c3" style="flex:0.000004" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>670.0B</td><td>99.7%</td><td>363.6K</td><td>2.1%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>926.7M</td><td>0.1%</td><td>8.3M</td><td>48.9%</td></tr><tr><td><i class="c2"></i>lm_head</td><td>926.7M</td><td>0.1%</td><td>8.3M</td><td>48.9%</td></tr><tr><td><i class="c3"></i>model.norm</td><td>7.2K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>MHA (128 heads, 128 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-8 of 256 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>yarn</td><td>yarn</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>deepseek_v32</td><td>deepseek_v32</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>129280</td><td>129280</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>163840</td><td>163840</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix DeepseekV32 omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>672</td><td>23</td></tr><tr class=""><td>RMSNorm</td><td>245</td><td>9</td></tr><tr class=""><td>SiLUActivation</td><td>119</td><td>3</td></tr><tr class=""><td>Attention</td><td>61</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>61</td><td>2</td></tr><tr class=""><td>Indexer</td><td>61</td><td>2</td></tr><tr class=""><td>MLP</td><td>61</td><td>2</td></tr><tr class=""><td>LayerNorm</td><td>61</td><td>2</td></tr><tr class=""><td>Experts</td><td>58</td><td>1</td></tr><tr class=""><td>MoE</td><td>58</td><td>1</td></tr><tr class=""><td>TopkRouter</td><td>58</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">20 fields changed, 36 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>first_k_dense_replace</td><td>3</td><td>1</td></tr><tr><td>head_dim</td><td>64</td><td>8</td></tr><tr><td>hidden_size</td><td>7168</td><td>64</td></tr><tr><td>intermediate_size</td><td>18432</td><td>128</td></tr><tr><td>kv_lora_rank</td><td>512</td><td>16</td></tr><tr><td>layer_types</td><td>61 entries: 61Γ— indexed_attention</td><td>[&quot;indexed_attention&quot;, &quot;indexed_attention&quot;]</td></tr><tr><td>mlp_layer_types</td><td>61 entries: 58Γ— sparse, 3Γ— dense</td><td>[&quot;dense&quot;, &quot;sparse&quot;]</td></tr><tr><td>moe_intermediate_size</td><td>2048</td><td>32</td></tr><tr><td>n_group</td><td>8</td><td>1</td></tr><tr><td>n_routed_experts</td><td>256</td><td>4</td></tr><tr><td>num_attention_heads</td><td>128</td><td>4</td></tr><tr><td>num_experts_per_tok</td><td>8</td><td>2</td></tr><tr><td>num_hidden_layers</td><td>61</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>128</td><td>4</td></tr><tr><td>q_lora_rank</td><td>1536</td><td>16</td></tr><tr><td>qk_nope_head_dim</td><td>128</td><td>16</td></tr><tr><td>qk_rope_head_dim</td><td>64</td><td>8</td></tr><tr><td>quantization_config</td><td>{&quot;activation_scheme&quot;: &quot;dynamic&quot;, &quot;fmt&quot;: &quot;e4m3&quot;, &quot;quant_method&quot;: &quot;fp8&quot;, &quot;scale…</td><td>β€”</td></tr><tr><td>topk_group</td><td>4</td><td>1</td></tr><tr><td>v_head_dim</td><td>128</td><td>16</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>deepseek-ai/DeepSeek-V3.2: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>deepseek-ai/DeepSeek-V3.2</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/DeepSeek-V3.2-mic/tree/2156444e97b761624604df50de0ac677d402f289" title="huggingface-mic/DeepSeek-V3.2-mic@2156444e97b761624604df50de0ac677d402f289">huggingface-mic/DeepSeek-V3.2-mic@2156444e97b7</a> ← <a href="https://huggingface.co/deepseek-ai/DeepSeek-V3.2/tree/a7e62ac04ecb2c0a54d736dc46601c5606cf10a6" title="deepseek-ai/DeepSeek-V3.2@a7e62ac04ecb2c0a54d736dc46601c5606cf10a6">deepseek-ai/DeepSeek-V3.2@a7e62ac04ecb</a> Β· DeepseekV32ForCausalLM Β· random weights Β· built before the shrink rule (no recorded recipe)</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">671.9B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">16.9M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 61</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c0" title="layer 1: no Experts"></i><i class="c0" title="layer 2: no Experts"></i><i class="c1" title="layer 3: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 4: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 5: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 6: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 7: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 8: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 9: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 10: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 11: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 12: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 13: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 14: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 15: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 16: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 17: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 18: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 19: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 20: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 21: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 22: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 23: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 24: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 25: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 26: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 27: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 28: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 29: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 30: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 31: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 32: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 33: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 34: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 35: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 36: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 37: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 38: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 39: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 40: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 41: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 42: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 43: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 44: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 45: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 46: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 47: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 48: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 49: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 50: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 51: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 52: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 53: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 54: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 55: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 56: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 57: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 58: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 59: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 60: Experts, MoE, TopkRouter"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, MoE, TopkRouter"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>no Experts</td><td>3</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>Experts, MoE, TopkRouter</td><td>58</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.997242" title="model.layers: 670.0B (99.7%)"></i><i class="c1" style="flex:0.001379" title="model.embed_tokens: 926.7M (0.1%)"></i><i class="c2" style="flex:0.001379" title="lm_head: 926.7M (0.1%)"></i><i class="c3" style="flex:0.000000" title="model.norm: 7.2K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.021499" title="model.layers: 363.6K (2.1%)"></i><i class="c1" style="flex:0.489248" title="model.embed_tokens: 8.3M (48.9%)"></i><i class="c2" style="flex:0.489248" title="lm_head: 8.3M (48.9%)"></i><i class="c3" style="flex:0.000004" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>670.0B</td><td>99.7%</td><td>363.6K</td><td>2.1%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>926.7M</td><td>0.1%</td><td>8.3M</td><td>48.9%</td></tr><tr><td><i class="c2"></i>lm_head</td><td>926.7M</td><td>0.1%</td><td>8.3M</td><td>48.9%</td></tr><tr><td><i class="c3"></i>model.norm</td><td>7.2K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>MHA (128 heads, 128 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-8 of 256 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>yarn</td><td>yarn</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>deepseek_v32</td><td>deepseek_v32</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>129280</td><td>129280</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>163840</td><td>163840</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix DeepseekV32 omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>672</td><td>23</td></tr><tr class=""><td>RMSNorm</td><td>245</td><td>9</td></tr><tr class=""><td>SiLUActivation</td><td>119</td><td>3</td></tr><tr class=""><td>Attention</td><td>61</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>61</td><td>2</td></tr><tr class=""><td>Indexer</td><td>61</td><td>2</td></tr><tr class=""><td>MLP</td><td>61</td><td>2</td></tr><tr class=""><td>LayerNorm</td><td>61</td><td>2</td></tr><tr class=""><td>Experts</td><td>58</td><td>1</td></tr><tr class=""><td>MoE</td><td>58</td><td>1</td></tr><tr class=""><td>TopkRouter</td><td>58</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">20 fields changed, 36 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>first_k_dense_replace</td><td>3</td><td>1</td></tr><tr><td>head_dim</td><td>64</td><td>8</td></tr><tr><td>hidden_size</td><td>7168</td><td>64</td></tr><tr><td>intermediate_size</td><td>18432</td><td>128</td></tr><tr><td>kv_lora_rank</td><td>512</td><td>16</td></tr><tr><td>layer_types</td><td>61 entries: 61Γ— indexed_attention</td><td>[&quot;indexed_attention&quot;, &quot;indexed_attention&quot;]</td></tr><tr><td>mlp_layer_types</td><td>61 entries: 58Γ— sparse, 3Γ— dense</td><td>[&quot;dense&quot;, &quot;sparse&quot;]</td></tr><tr><td>moe_intermediate_size</td><td>2048</td><td>32</td></tr><tr><td>n_group</td><td>8</td><td>1</td></tr><tr><td>n_routed_experts</td><td>256</td><td>4</td></tr><tr><td>num_attention_heads</td><td>128</td><td>4</td></tr><tr><td>num_experts_per_tok</td><td>8</td><td>2</td></tr><tr><td>num_hidden_layers</td><td>61</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>128</td><td>4</td></tr><tr><td>q_lora_rank</td><td>1536</td><td>16</td></tr><tr><td>qk_nope_head_dim</td><td>128</td><td>16</td></tr><tr><td>qk_rope_head_dim</td><td>64</td><td>8</td></tr><tr><td>quantization_config</td><td>{&quot;activation_scheme&quot;: &quot;dynamic&quot;, &quot;fmt&quot;: &quot;e4m3&quot;, &quot;quant_method&quot;: &quot;fp8&quot;, &quot;scale…</td><td>β€”</td></tr><tr><td>topk_group</td><td>4</td><td>1</td></tr><tr><td>v_head_dim</td><td>128</td><td>16</td></tr></table>
36
  </main></body></html>
shrink/DeepSeek-V3.2-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/DeepSeek-V3.2-mic@2156444e97b761624604df50de0ac677d402f289", "findings": []}
shrink/DeepSeek-V4-Flash-0731-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/DeepSeek-V4-Flash-0731-mic@7c36fb38ddaf52e554d784d9e6a2a7ed435b8311 ← deepseek-ai/DeepSeek-V4-Flash-0731@7872f01b1d1fe23eabc4c98b48bffcef5a386062 Β· DeepseekV4ForCausalLM Β· random weights</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">284.3B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">20.1M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 43</span><div class="cells"><i class="c0" title="layer 0: sliding_attention, HashRouter"></i><i class="c0" title="layer 1: sliding_attention, HashRouter"></i><i class="c1" title="layer 2: compressed_sparse_attention, CSACompressor, HashRouter …"></i><i class="c2" title="layer 3: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 4: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 5: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 6: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 7: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 8: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 9: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 10: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 11: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 12: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 13: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 14: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 15: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 16: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 17: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 18: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 19: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 20: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 21: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 22: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 23: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 24: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 25: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 26: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 27: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 28: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 29: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 30: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 31: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 32: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 33: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 34: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 35: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 36: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 37: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 38: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 39: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 40: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 41: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 42: compressed_sparse_attention, CSACompressor, Indexer …"></i></div></div><div class="row"><span class="lab">Fixture Β· 5</span><div class="cells"><i class="c0" title="layer 0: sliding_attention, HashRouter"></i><i class="c0" title="layer 1: sliding_attention, HashRouter"></i><i class="c1" title="layer 2: compressed_sparse_attention, CSACompressor, HashRouter …"></i><i class="c2" title="layer 3: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 4: compressed_sparse_attention, CSACompressor, Indexer …"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>sliding_attention, HashRouter</td><td>2</td><td>2</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>compressed_sparse_attention, CSACompressor, HashRouter …</td><td>1</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c2"></i>heavily_compressed_attention, HCACompressor, RotaryEmbedding …</td><td>20</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c3"></i>compressed_sparse_attention, CSACompressor, Indexer …</td><td>20</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.996275" title="model.layers: 283.3B (99.6%)"></i><i class="c1" style="flex:0.001862" title="lm_head: 529.5M (0.2%)"></i><i class="c2" style="flex:0.001862" title="model.embed_tokens: 529.5M (0.2%)"></i><i class="c3" style="flex:0.000000" title="model.hc_head: 65.5K (0.0%)"></i><i class="c4" style="flex:0.000000" title="model.norm: 4.1K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.175915" title="model.layers: 3.5M (17.6%)"></i><i class="c1" style="flex:0.412015" title="lm_head: 8.3M (41.2%)"></i><i class="c2" style="flex:0.412015" title="model.embed_tokens: 8.3M (41.2%)"></i><i class="c3" style="flex:0.000051" title="model.hc_head: 1.0K (0.0%)"></i><i class="c4" style="flex:0.000003" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>283.3B</td><td>99.6%</td><td>3.5M</td><td>17.6%</td></tr><tr><td><i class="c1"></i>lm_head</td><td>529.5M</td><td>0.2%</td><td>8.3M</td><td>41.2%</td></tr><tr><td><i class="c2"></i>model.embed_tokens</td><td>529.5M</td><td>0.2%</td><td>8.3M</td><td>41.2%</td></tr><tr><td><i class="c3"></i>model.hc_head</td><td>65.5K</td><td>0.0%</td><td>1.0K</td><td>0.0%</td></tr><tr><td><i class="c4"></i>model.norm</td><td>4.1K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>MQA (64 heads, 1 KV)</td><td>MQA (4 heads, 1 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-6 of 256 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>deepseek_v4</td><td>deepseek_v4</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>129280</td><td>129280</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>1048576</td><td>1048576</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>rope_theta</td><td>10000</td><td>10000</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix DeepseekV4 omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>468</td><td>50</td></tr><tr class=""><td>RMSNorm</td><td>235</td><td>26</td></tr><tr class=""><td>UnweightedRMSNorm</td><td>130</td><td>16</td></tr><tr class=""><td>HyperConnection</td><td>86</td><td>10</td></tr><tr class=""><td>SiLUActivation</td><td>86</td><td>10</td></tr><tr class=""><td>RotaryEmbedding</td><td>63</td><td>6</td></tr><tr class=""><td>Attention</td><td>43</td><td>5</td></tr><tr class=""><td>DecoderLayer</td><td>43</td><td>5</td></tr><tr class=""><td>Experts</td><td>43</td><td>5</td></tr><tr class=""><td>GroupedLinear</td><td>43</td><td>5</td></tr><tr class=""><td>MLP</td><td>43</td><td>5</td></tr><tr class=""><td>SparseMoeBlock</td><td>43</td><td>5</td></tr><tr class=""><td>SqrtSoftplusActivation</td><td>43</td><td>5</td></tr><tr class=""><td>TopKRouter</td><td>40</td><td>2</td></tr><tr class=""><td>CSACompressor</td><td>21</td><td>2</td></tr><tr class=""><td>Indexer</td><td>21</td><td>2</td></tr><tr class=""><td>IndexerScorer</td><td>21</td><td>2</td></tr><tr class=""><td>HCACompressor</td><td>20</td><td>1</td></tr><tr class=""><td>HashRouter</td><td>3</td><td>3</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>HyperHead</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">12 fields changed, 52 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>head_dim</td><td>512</td><td>16</td></tr><tr><td>hidden_size</td><td>4096</td><td>64</td></tr><tr><td>layer_types</td><td>43 entries: 21Γ— compressed_sparse_attention, 20Γ— heavily_compressed_attention, 2Γ— sliding_attention</td><td>[&quot;sliding_attention&quot;, &quot;sliding_attention&quot;, &quot;compressed_sparse_attention&quot;, &quot;he…</td></tr><tr><td>mlp_layer_types</td><td>43 entries: 40Γ— moe, 3Γ— hash_moe</td><td>[&quot;hash_moe&quot;, &quot;hash_moe&quot;, &quot;hash_moe&quot;, &quot;moe&quot;, &quot;moe&quot;]</td></tr><tr><td>moe_intermediate_size</td><td>2048</td><td>32</td></tr><tr><td>n_routed_experts</td><td>256</td><td>4</td></tr><tr><td>num_attention_heads</td><td>64</td><td>4</td></tr><tr><td>num_experts_per_tok</td><td>6</td><td>2</td></tr><tr><td>num_hidden_layers</td><td>43</td><td>5</td></tr><tr><td>q_lora_rank</td><td>1024</td><td>16</td></tr><tr><td>qk_rope_head_dim</td><td>64</td><td>2</td></tr><tr><td>quantization_config</td><td>{&quot;activation_scheme&quot;: &quot;dynamic&quot;, &quot;fmt&quot;: &quot;e4m3&quot;, &quot;quant_method&quot;: &quot;fp8&quot;, &quot;scale…</td><td>β€”</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>deepseek-ai/DeepSeek-V4-Flash-0731: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>deepseek-ai/DeepSeek-V4-Flash-0731</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/DeepSeek-V4-Flash-0731-mic/tree/7c36fb38ddaf52e554d784d9e6a2a7ed435b8311" title="huggingface-mic/DeepSeek-V4-Flash-0731-mic@7c36fb38ddaf52e554d784d9e6a2a7ed435b8311">huggingface-mic/DeepSeek-V4-Flash-0731-mic@7c36fb38ddaf</a> ← <a href="https://huggingface.co/deepseek-ai/DeepSeek-V4-Flash-0731/tree/7872f01b1d1fe23eabc4c98b48bffcef5a386062" title="deepseek-ai/DeepSeek-V4-Flash-0731@7872f01b1d1fe23eabc4c98b48bffcef5a386062">deepseek-ai/DeepSeek-V4-Flash-0731@7872f01b1d1f</a> Β· DeepseekV4ForCausalLM Β· random weights</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">284.3B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">20.1M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 43</span><div class="cells"><i class="c0" title="layer 0: sliding_attention, HashRouter"></i><i class="c0" title="layer 1: sliding_attention, HashRouter"></i><i class="c1" title="layer 2: compressed_sparse_attention, CSACompressor, HashRouter …"></i><i class="c2" title="layer 3: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 4: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 5: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 6: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 7: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 8: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 9: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 10: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 11: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 12: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 13: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 14: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 15: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 16: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 17: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 18: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 19: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 20: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 21: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 22: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 23: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 24: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 25: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 26: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 27: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 28: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 29: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 30: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 31: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 32: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 33: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 34: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 35: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 36: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 37: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 38: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 39: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 40: compressed_sparse_attention, CSACompressor, Indexer …"></i><i class="c2" title="layer 41: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 42: compressed_sparse_attention, CSACompressor, Indexer …"></i></div></div><div class="row"><span class="lab">Fixture Β· 5</span><div class="cells"><i class="c0" title="layer 0: sliding_attention, HashRouter"></i><i class="c0" title="layer 1: sliding_attention, HashRouter"></i><i class="c1" title="layer 2: compressed_sparse_attention, CSACompressor, HashRouter …"></i><i class="c2" title="layer 3: heavily_compressed_attention, HCACompressor, RotaryEmbedding …"></i><i class="c3" title="layer 4: compressed_sparse_attention, CSACompressor, Indexer …"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>sliding_attention, HashRouter</td><td>2</td><td>2</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>compressed_sparse_attention, CSACompressor, HashRouter …</td><td>1</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c2"></i>heavily_compressed_attention, HCACompressor, RotaryEmbedding …</td><td>20</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c3"></i>compressed_sparse_attention, CSACompressor, Indexer …</td><td>20</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.996275" title="model.layers: 283.3B (99.6%)"></i><i class="c1" style="flex:0.001862" title="lm_head: 529.5M (0.2%)"></i><i class="c2" style="flex:0.001862" title="model.embed_tokens: 529.5M (0.2%)"></i><i class="c3" style="flex:0.000000" title="model.hc_head: 65.5K (0.0%)"></i><i class="c4" style="flex:0.000000" title="model.norm: 4.1K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.175915" title="model.layers: 3.5M (17.6%)"></i><i class="c1" style="flex:0.412015" title="lm_head: 8.3M (41.2%)"></i><i class="c2" style="flex:0.412015" title="model.embed_tokens: 8.3M (41.2%)"></i><i class="c3" style="flex:0.000051" title="model.hc_head: 1.0K (0.0%)"></i><i class="c4" style="flex:0.000003" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>283.3B</td><td>99.6%</td><td>3.5M</td><td>17.6%</td></tr><tr><td><i class="c1"></i>lm_head</td><td>529.5M</td><td>0.2%</td><td>8.3M</td><td>41.2%</td></tr><tr><td><i class="c2"></i>model.embed_tokens</td><td>529.5M</td><td>0.2%</td><td>8.3M</td><td>41.2%</td></tr><tr><td><i class="c3"></i>model.hc_head</td><td>65.5K</td><td>0.0%</td><td>1.0K</td><td>0.0%</td></tr><tr><td><i class="c4"></i>model.norm</td><td>4.1K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>MQA (64 heads, 1 KV)</td><td>MQA (4 heads, 1 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-6 of 256 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>deepseek_v4</td><td>deepseek_v4</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>129280</td><td>129280</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>1048576</td><td>1048576</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>rope_theta</td><td>10000</td><td>10000</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix DeepseekV4 omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>468</td><td>50</td></tr><tr class=""><td>RMSNorm</td><td>235</td><td>26</td></tr><tr class=""><td>UnweightedRMSNorm</td><td>130</td><td>16</td></tr><tr class=""><td>HyperConnection</td><td>86</td><td>10</td></tr><tr class=""><td>SiLUActivation</td><td>86</td><td>10</td></tr><tr class=""><td>RotaryEmbedding</td><td>63</td><td>6</td></tr><tr class=""><td>Attention</td><td>43</td><td>5</td></tr><tr class=""><td>DecoderLayer</td><td>43</td><td>5</td></tr><tr class=""><td>Experts</td><td>43</td><td>5</td></tr><tr class=""><td>GroupedLinear</td><td>43</td><td>5</td></tr><tr class=""><td>MLP</td><td>43</td><td>5</td></tr><tr class=""><td>SparseMoeBlock</td><td>43</td><td>5</td></tr><tr class=""><td>SqrtSoftplusActivation</td><td>43</td><td>5</td></tr><tr class=""><td>TopKRouter</td><td>40</td><td>2</td></tr><tr class=""><td>CSACompressor</td><td>21</td><td>2</td></tr><tr class=""><td>Indexer</td><td>21</td><td>2</td></tr><tr class=""><td>IndexerScorer</td><td>21</td><td>2</td></tr><tr class=""><td>HCACompressor</td><td>20</td><td>1</td></tr><tr class=""><td>HashRouter</td><td>3</td><td>3</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>HyperHead</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">12 fields changed, 52 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>head_dim</td><td>512</td><td>16</td></tr><tr><td>hidden_size</td><td>4096</td><td>64</td></tr><tr><td>layer_types</td><td>43 entries: 21Γ— compressed_sparse_attention, 20Γ— heavily_compressed_attention, 2Γ— sliding_attention</td><td>[&quot;sliding_attention&quot;, &quot;sliding_attention&quot;, &quot;compressed_sparse_attention&quot;, &quot;he…</td></tr><tr><td>mlp_layer_types</td><td>43 entries: 40Γ— moe, 3Γ— hash_moe</td><td>[&quot;hash_moe&quot;, &quot;hash_moe&quot;, &quot;hash_moe&quot;, &quot;moe&quot;, &quot;moe&quot;]</td></tr><tr><td>moe_intermediate_size</td><td>2048</td><td>32</td></tr><tr><td>n_routed_experts</td><td>256</td><td>4</td></tr><tr><td>num_attention_heads</td><td>64</td><td>4</td></tr><tr><td>num_experts_per_tok</td><td>6</td><td>2</td></tr><tr><td>num_hidden_layers</td><td>43</td><td>5</td></tr><tr><td>q_lora_rank</td><td>1024</td><td>16</td></tr><tr><td>qk_rope_head_dim</td><td>64</td><td>2</td></tr><tr><td>quantization_config</td><td>{&quot;activation_scheme&quot;: &quot;dynamic&quot;, &quot;fmt&quot;: &quot;e4m3&quot;, &quot;quant_method&quot;: &quot;fp8&quot;, &quot;scale…</td><td>β€”</td></tr></table>
36
  </main></body></html>
shrink/DeepSeek-V4-Flash-0731-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/DeepSeek-V4-Flash-0731-mic@7c36fb38ddaf52e554d784d9e6a2a7ed435b8311", "findings": []}
shrink/ESMC-300M-hf-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/ESMC-300M-hf-mic@7c11c22ceffcbddb7fb1bb3216255d34ff38a43c ← biohub/ESMC-300M-hf@5554e4b3ab19eb66f04d7ff8e8c2d36c69b52fa0 Β· EsmcForMaskedLM Β· random weights</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">333.0M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">95.3K</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>esmc.layers</h3><div class="row"><span class="lab">Original Β· 30</span><div class="cells"><i class="c0" title="layer 0: Layer"></i><i class="c0" title="layer 1: Layer"></i><i class="c0" title="layer 2: Layer"></i><i class="c0" title="layer 3: Layer"></i><i class="c0" title="layer 4: Layer"></i><i class="c0" title="layer 5: Layer"></i><i class="c0" title="layer 6: Layer"></i><i class="c0" title="layer 7: Layer"></i><i class="c0" title="layer 8: Layer"></i><i class="c0" title="layer 9: Layer"></i><i class="c0" title="layer 10: Layer"></i><i class="c0" title="layer 11: Layer"></i><i class="c0" title="layer 12: Layer"></i><i class="c0" title="layer 13: Layer"></i><i class="c0" title="layer 14: Layer"></i><i class="c0" title="layer 15: Layer"></i><i class="c0" title="layer 16: Layer"></i><i class="c0" title="layer 17: Layer"></i><i class="c0" title="layer 18: Layer"></i><i class="c0" title="layer 19: Layer"></i><i class="c0" title="layer 20: Layer"></i><i class="c0" title="layer 21: Layer"></i><i class="c0" title="layer 22: Layer"></i><i class="c0" title="layer 23: Layer"></i><i class="c0" title="layer 24: Layer"></i><i class="c0" title="layer 25: Layer"></i><i class="c0" title="layer 26: Layer"></i><i class="c0" title="layer 27: Layer"></i><i class="c0" title="layer 28: Layer"></i><i class="c0" title="layer 29: Layer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: Layer"></i><i class="c0" title="layer 1: Layer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>Layer</td><td>30</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.996852" title="esmc.layers: 331.9M (99.7%)"></i><i class="c1" style="flex:0.002770" title="lm_head.dense: 922.6K (0.3%)"></i><i class="c2" style="flex:0.000185" title="lm_head.decoder: 61.5K (0.0%)"></i><i class="c3" style="flex:0.000185" title="esmc.embed_tokens: 61.4K (0.0%)"></i><i class="c4" style="flex:0.000006" title="lm_head.layer_norm: 1.9K (0.0%)"></i><i class="c5" style="flex:0.000003" title="esmc.norm: 960 (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.867696" title="esmc.layers: 82.7K (86.8%)"></i><i class="c1" style="flex:0.043653" title="lm_head.dense: 4.2K (4.4%)"></i><i class="c2" style="flex:0.043653" title="lm_head.decoder: 4.2K (4.4%)"></i><i class="c3" style="flex:0.042982" title="esmc.embed_tokens: 4.1K (4.3%)"></i><i class="c4" style="flex:0.001343" title="lm_head.layer_norm: 128 (0.1%)"></i><i class="c5" style="flex:0.000672" title="esmc.norm: 64 (0.1%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>esmc.layers</td><td>331.9M</td><td>99.7%</td><td>82.7K</td><td>86.8%</td></tr><tr><td><i class="c1"></i>lm_head.dense</td><td>922.6K</td><td>0.3%</td><td>4.2K</td><td>4.4%</td></tr><tr><td><i class="c2"></i>lm_head.decoder</td><td>61.5K</td><td>0.0%</td><td>4.2K</td><td>4.4%</td></tr><tr><td><i class="c3"></i>esmc.embed_tokens</td><td>61.4K</td><td>0.0%</td><td>4.1K</td><td>4.3%</td></tr><tr><td><i class="c4"></i>lm_head.layer_norm</td><td>1.9K</td><td>0.0%</td><td>128</td><td>0.1%</td></tr><tr><td><i class="c5"></i>esmc.norm</td><td>960</td><td>0.0%</td><td>64</td><td>0.1%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>MHA (15 heads, 15 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>esmc</td><td>esmc</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>64</td><td>64</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>2048</td><td>2048</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Esmc omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>212</td><td>16</td></tr><tr class=""><td>LayerNorm</td><td>121</td><td>9</td></tr><tr class=""><td>Attention</td><td>30</td><td>2</td></tr><tr class=""><td>Layer</td><td>30</td><td>2</td></tr><tr class=""><td>MLP</td><td>30</td><td>2</td></tr><tr class=""><td>SiLUActivation</td><td>30</td><td>2</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForMaskedLM</td><td>1</td><td>1</td></tr><tr class=""><td>MaskedLMHead</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>LayerNorm</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">6 fields changed, 23 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>head_dim</td><td>64</td><td>16</td></tr><tr><td>hidden_size</td><td>960</td><td>64</td></tr><tr><td>intermediate_size</td><td>2560</td><td>128</td></tr><tr><td>num_attention_heads</td><td>15</td><td>4</td></tr><tr><td>num_hidden_layers</td><td>30</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>15</td><td>4</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>biohub/ESMC-300M-hf: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>biohub/ESMC-300M-hf</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/ESMC-300M-hf-mic/tree/7c11c22ceffcbddb7fb1bb3216255d34ff38a43c" title="huggingface-mic/ESMC-300M-hf-mic@7c11c22ceffcbddb7fb1bb3216255d34ff38a43c">huggingface-mic/ESMC-300M-hf-mic@7c11c22ceffc</a> ← <a href="https://huggingface.co/biohub/ESMC-300M-hf/tree/5554e4b3ab19eb66f04d7ff8e8c2d36c69b52fa0" title="biohub/ESMC-300M-hf@5554e4b3ab19eb66f04d7ff8e8c2d36c69b52fa0">biohub/ESMC-300M-hf@5554e4b3ab19</a> Β· EsmcForMaskedLM Β· random weights</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">333.0M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">95.3K</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>esmc.layers</h3><div class="row"><span class="lab">Original Β· 30</span><div class="cells"><i class="c0" title="layer 0: Layer"></i><i class="c0" title="layer 1: Layer"></i><i class="c0" title="layer 2: Layer"></i><i class="c0" title="layer 3: Layer"></i><i class="c0" title="layer 4: Layer"></i><i class="c0" title="layer 5: Layer"></i><i class="c0" title="layer 6: Layer"></i><i class="c0" title="layer 7: Layer"></i><i class="c0" title="layer 8: Layer"></i><i class="c0" title="layer 9: Layer"></i><i class="c0" title="layer 10: Layer"></i><i class="c0" title="layer 11: Layer"></i><i class="c0" title="layer 12: Layer"></i><i class="c0" title="layer 13: Layer"></i><i class="c0" title="layer 14: Layer"></i><i class="c0" title="layer 15: Layer"></i><i class="c0" title="layer 16: Layer"></i><i class="c0" title="layer 17: Layer"></i><i class="c0" title="layer 18: Layer"></i><i class="c0" title="layer 19: Layer"></i><i class="c0" title="layer 20: Layer"></i><i class="c0" title="layer 21: Layer"></i><i class="c0" title="layer 22: Layer"></i><i class="c0" title="layer 23: Layer"></i><i class="c0" title="layer 24: Layer"></i><i class="c0" title="layer 25: Layer"></i><i class="c0" title="layer 26: Layer"></i><i class="c0" title="layer 27: Layer"></i><i class="c0" title="layer 28: Layer"></i><i class="c0" title="layer 29: Layer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: Layer"></i><i class="c0" title="layer 1: Layer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>Layer</td><td>30</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.996852" title="esmc.layers: 331.9M (99.7%)"></i><i class="c1" style="flex:0.002770" title="lm_head.dense: 922.6K (0.3%)"></i><i class="c2" style="flex:0.000185" title="lm_head.decoder: 61.5K (0.0%)"></i><i class="c3" style="flex:0.000185" title="esmc.embed_tokens: 61.4K (0.0%)"></i><i class="c4" style="flex:0.000006" title="lm_head.layer_norm: 1.9K (0.0%)"></i><i class="c5" style="flex:0.000003" title="esmc.norm: 960 (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.867696" title="esmc.layers: 82.7K (86.8%)"></i><i class="c1" style="flex:0.043653" title="lm_head.dense: 4.2K (4.4%)"></i><i class="c2" style="flex:0.043653" title="lm_head.decoder: 4.2K (4.4%)"></i><i class="c3" style="flex:0.042982" title="esmc.embed_tokens: 4.1K (4.3%)"></i><i class="c4" style="flex:0.001343" title="lm_head.layer_norm: 128 (0.1%)"></i><i class="c5" style="flex:0.000672" title="esmc.norm: 64 (0.1%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>esmc.layers</td><td>331.9M</td><td>99.7%</td><td>82.7K</td><td>86.8%</td></tr><tr><td><i class="c1"></i>lm_head.dense</td><td>922.6K</td><td>0.3%</td><td>4.2K</td><td>4.4%</td></tr><tr><td><i class="c2"></i>lm_head.decoder</td><td>61.5K</td><td>0.0%</td><td>4.2K</td><td>4.4%</td></tr><tr><td><i class="c3"></i>esmc.embed_tokens</td><td>61.4K</td><td>0.0%</td><td>4.1K</td><td>4.3%</td></tr><tr><td><i class="c4"></i>lm_head.layer_norm</td><td>1.9K</td><td>0.0%</td><td>128</td><td>0.1%</td></tr><tr><td><i class="c5"></i>esmc.norm</td><td>960</td><td>0.0%</td><td>64</td><td>0.1%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>MHA (15 heads, 15 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>esmc</td><td>esmc</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>64</td><td>64</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>2048</td><td>2048</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Esmc omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>212</td><td>16</td></tr><tr class=""><td>LayerNorm</td><td>121</td><td>9</td></tr><tr class=""><td>Attention</td><td>30</td><td>2</td></tr><tr class=""><td>Layer</td><td>30</td><td>2</td></tr><tr class=""><td>MLP</td><td>30</td><td>2</td></tr><tr class=""><td>SiLUActivation</td><td>30</td><td>2</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForMaskedLM</td><td>1</td><td>1</td></tr><tr class=""><td>MaskedLMHead</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>LayerNorm</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">6 fields changed, 23 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>head_dim</td><td>64</td><td>16</td></tr><tr><td>hidden_size</td><td>960</td><td>64</td></tr><tr><td>intermediate_size</td><td>2560</td><td>128</td></tr><tr><td>num_attention_heads</td><td>15</td><td>4</td></tr><tr><td>num_hidden_layers</td><td>30</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>15</td><td>4</td></tr></table>
36
  </main></body></html>
shrink/ESMC-300M-hf-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/ESMC-300M-hf-mic@7c11c22ceffcbddb7fb1bb3216255d34ff38a43c", "findings": []}
shrink/EuroBERT-210m-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/EuroBERT-210m-mic@514cea1b679f118f051124ef88272fb809f25f16 ← EuroBERT/EuroBERT-210m@39b51e15dd1f1a06f58b5cbf6a8a188cec60bd0e Β· EuroBertForMaskedLM Β· random weights</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">310.3M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">16.5M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 12</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i><i class="c0" title="layer 2: DecoderLayer"></i><i class="c0" title="layer 3: DecoderLayer"></i><i class="c0" title="layer 4: DecoderLayer"></i><i class="c0" title="layer 5: DecoderLayer"></i><i class="c0" title="layer 6: DecoderLayer"></i><i class="c0" title="layer 7: DecoderLayer"></i><i class="c0" title="layer 8: DecoderLayer"></i><i class="c0" title="layer 9: DecoderLayer"></i><i class="c0" title="layer 10: DecoderLayer"></i><i class="c0" title="layer 11: DecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>DecoderLayer</td><td>12</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.365056" title="model.layers: 113.3M (36.5%)"></i><i class="c1" style="flex:0.317471" title="model.embed_tokens: 98.5M (31.7%)"></i><i class="c2" style="flex:0.317471" title="lm_head: 98.5M (31.7%)"></i><i class="c3" style="flex:0.000002" title="model.norm: 768 (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.004981" title="model.layers: 82.2K (0.5%)"></i><i class="c1" style="flex:0.497508" title="model.embed_tokens: 8.2M (49.8%)"></i><i class="c2" style="flex:0.497508" title="lm_head: 8.2M (49.8%)"></i><i class="c3" style="flex:0.000004" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>113.3M</td><td>36.5%</td><td>82.2K</td><td>0.5%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>98.5M</td><td>31.7%</td><td>8.2M</td><td>49.8%</td></tr><tr><td><i class="c2"></i>lm_head</td><td>98.5M</td><td>31.7%</td><td>8.2M</td><td>49.8%</td></tr><tr><td><i class="c3"></i>model.norm</td><td>768</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>MHA (12 heads, 12 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>eurobert</td><td>eurobert</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>128256</td><td>128256</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>8192</td><td>8192</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix EuroBert omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>85</td><td>15</td></tr><tr class=""><td>RMSNorm</td><td>25</td><td>5</td></tr><tr class=""><td>Attention</td><td>12</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>12</td><td>2</td></tr><tr class=""><td>MLP</td><td>12</td><td>2</td></tr><tr class=""><td>SiLUActivation</td><td>12</td><td>2</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForMaskedLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">6 fields changed, 33 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>head_dim</td><td>64</td><td>16</td></tr><tr><td>hidden_size</td><td>768</td><td>64</td></tr><tr><td>intermediate_size</td><td>3072</td><td>128</td></tr><tr><td>num_attention_heads</td><td>12</td><td>4</td></tr><tr><td>num_hidden_layers</td><td>12</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>12</td><td>4</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>EuroBERT/EuroBERT-210m: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>EuroBERT/EuroBERT-210m</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/EuroBERT-210m-mic/tree/514cea1b679f118f051124ef88272fb809f25f16" title="huggingface-mic/EuroBERT-210m-mic@514cea1b679f118f051124ef88272fb809f25f16">huggingface-mic/EuroBERT-210m-mic@514cea1b679f</a> ← <a href="https://huggingface.co/EuroBERT/EuroBERT-210m/tree/39b51e15dd1f1a06f58b5cbf6a8a188cec60bd0e" title="EuroBERT/EuroBERT-210m@39b51e15dd1f1a06f58b5cbf6a8a188cec60bd0e">EuroBERT/EuroBERT-210m@39b51e15dd1f</a> Β· EuroBertForMaskedLM Β· random weights</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">310.3M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">16.5M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 12</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i><i class="c0" title="layer 2: DecoderLayer"></i><i class="c0" title="layer 3: DecoderLayer"></i><i class="c0" title="layer 4: DecoderLayer"></i><i class="c0" title="layer 5: DecoderLayer"></i><i class="c0" title="layer 6: DecoderLayer"></i><i class="c0" title="layer 7: DecoderLayer"></i><i class="c0" title="layer 8: DecoderLayer"></i><i class="c0" title="layer 9: DecoderLayer"></i><i class="c0" title="layer 10: DecoderLayer"></i><i class="c0" title="layer 11: DecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>DecoderLayer</td><td>12</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.365056" title="model.layers: 113.3M (36.5%)"></i><i class="c1" style="flex:0.317471" title="model.embed_tokens: 98.5M (31.7%)"></i><i class="c2" style="flex:0.317471" title="lm_head: 98.5M (31.7%)"></i><i class="c3" style="flex:0.000002" title="model.norm: 768 (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.004981" title="model.layers: 82.2K (0.5%)"></i><i class="c1" style="flex:0.497508" title="model.embed_tokens: 8.2M (49.8%)"></i><i class="c2" style="flex:0.497508" title="lm_head: 8.2M (49.8%)"></i><i class="c3" style="flex:0.000004" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>113.3M</td><td>36.5%</td><td>82.2K</td><td>0.5%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>98.5M</td><td>31.7%</td><td>8.2M</td><td>49.8%</td></tr><tr><td><i class="c2"></i>lm_head</td><td>98.5M</td><td>31.7%</td><td>8.2M</td><td>49.8%</td></tr><tr><td><i class="c3"></i>model.norm</td><td>768</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>MHA (12 heads, 12 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>eurobert</td><td>eurobert</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>128256</td><td>128256</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>8192</td><td>8192</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix EuroBert omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>85</td><td>15</td></tr><tr class=""><td>RMSNorm</td><td>25</td><td>5</td></tr><tr class=""><td>Attention</td><td>12</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>12</td><td>2</td></tr><tr class=""><td>MLP</td><td>12</td><td>2</td></tr><tr class=""><td>SiLUActivation</td><td>12</td><td>2</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForMaskedLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">6 fields changed, 33 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>head_dim</td><td>64</td><td>16</td></tr><tr><td>hidden_size</td><td>768</td><td>64</td></tr><tr><td>intermediate_size</td><td>3072</td><td>128</td></tr><tr><td>num_attention_heads</td><td>12</td><td>4</td></tr><tr><td>num_hidden_layers</td><td>12</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>12</td><td>4</td></tr></table>
36
  </main></body></html>
shrink/EuroBERT-210m-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/EuroBERT-210m-mic@514cea1b679f118f051124ef88272fb809f25f16", "findings": []}
shrink/Falcon-H1-Tiny-90M-Instruct-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/Falcon-H1-Tiny-90M-Instruct-mic@0db75b996c3e47fea9cc1a9c213983988b24495a ← tiiuae/Falcon-H1-Tiny-90M-Instruct@e6389502a0b12cd8da894b395ba5bf7436873b16 Β· FalconH1ForCausalLM Β· random weights</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">91.1M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">2.5M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 24</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i><i class="c0" title="layer 2: DecoderLayer"></i><i class="c0" title="layer 3: DecoderLayer"></i><i class="c0" title="layer 4: DecoderLayer"></i><i class="c0" title="layer 5: DecoderLayer"></i><i class="c0" title="layer 6: DecoderLayer"></i><i class="c0" title="layer 7: DecoderLayer"></i><i class="c0" title="layer 8: DecoderLayer"></i><i class="c0" title="layer 9: DecoderLayer"></i><i class="c0" title="layer 10: DecoderLayer"></i><i class="c0" title="layer 11: DecoderLayer"></i><i class="c0" title="layer 12: DecoderLayer"></i><i class="c0" title="layer 13: DecoderLayer"></i><i class="c0" title="layer 14: DecoderLayer"></i><i class="c0" title="layer 15: DecoderLayer"></i><i class="c0" title="layer 16: DecoderLayer"></i><i class="c0" title="layer 17: DecoderLayer"></i><i class="c0" title="layer 18: DecoderLayer"></i><i class="c0" title="layer 19: DecoderLayer"></i><i class="c0" title="layer 20: DecoderLayer"></i><i class="c0" title="layer 21: DecoderLayer"></i><i class="c0" title="layer 22: DecoderLayer"></i><i class="c0" title="layer 23: DecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>DecoderLayer</td><td>24</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.815895" title="model.layers: 74.4M (81.6%)"></i><i class="c1" style="flex:0.184100" title="model.embed_tokens: 16.8M (18.4%)"></i><i class="c2" style="flex:0.000006" title="model.final_layernorm: 512 (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.154103" title="model.layers: 382.1K (15.4%)"></i><i class="c1" style="flex:0.845871" title="model.embed_tokens: 2.1M (84.6%)"></i><i class="c2" style="flex:0.000026" title="model.final_layernorm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>74.4M</td><td>81.6%</td><td>382.1K</td><td>15.4%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>16.8M</td><td>18.4%</td><td>2.1M</td><td>84.6%</td></tr><tr><td><i class="c2"></i>model.final_layernorm</td><td>512</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>GQA (8 heads, 2 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>falcon_h1</td><td>falcon_h1</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>32768</td><td>32768</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>262144</td><td>262144</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix FalconH1 omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>217</td><td>19</td></tr><tr class=""><td>RMSNorm</td><td>49</td><td>5</td></tr><tr class=""><td>SiLUActivation</td><td>48</td><td>4</td></tr><tr class=""><td>Conv1d</td><td>24</td><td>2</td></tr><tr class=""><td>Attention</td><td>24</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>24</td><td>2</td></tr><tr class=""><td>MLP</td><td>24</td><td>2</td></tr><tr class=""><td>Mixer</td><td>24</td><td>2</td></tr><tr class=""><td>Identity</td><td>24</td><td>2</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">9 fields changed, 53 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>head_dim</td><td>64</td><td>16</td></tr><tr><td>hidden_size</td><td>512</td><td>64</td></tr><tr><td>intermediate_size</td><td>768</td><td>128</td></tr><tr><td>mamba_chunk_size</td><td>128</td><td>8</td></tr><tr><td>mamba_d_head</td><td>32</td><td>96</td></tr><tr><td>mamba_d_state</td><td>64</td><td>16</td></tr><tr><td>mamba_n_heads</td><td>24</td><td>8</td></tr><tr><td>num_attention_heads</td><td>8</td><td>4</td></tr><tr><td>num_hidden_layers</td><td>24</td><td>2</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>tiiuae/Falcon-H1-Tiny-90M-Instruct: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>tiiuae/Falcon-H1-Tiny-90M-Instruct</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/Falcon-H1-Tiny-90M-Instruct-mic/tree/0db75b996c3e47fea9cc1a9c213983988b24495a" title="huggingface-mic/Falcon-H1-Tiny-90M-Instruct-mic@0db75b996c3e47fea9cc1a9c213983988b24495a">huggingface-mic/Falcon-H1-Tiny-90M-Instruct-mic@0db75b996c3e</a> ← <a href="https://huggingface.co/tiiuae/Falcon-H1-Tiny-90M-Instruct/tree/e6389502a0b12cd8da894b395ba5bf7436873b16" title="tiiuae/Falcon-H1-Tiny-90M-Instruct@e6389502a0b12cd8da894b395ba5bf7436873b16">tiiuae/Falcon-H1-Tiny-90M-Instruct@e6389502a0b1</a> Β· FalconH1ForCausalLM Β· random weights</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">91.1M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">2.5M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 24</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i><i class="c0" title="layer 2: DecoderLayer"></i><i class="c0" title="layer 3: DecoderLayer"></i><i class="c0" title="layer 4: DecoderLayer"></i><i class="c0" title="layer 5: DecoderLayer"></i><i class="c0" title="layer 6: DecoderLayer"></i><i class="c0" title="layer 7: DecoderLayer"></i><i class="c0" title="layer 8: DecoderLayer"></i><i class="c0" title="layer 9: DecoderLayer"></i><i class="c0" title="layer 10: DecoderLayer"></i><i class="c0" title="layer 11: DecoderLayer"></i><i class="c0" title="layer 12: DecoderLayer"></i><i class="c0" title="layer 13: DecoderLayer"></i><i class="c0" title="layer 14: DecoderLayer"></i><i class="c0" title="layer 15: DecoderLayer"></i><i class="c0" title="layer 16: DecoderLayer"></i><i class="c0" title="layer 17: DecoderLayer"></i><i class="c0" title="layer 18: DecoderLayer"></i><i class="c0" title="layer 19: DecoderLayer"></i><i class="c0" title="layer 20: DecoderLayer"></i><i class="c0" title="layer 21: DecoderLayer"></i><i class="c0" title="layer 22: DecoderLayer"></i><i class="c0" title="layer 23: DecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>DecoderLayer</td><td>24</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.815895" title="model.layers: 74.4M (81.6%)"></i><i class="c1" style="flex:0.184100" title="model.embed_tokens: 16.8M (18.4%)"></i><i class="c2" style="flex:0.000006" title="model.final_layernorm: 512 (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.154103" title="model.layers: 382.1K (15.4%)"></i><i class="c1" style="flex:0.845871" title="model.embed_tokens: 2.1M (84.6%)"></i><i class="c2" style="flex:0.000026" title="model.final_layernorm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>74.4M</td><td>81.6%</td><td>382.1K</td><td>15.4%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>16.8M</td><td>18.4%</td><td>2.1M</td><td>84.6%</td></tr><tr><td><i class="c2"></i>model.final_layernorm</td><td>512</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>GQA (8 heads, 2 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>falcon_h1</td><td>falcon_h1</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>32768</td><td>32768</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>262144</td><td>262144</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix FalconH1 omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>217</td><td>19</td></tr><tr class=""><td>RMSNorm</td><td>49</td><td>5</td></tr><tr class=""><td>SiLUActivation</td><td>48</td><td>4</td></tr><tr class=""><td>Conv1d</td><td>24</td><td>2</td></tr><tr class=""><td>Attention</td><td>24</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>24</td><td>2</td></tr><tr class=""><td>MLP</td><td>24</td><td>2</td></tr><tr class=""><td>Mixer</td><td>24</td><td>2</td></tr><tr class=""><td>Identity</td><td>24</td><td>2</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">9 fields changed, 53 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>head_dim</td><td>64</td><td>16</td></tr><tr><td>hidden_size</td><td>512</td><td>64</td></tr><tr><td>intermediate_size</td><td>768</td><td>128</td></tr><tr><td>mamba_chunk_size</td><td>128</td><td>8</td></tr><tr><td>mamba_d_head</td><td>32</td><td>96</td></tr><tr><td>mamba_d_state</td><td>64</td><td>16</td></tr><tr><td>mamba_n_heads</td><td>24</td><td>8</td></tr><tr><td>num_attention_heads</td><td>8</td><td>4</td></tr><tr><td>num_hidden_layers</td><td>24</td><td>2</td></tr></table>
36
  </main></body></html>
shrink/Falcon-H1-Tiny-90M-Instruct-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/Falcon-H1-Tiny-90M-Instruct-mic@0db75b996c3e47fea9cc1a9c213983988b24495a", "findings": []}
shrink/Florence-2-base-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/Florence-2-base-mic@85c1b12e1d9565763f6f2b9466669c58bc7aac33 ← florence-community/Florence-2-base@00921df66db728a9ceb750f5eca43e5c203a2051 Β· Florence2ForConditionalGeneration Β· overfit weights Β· built before the shrink rule (no recorded recipe)</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">231.4M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">6.4M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">4</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.vision_tower.convs</h3><div class="row"><span class="lab">Original Β· 4</span><div class="cells"><i class="c0" title="layer 0: Conv2d"></i><i class="c0" title="layer 1: Conv2d"></i><i class="c0" title="layer 2: Conv2d"></i><i class="c0" title="layer 3: Conv2d"></i></div></div><div class="row"><span class="lab">Fixture Β· 4</span><div class="cells"><i class="c0" title="layer 0: Conv2d"></i><i class="c0" title="layer 1: Conv2d"></i><i class="c0" title="layer 2: Conv2d"></i><i class="c0" title="layer 3: Conv2d"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>Conv2d</td><td>4</td><td>4</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.vision_tower.blocks</h3><div class="row"><span class="lab">Original Β· 4</span><div class="cells"><i class="c0" title="layer 0: VisionBlock"></i><i class="c0" title="layer 1: VisionBlock"></i><i class="c0" title="layer 2: VisionBlock"></i><i class="c0" title="layer 3: VisionBlock"></i></div></div><div class="row"><span class="lab">Fixture Β· 4</span><div class="cells"><i class="c0" title="layer 0: VisionBlock"></i><i class="c0" title="layer 1: VisionBlock"></i><i class="c0" title="layer 2: VisionBlock"></i><i class="c0" title="layer 3: VisionBlock"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>VisionBlock</td><td>4</td><td>4</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.language_model.encoder.layers</h3><div class="row"><span class="lab">Original Β· 6</span><div class="cells"><i class="c0" title="layer 0: BartEncoderLayer"></i><i class="c0" title="layer 1: BartEncoderLayer"></i><i class="c0" title="layer 2: BartEncoderLayer"></i><i class="c0" title="layer 3: BartEncoderLayer"></i><i class="c0" title="layer 4: BartEncoderLayer"></i><i class="c0" title="layer 5: BartEncoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 1</span><div class="cells"><i class="c0" title="layer 0: BartEncoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>BartEncoderLayer</td><td>6</td><td>1</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.language_model.decoder.layers</h3><div class="row"><span class="lab">Original Β· 6</span><div class="cells"><i class="c0" title="layer 0: BartDecoderLayer"></i><i class="c0" title="layer 1: BartDecoderLayer"></i><i class="c0" title="layer 2: BartDecoderLayer"></i><i class="c0" title="layer 3: BartDecoderLayer"></i><i class="c0" title="layer 4: BartDecoderLayer"></i><i class="c0" title="layer 5: BartDecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 1</span><div class="cells"><i class="c0" title="layer 0: BartDecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>BartDecoderLayer</td><td>6</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.363596" title="model.vision_tower.blocks: 84.2M (36.4%)"></i><i class="c1" style="flex:0.245030" title="model.language_model.decoder.layers: 56.7M (24.5%)"></i><i class="c2" style="flex:0.183747" title="model.language_model.encoder.layers: 42.5M (18.4%)"></i><i class="c3" style="flex:0.177144" title="model.language_model: 41.0M (17.7%)"></i><i class="c4" style="flex:0.026857" title="model.vision_tower.convs: 6.2M (2.7%)"></i><i class="c5" style="flex:0.003626" title="model.multi_modal_projector: 839.2K (0.4%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.391378" title="model.vision_tower.blocks: 2.5M (39.1%)"></i><i class="c1" style="flex:0.007796" title="model.language_model.decoder.layers: 50.2K (0.8%)"></i><i class="c2" style="flex:0.005194" title="model.language_model.encoder.layers: 33.5K (0.5%)"></i><i class="c3" style="flex:0.530140" title="model.language_model: 3.4M (53.0%)"></i><i class="c4" style="flex:0.060945" title="model.vision_tower.convs: 392.8K (6.1%)"></i><i class="c5" style="flex:0.004548" title="model.multi_modal_projector: 29.3K (0.5%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.vision_tower.blocks</td><td>84.2M</td><td>36.4%</td><td>2.5M</td><td>39.1%</td></tr><tr><td><i class="c1"></i>model.language_model.decoder.layers</td><td>56.7M</td><td>24.5%</td><td>50.2K</td><td>0.8%</td></tr><tr><td><i class="c2"></i>model.language_model.encoder.layers</td><td>42.5M</td><td>18.4%</td><td>33.5K</td><td>0.5%</td></tr><tr><td><i class="c3"></i>model.language_model</td><td>41.0M</td><td>17.7%</td><td>3.4M</td><td>53.0%</td></tr><tr><td><i class="c4"></i>model.vision_tower.convs</td><td>6.2M</td><td>2.7%</td><td>392.8K</td><td>6.1%</td></tr><tr><td><i class="c5"></i>model.multi_modal_projector</td><td>839.2K</td><td>0.4%</td><td>29.3K</td><td>0.5%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>model_type</td><td>florence2</td><td>florence2</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>model_type</td><td>bart</td><td>bart</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>vocab_size</td><td>51328</td><td>51328</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>max_position_embeddings</td><td>1024</td><td>1024</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>model_type</td><td>florence_vision</td><td>florence_vision</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>max_position_embeddings</td><td>50</td><td>50</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>patch_size</td><td>[7, 3, 3, 3]</td><td>[7, 3, 3, 3]</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Florence2 omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>194</td><td>58</td></tr><tr class=""><td>LayerNorm</td><td>85</td><td>32</td></tr><tr class=""><td>Conv2d</td><td>52</td><td>24</td></tr><tr class=""><td>DropPath</td><td>46</td><td>18</td></tr><tr class=""><td>GELUActivation</td><td>36</td><td>12</td></tr><tr class=""><td>VisionMLP</td><td>24</td><td>10</td></tr><tr class=""><td>BartAttention</td><td>18</td><td>3</td></tr><tr class=""><td>VisionBlock</td><td>12</td><td>5</td></tr><tr class=""><td>VisionChannelAttention</td><td>12</td><td>5</td></tr><tr class=""><td>VisionChannelBlock</td><td>12</td><td>5</td></tr><tr class=""><td>VisionSpatialBlock</td><td>12</td><td>5</td></tr><tr class=""><td>VisionWindowAttention</td><td>12</td><td>5</td></tr><tr class=""><td>ModuleList</td><td>8</td><td>8</td></tr><tr class=""><td>BartDecoderLayer</td><td>6</td><td>1</td></tr><tr class=""><td>BartEncoderLayer</td><td>6</td><td>1</td></tr><tr class=""><td>VisionConvEmbed</td><td>4</td><td>4</td></tr><tr class=""><td>BartScaledWordEmbedding</td><td>3</td><td>3</td></tr><tr class=""><td>BartLearnedPositionalEmbedding</td><td>2</td><td>2</td></tr><tr class=""><td>Embedding</td><td>2</td><td>2</td></tr><tr class=""><td>Identity</td><td>2</td><td>2</td></tr><tr class=""><td>BartDecoder</td><td>1</td><td>1</td></tr><tr class=""><td>BartEncoder</td><td>1</td><td>1</td></tr><tr class=""><td>BartModel</td><td>1</td><td>1</td></tr><tr class=""><td>ForConditionalGeneration</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>MultiModalProjector</td><td>1</td><td>1</td></tr><tr class=""><td>VisionBackbone</td><td>1</td><td>1</td></tr><tr class=""><td>VisionLearnedAbsolutePositionEmbedding2D</td><td>1</td><td>1</td></tr><tr class=""><td>VisionPositionalEmbeddingCosine1D</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">12 fields changed, 72 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>text_config.d_model</td><td>768</td><td>64</td></tr><tr><td>text_config.decoder_attention_heads</td><td>12</td><td>2</td></tr><tr><td>text_config.decoder_ffn_dim</td><td>3072</td><td>128</td></tr><tr><td>text_config.decoder_layers</td><td>6</td><td>1</td></tr><tr><td>text_config.encoder_attention_heads</td><td>12</td><td>2</td></tr><tr><td>text_config.encoder_ffn_dim</td><td>3072</td><td>128</td></tr><tr><td>text_config.encoder_layers</td><td>6</td><td>1</td></tr><tr><td>vision_config.depths</td><td>[1, 1, 9, 1]</td><td>[1, 1, 2, 1]</td></tr><tr><td>vision_config.embed_dim</td><td>[128, 256, 512, 1024]</td><td>[32, 64, 128, 256]</td></tr><tr><td>vision_config.num_groups</td><td>[4, 8, 16, 32]</td><td>[2, 4, 8, 16]</td></tr><tr><td>vision_config.num_heads</td><td>[4, 8, 16, 32]</td><td>[2, 4, 8, 16]</td></tr><tr><td>vision_config.projection_dim</td><td>768</td><td>64</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>florence-community/Florence-2-base: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>florence-community/Florence-2-base</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/Florence-2-base-mic/tree/85c1b12e1d9565763f6f2b9466669c58bc7aac33" title="huggingface-mic/Florence-2-base-mic@85c1b12e1d9565763f6f2b9466669c58bc7aac33">huggingface-mic/Florence-2-base-mic@85c1b12e1d95</a> ← <a href="https://huggingface.co/florence-community/Florence-2-base/tree/00921df66db728a9ceb750f5eca43e5c203a2051" title="florence-community/Florence-2-base@00921df66db728a9ceb750f5eca43e5c203a2051">florence-community/Florence-2-base@00921df66db7</a> Β· Florence2ForConditionalGeneration Β· overfit weights Β· built before the shrink rule (no recorded recipe)</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">231.4M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">6.4M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">4</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.vision_tower.convs</h3><div class="row"><span class="lab">Original Β· 4</span><div class="cells"><i class="c0" title="layer 0: Conv2d"></i><i class="c0" title="layer 1: Conv2d"></i><i class="c0" title="layer 2: Conv2d"></i><i class="c0" title="layer 3: Conv2d"></i></div></div><div class="row"><span class="lab">Fixture Β· 4</span><div class="cells"><i class="c0" title="layer 0: Conv2d"></i><i class="c0" title="layer 1: Conv2d"></i><i class="c0" title="layer 2: Conv2d"></i><i class="c0" title="layer 3: Conv2d"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>Conv2d</td><td>4</td><td>4</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.vision_tower.blocks</h3><div class="row"><span class="lab">Original Β· 4</span><div class="cells"><i class="c0" title="layer 0: VisionBlock"></i><i class="c0" title="layer 1: VisionBlock"></i><i class="c0" title="layer 2: VisionBlock"></i><i class="c0" title="layer 3: VisionBlock"></i></div></div><div class="row"><span class="lab">Fixture Β· 4</span><div class="cells"><i class="c0" title="layer 0: VisionBlock"></i><i class="c0" title="layer 1: VisionBlock"></i><i class="c0" title="layer 2: VisionBlock"></i><i class="c0" title="layer 3: VisionBlock"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>VisionBlock</td><td>4</td><td>4</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.language_model.encoder.layers</h3><div class="row"><span class="lab">Original Β· 6</span><div class="cells"><i class="c0" title="layer 0: BartEncoderLayer"></i><i class="c0" title="layer 1: BartEncoderLayer"></i><i class="c0" title="layer 2: BartEncoderLayer"></i><i class="c0" title="layer 3: BartEncoderLayer"></i><i class="c0" title="layer 4: BartEncoderLayer"></i><i class="c0" title="layer 5: BartEncoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 1</span><div class="cells"><i class="c0" title="layer 0: BartEncoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>BartEncoderLayer</td><td>6</td><td>1</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.language_model.decoder.layers</h3><div class="row"><span class="lab">Original Β· 6</span><div class="cells"><i class="c0" title="layer 0: BartDecoderLayer"></i><i class="c0" title="layer 1: BartDecoderLayer"></i><i class="c0" title="layer 2: BartDecoderLayer"></i><i class="c0" title="layer 3: BartDecoderLayer"></i><i class="c0" title="layer 4: BartDecoderLayer"></i><i class="c0" title="layer 5: BartDecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 1</span><div class="cells"><i class="c0" title="layer 0: BartDecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>BartDecoderLayer</td><td>6</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.363596" title="model.vision_tower.blocks: 84.2M (36.4%)"></i><i class="c1" style="flex:0.245030" title="model.language_model.decoder.layers: 56.7M (24.5%)"></i><i class="c2" style="flex:0.183747" title="model.language_model.encoder.layers: 42.5M (18.4%)"></i><i class="c3" style="flex:0.177144" title="model.language_model: 41.0M (17.7%)"></i><i class="c4" style="flex:0.026857" title="model.vision_tower.convs: 6.2M (2.7%)"></i><i class="c5" style="flex:0.003626" title="model.multi_modal_projector: 839.2K (0.4%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.391378" title="model.vision_tower.blocks: 2.5M (39.1%)"></i><i class="c1" style="flex:0.007796" title="model.language_model.decoder.layers: 50.2K (0.8%)"></i><i class="c2" style="flex:0.005194" title="model.language_model.encoder.layers: 33.5K (0.5%)"></i><i class="c3" style="flex:0.530140" title="model.language_model: 3.4M (53.0%)"></i><i class="c4" style="flex:0.060945" title="model.vision_tower.convs: 392.8K (6.1%)"></i><i class="c5" style="flex:0.004548" title="model.multi_modal_projector: 29.3K (0.5%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.vision_tower.blocks</td><td>84.2M</td><td>36.4%</td><td>2.5M</td><td>39.1%</td></tr><tr><td><i class="c1"></i>model.language_model.decoder.layers</td><td>56.7M</td><td>24.5%</td><td>50.2K</td><td>0.8%</td></tr><tr><td><i class="c2"></i>model.language_model.encoder.layers</td><td>42.5M</td><td>18.4%</td><td>33.5K</td><td>0.5%</td></tr><tr><td><i class="c3"></i>model.language_model</td><td>41.0M</td><td>17.7%</td><td>3.4M</td><td>53.0%</td></tr><tr><td><i class="c4"></i>model.vision_tower.convs</td><td>6.2M</td><td>2.7%</td><td>392.8K</td><td>6.1%</td></tr><tr><td><i class="c5"></i>model.multi_modal_projector</td><td>839.2K</td><td>0.4%</td><td>29.3K</td><td>0.5%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>model_type</td><td>florence2</td><td>florence2</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>model_type</td><td>bart</td><td>bart</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>vocab_size</td><td>51328</td><td>51328</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>max_position_embeddings</td><td>1024</td><td>1024</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>model_type</td><td>florence_vision</td><td>florence_vision</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>max_position_embeddings</td><td>50</td><td>50</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>patch_size</td><td>[7, 3, 3, 3]</td><td>[7, 3, 3, 3]</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Florence2 omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>194</td><td>58</td></tr><tr class=""><td>LayerNorm</td><td>85</td><td>32</td></tr><tr class=""><td>Conv2d</td><td>52</td><td>24</td></tr><tr class=""><td>DropPath</td><td>46</td><td>18</td></tr><tr class=""><td>GELUActivation</td><td>36</td><td>12</td></tr><tr class=""><td>VisionMLP</td><td>24</td><td>10</td></tr><tr class=""><td>BartAttention</td><td>18</td><td>3</td></tr><tr class=""><td>VisionBlock</td><td>12</td><td>5</td></tr><tr class=""><td>VisionChannelAttention</td><td>12</td><td>5</td></tr><tr class=""><td>VisionChannelBlock</td><td>12</td><td>5</td></tr><tr class=""><td>VisionSpatialBlock</td><td>12</td><td>5</td></tr><tr class=""><td>VisionWindowAttention</td><td>12</td><td>5</td></tr><tr class=""><td>ModuleList</td><td>8</td><td>8</td></tr><tr class=""><td>BartDecoderLayer</td><td>6</td><td>1</td></tr><tr class=""><td>BartEncoderLayer</td><td>6</td><td>1</td></tr><tr class=""><td>VisionConvEmbed</td><td>4</td><td>4</td></tr><tr class=""><td>BartScaledWordEmbedding</td><td>3</td><td>3</td></tr><tr class=""><td>BartLearnedPositionalEmbedding</td><td>2</td><td>2</td></tr><tr class=""><td>Embedding</td><td>2</td><td>2</td></tr><tr class=""><td>Identity</td><td>2</td><td>2</td></tr><tr class=""><td>BartDecoder</td><td>1</td><td>1</td></tr><tr class=""><td>BartEncoder</td><td>1</td><td>1</td></tr><tr class=""><td>BartModel</td><td>1</td><td>1</td></tr><tr class=""><td>ForConditionalGeneration</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>MultiModalProjector</td><td>1</td><td>1</td></tr><tr class=""><td>VisionBackbone</td><td>1</td><td>1</td></tr><tr class=""><td>VisionLearnedAbsolutePositionEmbedding2D</td><td>1</td><td>1</td></tr><tr class=""><td>VisionPositionalEmbeddingCosine1D</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">12 fields changed, 72 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>text_config.d_model</td><td>768</td><td>64</td></tr><tr><td>text_config.decoder_attention_heads</td><td>12</td><td>2</td></tr><tr><td>text_config.decoder_ffn_dim</td><td>3072</td><td>128</td></tr><tr><td>text_config.decoder_layers</td><td>6</td><td>1</td></tr><tr><td>text_config.encoder_attention_heads</td><td>12</td><td>2</td></tr><tr><td>text_config.encoder_ffn_dim</td><td>3072</td><td>128</td></tr><tr><td>text_config.encoder_layers</td><td>6</td><td>1</td></tr><tr><td>vision_config.depths</td><td>[1, 1, 9, 1]</td><td>[1, 1, 2, 1]</td></tr><tr><td>vision_config.embed_dim</td><td>[128, 256, 512, 1024]</td><td>[32, 64, 128, 256]</td></tr><tr><td>vision_config.num_groups</td><td>[4, 8, 16, 32]</td><td>[2, 4, 8, 16]</td></tr><tr><td>vision_config.num_heads</td><td>[4, 8, 16, 32]</td><td>[2, 4, 8, 16]</td></tr><tr><td>vision_config.projection_dim</td><td>768</td><td>64</td></tr></table>
36
  </main></body></html>
shrink/Florence-2-base-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/Florence-2-base-mic@85c1b12e1d9565763f6f2b9466669c58bc7aac33", "findings": []}
shrink/GLM-4-9B-0414-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/GLM-4-9B-0414-mic@140a76e1cd7001bc3d0aed3b715a2a902214ece6 ← zai-org/GLM-4-9B-0414@645b8482494e31b6b752272bf7f7f273ef0f3caf Β· Glm4ForCausalLM Β· random weights</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">9.4B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">19.5M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 40</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i><i class="c0" title="layer 2: DecoderLayer"></i><i class="c0" title="layer 3: DecoderLayer"></i><i class="c0" title="layer 4: DecoderLayer"></i><i class="c0" title="layer 5: DecoderLayer"></i><i class="c0" title="layer 6: DecoderLayer"></i><i class="c0" title="layer 7: DecoderLayer"></i><i class="c0" title="layer 8: DecoderLayer"></i><i class="c0" title="layer 9: DecoderLayer"></i><i class="c0" title="layer 10: DecoderLayer"></i><i class="c0" title="layer 11: DecoderLayer"></i><i class="c0" title="layer 12: DecoderLayer"></i><i class="c0" title="layer 13: DecoderLayer"></i><i class="c0" title="layer 14: DecoderLayer"></i><i class="c0" title="layer 15: DecoderLayer"></i><i class="c0" title="layer 16: DecoderLayer"></i><i class="c0" title="layer 17: DecoderLayer"></i><i class="c0" title="layer 18: DecoderLayer"></i><i class="c0" title="layer 19: DecoderLayer"></i><i class="c0" title="layer 20: DecoderLayer"></i><i class="c0" title="layer 21: DecoderLayer"></i><i class="c0" title="layer 22: DecoderLayer"></i><i class="c0" title="layer 23: DecoderLayer"></i><i class="c0" title="layer 24: DecoderLayer"></i><i class="c0" title="layer 25: DecoderLayer"></i><i class="c0" title="layer 26: DecoderLayer"></i><i class="c0" title="layer 27: DecoderLayer"></i><i class="c0" title="layer 28: DecoderLayer"></i><i class="c0" title="layer 29: DecoderLayer"></i><i class="c0" title="layer 30: DecoderLayer"></i><i class="c0" title="layer 31: DecoderLayer"></i><i class="c0" title="layer 32: DecoderLayer"></i><i class="c0" title="layer 33: DecoderLayer"></i><i class="c0" title="layer 34: DecoderLayer"></i><i class="c0" title="layer 35: DecoderLayer"></i><i class="c0" title="layer 36: DecoderLayer"></i><i class="c0" title="layer 37: DecoderLayer"></i><i class="c0" title="layer 38: DecoderLayer"></i><i class="c0" title="layer 39: DecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>DecoderLayer</td><td>40</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.867928" title="model.layers: 8.2B (86.8%)"></i><i class="c1" style="flex:0.066036" title="model.embed_tokens: 620.8M (6.6%)"></i><i class="c2" style="flex:0.066036" title="lm_head: 620.8M (6.6%)"></i><i class="c3" style="flex:0.000000" title="model.norm: 4.1K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.003826" title="model.layers: 74.5K (0.4%)"></i><i class="c1" style="flex:0.498086" title="model.embed_tokens: 9.7M (49.8%)"></i><i class="c2" style="flex:0.498086" title="lm_head: 9.7M (49.8%)"></i><i class="c3" style="flex:0.000003" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>8.2B</td><td>86.8%</td><td>74.5K</td><td>0.4%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>620.8M</td><td>6.6%</td><td>9.7M</td><td>49.8%</td></tr><tr><td><i class="c2"></i>lm_head</td><td>620.8M</td><td>6.6%</td><td>9.7M</td><td>49.8%</td></tr><tr><td><i class="c3"></i>model.norm</td><td>4.1K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>GQA (32 heads, 2 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>glm4</td><td>glm4</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>151552</td><td>151552</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>32768</td><td>32768</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Glm4 omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>241</td><td>13</td></tr><tr class=""><td>RMSNorm</td><td>161</td><td>9</td></tr><tr class=""><td>Attention</td><td>40</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>40</td><td>2</td></tr><tr class=""><td>MLP</td><td>40</td><td>2</td></tr><tr class=""><td>SiLUActivation</td><td>40</td><td>2</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">5 fields changed, 23 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>head_dim</td><td>128</td><td>16</td></tr><tr><td>hidden_size</td><td>4096</td><td>64</td></tr><tr><td>intermediate_size</td><td>13696</td><td>128</td></tr><tr><td>num_attention_heads</td><td>32</td><td>4</td></tr><tr><td>num_hidden_layers</td><td>40</td><td>2</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>zai-org/GLM-4-9B-0414: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>zai-org/GLM-4-9B-0414</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/GLM-4-9B-0414-mic/tree/140a76e1cd7001bc3d0aed3b715a2a902214ece6" title="huggingface-mic/GLM-4-9B-0414-mic@140a76e1cd7001bc3d0aed3b715a2a902214ece6">huggingface-mic/GLM-4-9B-0414-mic@140a76e1cd70</a> ← <a href="https://huggingface.co/zai-org/GLM-4-9B-0414/tree/645b8482494e31b6b752272bf7f7f273ef0f3caf" title="zai-org/GLM-4-9B-0414@645b8482494e31b6b752272bf7f7f273ef0f3caf">zai-org/GLM-4-9B-0414@645b8482494e</a> Β· Glm4ForCausalLM Β· random weights</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">9.4B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">19.5M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 40</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i><i class="c0" title="layer 2: DecoderLayer"></i><i class="c0" title="layer 3: DecoderLayer"></i><i class="c0" title="layer 4: DecoderLayer"></i><i class="c0" title="layer 5: DecoderLayer"></i><i class="c0" title="layer 6: DecoderLayer"></i><i class="c0" title="layer 7: DecoderLayer"></i><i class="c0" title="layer 8: DecoderLayer"></i><i class="c0" title="layer 9: DecoderLayer"></i><i class="c0" title="layer 10: DecoderLayer"></i><i class="c0" title="layer 11: DecoderLayer"></i><i class="c0" title="layer 12: DecoderLayer"></i><i class="c0" title="layer 13: DecoderLayer"></i><i class="c0" title="layer 14: DecoderLayer"></i><i class="c0" title="layer 15: DecoderLayer"></i><i class="c0" title="layer 16: DecoderLayer"></i><i class="c0" title="layer 17: DecoderLayer"></i><i class="c0" title="layer 18: DecoderLayer"></i><i class="c0" title="layer 19: DecoderLayer"></i><i class="c0" title="layer 20: DecoderLayer"></i><i class="c0" title="layer 21: DecoderLayer"></i><i class="c0" title="layer 22: DecoderLayer"></i><i class="c0" title="layer 23: DecoderLayer"></i><i class="c0" title="layer 24: DecoderLayer"></i><i class="c0" title="layer 25: DecoderLayer"></i><i class="c0" title="layer 26: DecoderLayer"></i><i class="c0" title="layer 27: DecoderLayer"></i><i class="c0" title="layer 28: DecoderLayer"></i><i class="c0" title="layer 29: DecoderLayer"></i><i class="c0" title="layer 30: DecoderLayer"></i><i class="c0" title="layer 31: DecoderLayer"></i><i class="c0" title="layer 32: DecoderLayer"></i><i class="c0" title="layer 33: DecoderLayer"></i><i class="c0" title="layer 34: DecoderLayer"></i><i class="c0" title="layer 35: DecoderLayer"></i><i class="c0" title="layer 36: DecoderLayer"></i><i class="c0" title="layer 37: DecoderLayer"></i><i class="c0" title="layer 38: DecoderLayer"></i><i class="c0" title="layer 39: DecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>DecoderLayer</td><td>40</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.867928" title="model.layers: 8.2B (86.8%)"></i><i class="c1" style="flex:0.066036" title="model.embed_tokens: 620.8M (6.6%)"></i><i class="c2" style="flex:0.066036" title="lm_head: 620.8M (6.6%)"></i><i class="c3" style="flex:0.000000" title="model.norm: 4.1K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.003826" title="model.layers: 74.5K (0.4%)"></i><i class="c1" style="flex:0.498086" title="model.embed_tokens: 9.7M (49.8%)"></i><i class="c2" style="flex:0.498086" title="lm_head: 9.7M (49.8%)"></i><i class="c3" style="flex:0.000003" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>8.2B</td><td>86.8%</td><td>74.5K</td><td>0.4%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>620.8M</td><td>6.6%</td><td>9.7M</td><td>49.8%</td></tr><tr><td><i class="c2"></i>lm_head</td><td>620.8M</td><td>6.6%</td><td>9.7M</td><td>49.8%</td></tr><tr><td><i class="c3"></i>model.norm</td><td>4.1K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>GQA (32 heads, 2 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>glm4</td><td>glm4</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>151552</td><td>151552</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>32768</td><td>32768</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Glm4 omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>241</td><td>13</td></tr><tr class=""><td>RMSNorm</td><td>161</td><td>9</td></tr><tr class=""><td>Attention</td><td>40</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>40</td><td>2</td></tr><tr class=""><td>MLP</td><td>40</td><td>2</td></tr><tr class=""><td>SiLUActivation</td><td>40</td><td>2</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">5 fields changed, 23 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>head_dim</td><td>128</td><td>16</td></tr><tr><td>hidden_size</td><td>4096</td><td>64</td></tr><tr><td>intermediate_size</td><td>13696</td><td>128</td></tr><tr><td>num_attention_heads</td><td>32</td><td>4</td></tr><tr><td>num_hidden_layers</td><td>40</td><td>2</td></tr></table>
36
  </main></body></html>
shrink/GLM-4-9B-0414-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/GLM-4-9B-0414-mic@140a76e1cd7001bc3d0aed3b715a2a902214ece6", "findings": []}
shrink/GLM-4.5-Air-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/GLM-4.5-Air-mic@abc7ea97d6d633936cfd54900edce3b610755f86 ← zai-org/GLM-4.5-Air@a24ceef6ce4f3536971efe9b778bdaa1bab18daa Β· Glm4MoeForCausalLM Β· random weights</p>
33
- <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">106.9B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">19.5M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 46</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 2: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 3: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 4: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 5: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 6: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 7: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 8: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 9: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 10: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 11: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 12: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 13: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 14: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 15: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 16: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 17: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 18: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 19: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 20: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 21: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 22: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 23: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 24: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 25: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 26: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 27: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 28: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 29: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 30: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 31: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 32: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 33: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 34: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 35: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 36: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 37: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 38: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 39: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 40: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 41: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 42: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 43: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 44: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 45: Experts, MoE, TopkRouter"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, MoE, TopkRouter"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>no Experts</td><td>1</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>Experts, MoE, TopkRouter</td><td>45</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.988381" title="model.layers: 105.6B (98.8%)"></i><i class="c1" style="flex:0.005809" title="model.embed_tokens: 620.8M (0.6%)"></i><i class="c2" style="flex:0.005809" title="lm_head: 620.8M (0.6%)"></i><i class="c3" style="flex:0.000000" title="model.norm: 4.1K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.004140" title="model.layers: 80.6K (0.4%)"></i><i class="c1" style="flex:0.497928" title="model.embed_tokens: 9.7M (49.8%)"></i><i class="c2" style="flex:0.497928" title="lm_head: 9.7M (49.8%)"></i><i class="c3" style="flex:0.000003" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>105.6B</td><td>98.8%</td><td>80.6K</td><td>0.4%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>620.8M</td><td>0.6%</td><td>9.7M</td><td>49.8%</td></tr><tr><td><i class="c2"></i>lm_head</td><td>620.8M</td><td>0.6%</td><td>9.7M</td><td>49.8%</td></tr><tr><td><i class="c3"></i>model.norm</td><td>4.1K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>GQA (96 heads, 8 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-8 of 128 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>glm4_moe</td><td>glm4_moe</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>151552</td><td>151552</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>131072</td><td>131072</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Glm4Moe omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>323</td><td>15</td></tr><tr class=""><td>RMSNorm</td><td>93</td><td>5</td></tr><tr class=""><td>SiLUActivation</td><td>91</td><td>3</td></tr><tr class=""><td>Attention</td><td>46</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>46</td><td>2</td></tr><tr class=""><td>MLP</td><td>46</td><td>2</td></tr><tr class=""><td>Experts</td><td>45</td><td>1</td></tr><tr class=""><td>MoE</td><td>45</td><td>1</td></tr><tr class=""><td>TopkRouter</td><td>45</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">9 fields changed, 31 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>head_dim</td><td>128</td><td>16</td></tr><tr><td>hidden_size</td><td>4096</td><td>64</td></tr><tr><td>intermediate_size</td><td>10944</td><td>128</td></tr><tr><td>moe_intermediate_size</td><td>1408</td><td>32</td></tr><tr><td>n_routed_experts</td><td>128</td><td>4</td></tr><tr><td>num_attention_heads</td><td>96</td><td>4</td></tr><tr><td>num_experts_per_tok</td><td>8</td><td>2</td></tr><tr><td>num_hidden_layers</td><td>46</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>8</td><td>2</td></tr></table>
 
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>zai-org/GLM-4.5-Air: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>zai-org/GLM-4.5-Air</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/GLM-4.5-Air-mic/tree/abc7ea97d6d633936cfd54900edce3b610755f86" title="huggingface-mic/GLM-4.5-Air-mic@abc7ea97d6d633936cfd54900edce3b610755f86">huggingface-mic/GLM-4.5-Air-mic@abc7ea97d6d6</a> ← <a href="https://huggingface.co/zai-org/GLM-4.5-Air/tree/a24ceef6ce4f3536971efe9b778bdaa1bab18daa" title="zai-org/GLM-4.5-Air@a24ceef6ce4f3536971efe9b778bdaa1bab18daa">zai-org/GLM-4.5-Air@a24ceef6ce4f</a> Β· Glm4MoeForCausalLM Β· random weights</p>
35
+ <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">106.9B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">19.5M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 46</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 2: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 3: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 4: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 5: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 6: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 7: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 8: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 9: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 10: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 11: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 12: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 13: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 14: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 15: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 16: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 17: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 18: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 19: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 20: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 21: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 22: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 23: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 24: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 25: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 26: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 27: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 28: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 29: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 30: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 31: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 32: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 33: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 34: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 35: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 36: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 37: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 38: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 39: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 40: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 41: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 42: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 43: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 44: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 45: Experts, MoE, TopkRouter"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, MoE, TopkRouter"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>no Experts</td><td>1</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>Experts, MoE, TopkRouter</td><td>45</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.988381" title="model.layers: 105.6B (98.8%)"></i><i class="c1" style="flex:0.005809" title="lm_head: 620.8M (0.6%)"></i><i class="c2" style="flex:0.005809" title="model.embed_tokens: 620.8M (0.6%)"></i><i class="c3" style="flex:0.000000" title="model.norm: 4.1K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.004140" title="model.layers: 80.6K (0.4%)"></i><i class="c1" style="flex:0.497928" title="lm_head: 9.7M (49.8%)"></i><i class="c2" style="flex:0.497928" title="model.embed_tokens: 9.7M (49.8%)"></i><i class="c3" style="flex:0.000003" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>105.6B</td><td>98.8%</td><td>80.6K</td><td>0.4%</td></tr><tr><td><i class="c1"></i>lm_head</td><td>620.8M</td><td>0.6%</td><td>9.7M</td><td>49.8%</td></tr><tr><td><i class="c2"></i>model.embed_tokens</td><td>620.8M</td><td>0.6%</td><td>9.7M</td><td>49.8%</td></tr><tr><td><i class="c3"></i>model.norm</td><td>4.1K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>GQA (96 heads, 8 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-8 of 128 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>glm4_moe</td><td>glm4_moe</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>151552</td><td>151552</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>131072</td><td>131072</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Glm4Moe omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>323</td><td>15</td></tr><tr class=""><td>RMSNorm</td><td>93</td><td>5</td></tr><tr class=""><td>SiLUActivation</td><td>91</td><td>3</td></tr><tr class=""><td>Attention</td><td>46</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>46</td><td>2</td></tr><tr class=""><td>MLP</td><td>46</td><td>2</td></tr><tr class=""><td>Experts</td><td>45</td><td>1</td></tr><tr class=""><td>MoE</td><td>45</td><td>1</td></tr><tr class=""><td>TopkRouter</td><td>45</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">9 fields changed, 31 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>head_dim</td><td>128</td><td>16</td></tr><tr><td>hidden_size</td><td>4096</td><td>64</td></tr><tr><td>intermediate_size</td><td>10944</td><td>128</td></tr><tr><td>moe_intermediate_size</td><td>1408</td><td>32</td></tr><tr><td>n_routed_experts</td><td>128</td><td>4</td></tr><tr><td>num_attention_heads</td><td>96</td><td>4</td></tr><tr><td>num_experts_per_tok</td><td>8</td><td>2</td></tr><tr><td>num_hidden_layers</td><td>46</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>8</td><td>2</td></tr></table>
36
  </main></body></html>
shrink/GLM-4.5-Air-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/GLM-4.5-Air-mic@abc7ea97d6d633936cfd54900edce3b610755f86", "findings": []}
shrink/GLM-4.7-Flash-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/GLM-4.7-Flash-mic@8b4c9af3fc9e6a8325e3dade3558514edd2cd694 ← zai-org/GLM-4.7-Flash@7dd20894a642a0aa287e9827cb1a1f7f91386b67 Β· Glm4MoeLiteForCausalLM Β· random weights</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">29.9B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">19.9M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 47</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 2: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 3: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 4: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 5: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 6: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 7: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 8: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 9: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 10: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 11: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 12: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 13: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 14: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 15: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 16: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 17: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 18: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 19: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 20: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 21: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 22: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 23: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 24: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 25: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 26: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 27: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 28: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 29: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 30: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 31: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 32: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 33: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 34: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 35: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 36: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 37: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 38: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 39: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 40: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 41: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 42: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 43: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 44: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 45: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 46: Experts, MoE, TopkRouter"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, MoE, TopkRouter"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>no Experts</td><td>1</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>Experts, MoE, TopkRouter</td><td>46</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.978814" title="model.layers: 29.3B (97.9%)"></i><i class="c1" style="flex:0.010593" title="model.embed_tokens: 317.2M (1.1%)"></i><i class="c2" style="flex:0.010593" title="lm_head: 317.2M (1.1%)"></i><i class="c3" style="flex:0.000000" title="model.norm: 2.0K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.003837" title="model.layers: 76.4K (0.4%)"></i><i class="c1" style="flex:0.498080" title="model.embed_tokens: 9.9M (49.8%)"></i><i class="c2" style="flex:0.498080" title="lm_head: 9.9M (49.8%)"></i><i class="c3" style="flex:0.000003" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>29.3B</td><td>97.9%</td><td>76.4K</td><td>0.4%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>317.2M</td><td>1.1%</td><td>9.9M</td><td>49.8%</td></tr><tr><td><i class="c2"></i>lm_head</td><td>317.2M</td><td>1.1%</td><td>9.9M</td><td>49.8%</td></tr><tr><td><i class="c3"></i>model.norm</td><td>2.0K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>MHA (20 heads, 20 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-4 of 64 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>glm4_moe_lite</td><td>glm4_moe_lite</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>154880</td><td>154880</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>202752</td><td>202752</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Glm4MoeLite omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>377</td><td>17</td></tr><tr class=""><td>RMSNorm</td><td>189</td><td>9</td></tr><tr class=""><td>SiLUActivation</td><td>93</td><td>3</td></tr><tr class=""><td>Attention</td><td>47</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>47</td><td>2</td></tr><tr class=""><td>MLP</td><td>47</td><td>2</td></tr><tr class=""><td>Experts</td><td>46</td><td>1</td></tr><tr class=""><td>MoE</td><td>46</td><td>1</td></tr><tr class=""><td>TopkRouter</td><td>46</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">14 fields changed, 34 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>hidden_size</td><td>2048</td><td>64</td></tr><tr><td>intermediate_size</td><td>10240</td><td>128</td></tr><tr><td>kv_lora_rank</td><td>512</td><td>16</td></tr><tr><td>mlp_layer_types</td><td>47 entries: 46Γ— sparse, 1Γ— dense</td><td>[&quot;dense&quot;, &quot;sparse&quot;]</td></tr><tr><td>moe_intermediate_size</td><td>1536</td><td>32</td></tr><tr><td>n_routed_experts</td><td>64</td><td>4</td></tr><tr><td>num_attention_heads</td><td>20</td><td>4</td></tr><tr><td>num_experts_per_tok</td><td>4</td><td>2</td></tr><tr><td>num_hidden_layers</td><td>47</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>20</td><td>4</td></tr><tr><td>q_lora_rank</td><td>768</td><td>16</td></tr><tr><td>qk_nope_head_dim</td><td>192</td><td>16</td></tr><tr><td>qk_rope_head_dim</td><td>64</td><td>8</td></tr><tr><td>v_head_dim</td><td>256</td><td>16</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>zai-org/GLM-4.7-Flash: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>zai-org/GLM-4.7-Flash</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/GLM-4.7-Flash-mic/tree/8b4c9af3fc9e6a8325e3dade3558514edd2cd694" title="huggingface-mic/GLM-4.7-Flash-mic@8b4c9af3fc9e6a8325e3dade3558514edd2cd694">huggingface-mic/GLM-4.7-Flash-mic@8b4c9af3fc9e</a> ← <a href="https://huggingface.co/zai-org/GLM-4.7-Flash/tree/7dd20894a642a0aa287e9827cb1a1f7f91386b67" title="zai-org/GLM-4.7-Flash@7dd20894a642a0aa287e9827cb1a1f7f91386b67">zai-org/GLM-4.7-Flash@7dd20894a642</a> Β· Glm4MoeLiteForCausalLM Β· random weights</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">29.9B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">19.9M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 47</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 2: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 3: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 4: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 5: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 6: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 7: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 8: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 9: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 10: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 11: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 12: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 13: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 14: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 15: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 16: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 17: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 18: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 19: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 20: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 21: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 22: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 23: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 24: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 25: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 26: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 27: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 28: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 29: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 30: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 31: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 32: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 33: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 34: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 35: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 36: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 37: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 38: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 39: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 40: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 41: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 42: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 43: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 44: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 45: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 46: Experts, MoE, TopkRouter"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, MoE, TopkRouter"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>no Experts</td><td>1</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>Experts, MoE, TopkRouter</td><td>46</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.978814" title="model.layers: 29.3B (97.9%)"></i><i class="c1" style="flex:0.010593" title="model.embed_tokens: 317.2M (1.1%)"></i><i class="c2" style="flex:0.010593" title="lm_head: 317.2M (1.1%)"></i><i class="c3" style="flex:0.000000" title="model.norm: 2.0K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.003837" title="model.layers: 76.4K (0.4%)"></i><i class="c1" style="flex:0.498080" title="model.embed_tokens: 9.9M (49.8%)"></i><i class="c2" style="flex:0.498080" title="lm_head: 9.9M (49.8%)"></i><i class="c3" style="flex:0.000003" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>29.3B</td><td>97.9%</td><td>76.4K</td><td>0.4%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>317.2M</td><td>1.1%</td><td>9.9M</td><td>49.8%</td></tr><tr><td><i class="c2"></i>lm_head</td><td>317.2M</td><td>1.1%</td><td>9.9M</td><td>49.8%</td></tr><tr><td><i class="c3"></i>model.norm</td><td>2.0K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>MHA (20 heads, 20 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-4 of 64 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>glm4_moe_lite</td><td>glm4_moe_lite</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>154880</td><td>154880</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>202752</td><td>202752</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Glm4MoeLite omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>377</td><td>17</td></tr><tr class=""><td>RMSNorm</td><td>189</td><td>9</td></tr><tr class=""><td>SiLUActivation</td><td>93</td><td>3</td></tr><tr class=""><td>Attention</td><td>47</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>47</td><td>2</td></tr><tr class=""><td>MLP</td><td>47</td><td>2</td></tr><tr class=""><td>Experts</td><td>46</td><td>1</td></tr><tr class=""><td>MoE</td><td>46</td><td>1</td></tr><tr class=""><td>TopkRouter</td><td>46</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">14 fields changed, 34 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>hidden_size</td><td>2048</td><td>64</td></tr><tr><td>intermediate_size</td><td>10240</td><td>128</td></tr><tr><td>kv_lora_rank</td><td>512</td><td>16</td></tr><tr><td>mlp_layer_types</td><td>47 entries: 46Γ— sparse, 1Γ— dense</td><td>[&quot;dense&quot;, &quot;sparse&quot;]</td></tr><tr><td>moe_intermediate_size</td><td>1536</td><td>32</td></tr><tr><td>n_routed_experts</td><td>64</td><td>4</td></tr><tr><td>num_attention_heads</td><td>20</td><td>4</td></tr><tr><td>num_experts_per_tok</td><td>4</td><td>2</td></tr><tr><td>num_hidden_layers</td><td>47</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>20</td><td>4</td></tr><tr><td>q_lora_rank</td><td>768</td><td>16</td></tr><tr><td>qk_nope_head_dim</td><td>192</td><td>16</td></tr><tr><td>qk_rope_head_dim</td><td>64</td><td>8</td></tr><tr><td>v_head_dim</td><td>256</td><td>16</td></tr></table>
36
  </main></body></html>
shrink/GLM-4.7-Flash-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/GLM-4.7-Flash-mic@8b4c9af3fc9e6a8325e3dade3558514edd2cd694", "findings": []}
shrink/GLM-5.3-Flash-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/GLM-5.3-Flash-mic@53f3e5bd10a67d2c75a170a3c2a28009e338c256 ← zai-org/GLM-5.3-Flash@46a050bec13002ad29ee795c16e61cf4fa3c4a9c Β· Glm5NextForConditionalGeneration Β· random weights Β· built before the shrink rule (no recorded recipe)</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">313.9B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">40.7M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">2</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">1</span><span class="k">to review</span></div></div><section class="findings warn"><h2>⚠ To review</h2><ul><li>vision_config: image_size 448 became 56</li></ul></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.visual.blocks</h3><div class="row"><span class="lab">Original Β· 24</span><div class="cells"><i class="c0" title="layer 0: VisionBlock"></i><i class="c0" title="layer 1: VisionBlock"></i><i class="c0" title="layer 2: VisionBlock"></i><i class="c0" title="layer 3: VisionBlock"></i><i class="c0" title="layer 4: VisionBlock"></i><i class="c0" title="layer 5: VisionBlock"></i><i class="c0" title="layer 6: VisionBlock"></i><i class="c0" title="layer 7: VisionBlock"></i><i class="c0" title="layer 8: VisionBlock"></i><i class="c0" title="layer 9: VisionBlock"></i><i class="c0" title="layer 10: VisionBlock"></i><i class="c0" title="layer 11: VisionBlock"></i><i class="c0" title="layer 12: VisionBlock"></i><i class="c0" title="layer 13: VisionBlock"></i><i class="c0" title="layer 14: VisionBlock"></i><i class="c0" title="layer 15: VisionBlock"></i><i class="c0" title="layer 16: VisionBlock"></i><i class="c0" title="layer 17: VisionBlock"></i><i class="c0" title="layer 18: VisionBlock"></i><i class="c0" title="layer 19: VisionBlock"></i><i class="c0" title="layer 20: VisionBlock"></i><i class="c0" title="layer 21: VisionBlock"></i><i class="c0" title="layer 22: VisionBlock"></i><i class="c0" title="layer 23: VisionBlock"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: VisionBlock"></i><i class="c0" title="layer 1: VisionBlock"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>VisionBlock</td><td>24</td><td>2</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.language_model.layers</h3><div class="row"><span class="lab">Original Β· 45</span><div class="cells"><i class="c0" title="layer 0: linear_attention, Conv1d, TextForgetGate …"></i><i class="c0" title="layer 1: linear_attention, Conv1d, TextForgetGate …"></i><i class="c0" title="layer 2: linear_attention, Conv1d, TextForgetGate …"></i><i class="c1" title="layer 3: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 4: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 5: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 6: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 7: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 8: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 9: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 10: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 11: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 12: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 13: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 14: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 15: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 16: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 17: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 18: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 19: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 20: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 21: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 22: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 23: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 24: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 25: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 26: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 27: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 28: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 29: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 30: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 31: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 32: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 33: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 34: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 35: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 36: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 37: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 38: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 39: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 40: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 41: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 42: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 43: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 44: linear_attention, Conv1d, TextExperts …"></i></div></div><div class="row"><span class="lab">Fixture Β· 3</span><div class="cells"><i class="c0" title="layer 0: linear_attention, Conv1d, TextForgetGate …"></i><i class="c1" title="layer 1: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 2: linear_attention, Conv1d, TextExperts …"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>linear_attention, Conv1d, TextForgetGate …</td><td>3</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>is_causal=True, TextAttention, TextExperts …</td><td>11</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c2"></i>linear_attention, Conv1d, TextExperts …</td><td>31</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.994162" title="model.language_model.layers: 312.1B (99.4%)"></i><i class="c1" style="flex:0.002021" title="model.language_model: 634.4M (0.2%)"></i><i class="c2" style="flex:0.002021" title="lm_head: 634.4M (0.2%)"></i><i class="c3" style="flex:0.001284" title="model.visual.blocks: 403.0M (0.1%)"></i><i class="c4" style="flex:0.000512" title="model.visual: 160.6M (0.1%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.019919" title="model.language_model.layers: 811.0K (2.0%)"></i><i class="c1" style="flex:0.486881" title="model.language_model: 19.8M (48.7%)"></i><i class="c2" style="flex:0.486877" title="lm_head: 19.8M (48.7%)"></i><i class="c3" style="flex:0.002048" title="model.visual.blocks: 83.4K (0.2%)"></i><i class="c4" style="flex:0.004275" title="model.visual: 174.1K (0.4%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.language_model.layers</td><td>312.1B</td><td>99.4%</td><td>811.0K</td><td>2.0%</td></tr><tr><td><i class="c1"></i>model.language_model</td><td>634.4M</td><td>0.2%</td><td>19.8M</td><td>48.7%</td></tr><tr><td><i class="c2"></i>lm_head</td><td>634.4M</td><td>0.2%</td><td>19.8M</td><td>48.7%</td></tr><tr><td><i class="c3"></i>model.visual.blocks</td><td>403.0M</td><td>0.1%</td><td>83.4K</td><td>0.2%</td></tr><tr><td><i class="c4"></i>model.visual</td><td>160.6M</td><td>0.1%</td><td>174.1K</td><td>0.4%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>model_type</td><td>glm5_next</td><td>glm5_next</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>attention style</td><td>MHA (64 heads, 64 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>MoE routing</td><td>top-8 of 288 experts</td><td>top-2 of 8 experts</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>model_type</td><td>glm5_next_text</td><td>glm5_next_text</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>vocab_size</td><td>154880</td><td>154880</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>max_position_embeddings</td><td>1048576</td><td>1048576</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>RoPE type</td><td>axial</td><td>axial</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>model_type</td><td>glm5_next_vision</td><td>glm5_next_vision</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class="bad"><td>vision_config</td><td>image_size</td><td>448</td><td>56</td><td>⚠ changed</td></tr><tr class=""><td>vision_config</td><td>patch_size</td><td>14</td><td>14</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Glm5Next omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>654</td><td>50</td></tr><tr class=""><td>TextRMSNorm</td><td>113</td><td>9</td></tr><tr class=""><td>RMSNorm</td><td>97</td><td>9</td></tr><tr class=""><td>TextHyperConnection</td><td>90</td><td>6</td></tr><tr class=""><td>TextUnweightedRMSNorm</td><td>90</td><td>6</td></tr><tr class=""><td>SiLUActivation</td><td>70</td><td>6</td></tr><tr class=""><td>TextDecoderLayer</td><td>45</td><td>3</td></tr><tr class=""><td>TextMLP</td><td>45</td><td>3</td></tr><tr class=""><td>TextExperts</td><td>42</td><td>2</td></tr><tr class=""><td>TextMoE</td><td>42</td><td>2</td></tr><tr class=""><td>TextTopkRouter</td><td>42</td><td>2</td></tr><tr class=""><td>Conv1d</td><td>34</td><td>2</td></tr><tr class=""><td>TextForgetGate</td><td>34</td><td>2</td></tr><tr class=""><td>TextLinearAttention</td><td>34</td><td>2</td></tr><tr class=""><td>TextRMSNormGated</td><td>34</td><td>2</td></tr><tr class=""><td>VisionAttention</td><td>24</td><td>2</td></tr><tr class=""><td>VisionBlock</td><td>24</td><td>2</td></tr><tr class=""><td>VisionMLP</td><td>24</td><td>2</td></tr><tr class=""><td>LayerNorm</td><td>12</td><td>2</td></tr><tr class=""><td>TextAttention</td><td>11</td><td>1</td></tr><tr class=""><td>TextIndexer</td><td>11</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>2</td><td>2</td></tr><tr class=""><td>Conv2d</td><td>1</td><td>1</td></tr><tr class=""><td>Conv3d</td><td>1</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>GELU</td><td>1</td><td>1</td></tr><tr class=""><td>ForConditionalGeneration</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>TextHyperHead</td><td>1</td><td>1</td></tr><tr class=""><td>TextModel</td><td>1</td><td>1</td></tr><tr class=""><td>VisionModel</td><td>1</td><td>1</td></tr><tr class=""><td>VisionPatchEmbed</td><td>1</td><td>1</td></tr><tr class=""><td>VisionPatchMerger</td><td>1</td><td>1</td></tr><tr class=""><td>VisionRotaryEmbedding</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">33 fields changed, 86 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>quantization_config</td><td>{&quot;activation_scheme&quot;: &quot;dynamic&quot;, &quot;fmt&quot;: &quot;e4m3&quot;, &quot;modules_to_not_convert&quot;: [&quot;a…</td><td>β€”</td></tr><tr><td>text_config.first_k_dense_replace</td><td>3</td><td>1</td></tr><tr><td>text_config.hidden_size</td><td>4096</td><td>128</td></tr><tr><td>text_config.index_head_dim</td><td>128</td><td>32</td></tr><tr><td>text_config.index_n_heads</td><td>32</td><td>4</td></tr><tr><td>text_config.index_topk</td><td>2048</td><td>64</td></tr><tr><td>text_config.indexer_types</td><td>45 entries: 45Γ— full</td><td>[&quot;full&quot;, &quot;full&quot;, &quot;full&quot;]</td></tr><tr><td>text_config.intermediate_size</td><td>12288</td><td>128</td></tr><tr><td>text_config.kv_lora_rank</td><td>512</td><td>64</td></tr><tr><td>text_config.layer_types</td><td>45 entries: 34Γ— linear_attention, 11Γ— indexed_attention</td><td>[&quot;linear_attention&quot;, &quot;indexed_attention&quot;, &quot;linear_attention&quot;]</td></tr><tr><td>text_config.linear_attn_config.full_attn_layers</td><td>11 entries</td><td>[1]</td></tr><tr><td>text_config.linear_attn_config.head_dim</td><td>128</td><td>32</td></tr><tr><td>text_config.linear_attn_config.kda_layers</td><td>34 entries</td><td>[0, 2]</td></tr><tr><td>text_config.linear_attn_config.num_heads</td><td>64</td><td>4</td></tr><tr><td>text_config.linear_head_dim</td><td>128</td><td>32</td></tr><tr><td>text_config.linear_num_heads</td><td>64</td><td>4</td></tr><tr><td>text_config.mlp_layer_types</td><td>45 entries: 42Γ— sparse, 3Γ— dense</td><td>[&quot;dense&quot;, &quot;sparse&quot;, &quot;sparse&quot;]</td></tr><tr><td>text_config.moe_intermediate_size</td><td>2048</td><td>64</td></tr><tr><td>text_config.n_routed_experts</td><td>288</td><td>8</td></tr><tr><td>text_config.num_attention_heads</td><td>64</td><td>4</td></tr><tr><td>text_config.num_experts_per_tok</td><td>8</td><td>2</td></tr><tr><td>text_config.num_hidden_layers</td><td>45</td><td>3</td></tr><tr><td>text_config.num_key_value_heads</td><td>64</td><td>4</td></tr><tr><td>text_config.q_lora_rank</td><td>1536</td><td>64</td></tr><tr><td>text_config.qk_nope_head_dim</td><td>256</td><td>32</td></tr><tr><td>text_config.v_head_dim</td><td>256</td><td>32</td></tr><tr><td>vision_config.depth</td><td>24</td><td>2</td></tr><tr><td>vision_config.hidden_size</td><td>1024</td><td>64</td></tr><tr><td>vision_config.image_size</td><td>448</td><td>56</td></tr><tr><td>vision_config.intermediate_size</td><td>4096</td><td>128</td></tr><tr><td>vision_config.num_heads</td><td>16</td><td>4</td></tr><tr><td>vision_config.out_hidden_size</td><td>4096</td><td>128</td></tr><tr><td>vision_config.projection_intermediate_size</td><td>10240</td><td>128</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>zai-org/GLM-5.3-Flash: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>zai-org/GLM-5.3-Flash</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/GLM-5.3-Flash-mic/tree/53f3e5bd10a67d2c75a170a3c2a28009e338c256" title="huggingface-mic/GLM-5.3-Flash-mic@53f3e5bd10a67d2c75a170a3c2a28009e338c256">huggingface-mic/GLM-5.3-Flash-mic@53f3e5bd10a6</a> ← <a href="https://huggingface.co/zai-org/GLM-5.3-Flash/tree/46a050bec13002ad29ee795c16e61cf4fa3c4a9c" title="zai-org/GLM-5.3-Flash@46a050bec13002ad29ee795c16e61cf4fa3c4a9c">zai-org/GLM-5.3-Flash@46a050bec130</a> Β· Glm5NextForConditionalGeneration Β· random weights Β· built before the shrink rule (no recorded recipe)</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">313.9B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">40.7M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">2</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">1</span><span class="k">to review</span></div></div><section class="findings warn"><h2>⚠ To review</h2><ul><li>vision_config: image_size 448 became 56</li></ul></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.visual.blocks</h3><div class="row"><span class="lab">Original Β· 24</span><div class="cells"><i class="c0" title="layer 0: VisionBlock"></i><i class="c0" title="layer 1: VisionBlock"></i><i class="c0" title="layer 2: VisionBlock"></i><i class="c0" title="layer 3: VisionBlock"></i><i class="c0" title="layer 4: VisionBlock"></i><i class="c0" title="layer 5: VisionBlock"></i><i class="c0" title="layer 6: VisionBlock"></i><i class="c0" title="layer 7: VisionBlock"></i><i class="c0" title="layer 8: VisionBlock"></i><i class="c0" title="layer 9: VisionBlock"></i><i class="c0" title="layer 10: VisionBlock"></i><i class="c0" title="layer 11: VisionBlock"></i><i class="c0" title="layer 12: VisionBlock"></i><i class="c0" title="layer 13: VisionBlock"></i><i class="c0" title="layer 14: VisionBlock"></i><i class="c0" title="layer 15: VisionBlock"></i><i class="c0" title="layer 16: VisionBlock"></i><i class="c0" title="layer 17: VisionBlock"></i><i class="c0" title="layer 18: VisionBlock"></i><i class="c0" title="layer 19: VisionBlock"></i><i class="c0" title="layer 20: VisionBlock"></i><i class="c0" title="layer 21: VisionBlock"></i><i class="c0" title="layer 22: VisionBlock"></i><i class="c0" title="layer 23: VisionBlock"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: VisionBlock"></i><i class="c0" title="layer 1: VisionBlock"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>VisionBlock</td><td>24</td><td>2</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.language_model.layers</h3><div class="row"><span class="lab">Original Β· 45</span><div class="cells"><i class="c0" title="layer 0: linear_attention, Conv1d, TextForgetGate …"></i><i class="c0" title="layer 1: linear_attention, Conv1d, TextForgetGate …"></i><i class="c0" title="layer 2: linear_attention, Conv1d, TextForgetGate …"></i><i class="c1" title="layer 3: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 4: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 5: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 6: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 7: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 8: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 9: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 10: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 11: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 12: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 13: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 14: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 15: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 16: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 17: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 18: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 19: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 20: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 21: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 22: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 23: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 24: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 25: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 26: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 27: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 28: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 29: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 30: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 31: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 32: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 33: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 34: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 35: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 36: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 37: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 38: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 39: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 40: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 41: linear_attention, Conv1d, TextExperts …"></i><i class="c2" title="layer 42: linear_attention, Conv1d, TextExperts …"></i><i class="c1" title="layer 43: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 44: linear_attention, Conv1d, TextExperts …"></i></div></div><div class="row"><span class="lab">Fixture Β· 3</span><div class="cells"><i class="c0" title="layer 0: linear_attention, Conv1d, TextForgetGate …"></i><i class="c1" title="layer 1: is_causal=True, TextAttention, TextExperts …"></i><i class="c2" title="layer 2: linear_attention, Conv1d, TextExperts …"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>linear_attention, Conv1d, TextForgetGate …</td><td>3</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>is_causal=True, TextAttention, TextExperts …</td><td>11</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c2"></i>linear_attention, Conv1d, TextExperts …</td><td>31</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.994162" title="model.language_model.layers: 312.1B (99.4%)"></i><i class="c1" style="flex:0.002021" title="model.language_model: 634.4M (0.2%)"></i><i class="c2" style="flex:0.002021" title="lm_head: 634.4M (0.2%)"></i><i class="c3" style="flex:0.001284" title="model.visual.blocks: 403.0M (0.1%)"></i><i class="c4" style="flex:0.000512" title="model.visual: 160.6M (0.1%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.019919" title="model.language_model.layers: 811.0K (2.0%)"></i><i class="c1" style="flex:0.486881" title="model.language_model: 19.8M (48.7%)"></i><i class="c2" style="flex:0.486877" title="lm_head: 19.8M (48.7%)"></i><i class="c3" style="flex:0.002048" title="model.visual.blocks: 83.4K (0.2%)"></i><i class="c4" style="flex:0.004275" title="model.visual: 174.1K (0.4%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.language_model.layers</td><td>312.1B</td><td>99.4%</td><td>811.0K</td><td>2.0%</td></tr><tr><td><i class="c1"></i>model.language_model</td><td>634.4M</td><td>0.2%</td><td>19.8M</td><td>48.7%</td></tr><tr><td><i class="c2"></i>lm_head</td><td>634.4M</td><td>0.2%</td><td>19.8M</td><td>48.7%</td></tr><tr><td><i class="c3"></i>model.visual.blocks</td><td>403.0M</td><td>0.1%</td><td>83.4K</td><td>0.2%</td></tr><tr><td><i class="c4"></i>model.visual</td><td>160.6M</td><td>0.1%</td><td>174.1K</td><td>0.4%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>model_type</td><td>glm5_next</td><td>glm5_next</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>attention style</td><td>MHA (64 heads, 64 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>MoE routing</td><td>top-8 of 288 experts</td><td>top-2 of 8 experts</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>model_type</td><td>glm5_next_text</td><td>glm5_next_text</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>vocab_size</td><td>154880</td><td>154880</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>max_position_embeddings</td><td>1048576</td><td>1048576</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>RoPE type</td><td>axial</td><td>axial</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>model_type</td><td>glm5_next_vision</td><td>glm5_next_vision</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class="bad"><td>vision_config</td><td>image_size</td><td>448</td><td>56</td><td>⚠ changed</td></tr><tr class=""><td>vision_config</td><td>patch_size</td><td>14</td><td>14</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Glm5Next omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>654</td><td>50</td></tr><tr class=""><td>TextRMSNorm</td><td>113</td><td>9</td></tr><tr class=""><td>RMSNorm</td><td>97</td><td>9</td></tr><tr class=""><td>TextHyperConnection</td><td>90</td><td>6</td></tr><tr class=""><td>TextUnweightedRMSNorm</td><td>90</td><td>6</td></tr><tr class=""><td>SiLUActivation</td><td>70</td><td>6</td></tr><tr class=""><td>TextDecoderLayer</td><td>45</td><td>3</td></tr><tr class=""><td>TextMLP</td><td>45</td><td>3</td></tr><tr class=""><td>TextExperts</td><td>42</td><td>2</td></tr><tr class=""><td>TextMoE</td><td>42</td><td>2</td></tr><tr class=""><td>TextTopkRouter</td><td>42</td><td>2</td></tr><tr class=""><td>Conv1d</td><td>34</td><td>2</td></tr><tr class=""><td>TextForgetGate</td><td>34</td><td>2</td></tr><tr class=""><td>TextLinearAttention</td><td>34</td><td>2</td></tr><tr class=""><td>TextRMSNormGated</td><td>34</td><td>2</td></tr><tr class=""><td>VisionAttention</td><td>24</td><td>2</td></tr><tr class=""><td>VisionBlock</td><td>24</td><td>2</td></tr><tr class=""><td>VisionMLP</td><td>24</td><td>2</td></tr><tr class=""><td>LayerNorm</td><td>12</td><td>2</td></tr><tr class=""><td>TextAttention</td><td>11</td><td>1</td></tr><tr class=""><td>TextIndexer</td><td>11</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>2</td><td>2</td></tr><tr class=""><td>Conv2d</td><td>1</td><td>1</td></tr><tr class=""><td>Conv3d</td><td>1</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>GELU</td><td>1</td><td>1</td></tr><tr class=""><td>ForConditionalGeneration</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>TextHyperHead</td><td>1</td><td>1</td></tr><tr class=""><td>TextModel</td><td>1</td><td>1</td></tr><tr class=""><td>VisionModel</td><td>1</td><td>1</td></tr><tr class=""><td>VisionPatchEmbed</td><td>1</td><td>1</td></tr><tr class=""><td>VisionPatchMerger</td><td>1</td><td>1</td></tr><tr class=""><td>VisionRotaryEmbedding</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">33 fields changed, 86 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>quantization_config</td><td>{&quot;activation_scheme&quot;: &quot;dynamic&quot;, &quot;fmt&quot;: &quot;e4m3&quot;, &quot;modules_to_not_convert&quot;: [&quot;a…</td><td>β€”</td></tr><tr><td>text_config.first_k_dense_replace</td><td>3</td><td>1</td></tr><tr><td>text_config.hidden_size</td><td>4096</td><td>128</td></tr><tr><td>text_config.index_head_dim</td><td>128</td><td>32</td></tr><tr><td>text_config.index_n_heads</td><td>32</td><td>4</td></tr><tr><td>text_config.index_topk</td><td>2048</td><td>64</td></tr><tr><td>text_config.indexer_types</td><td>45 entries: 45Γ— full</td><td>[&quot;full&quot;, &quot;full&quot;, &quot;full&quot;]</td></tr><tr><td>text_config.intermediate_size</td><td>12288</td><td>128</td></tr><tr><td>text_config.kv_lora_rank</td><td>512</td><td>64</td></tr><tr><td>text_config.layer_types</td><td>45 entries: 34Γ— linear_attention, 11Γ— indexed_attention</td><td>[&quot;linear_attention&quot;, &quot;indexed_attention&quot;, &quot;linear_attention&quot;]</td></tr><tr><td>text_config.linear_attn_config.full_attn_layers</td><td>11 entries</td><td>[1]</td></tr><tr><td>text_config.linear_attn_config.head_dim</td><td>128</td><td>32</td></tr><tr><td>text_config.linear_attn_config.kda_layers</td><td>34 entries</td><td>[0, 2]</td></tr><tr><td>text_config.linear_attn_config.num_heads</td><td>64</td><td>4</td></tr><tr><td>text_config.linear_head_dim</td><td>128</td><td>32</td></tr><tr><td>text_config.linear_num_heads</td><td>64</td><td>4</td></tr><tr><td>text_config.mlp_layer_types</td><td>45 entries: 42Γ— sparse, 3Γ— dense</td><td>[&quot;dense&quot;, &quot;sparse&quot;, &quot;sparse&quot;]</td></tr><tr><td>text_config.moe_intermediate_size</td><td>2048</td><td>64</td></tr><tr><td>text_config.n_routed_experts</td><td>288</td><td>8</td></tr><tr><td>text_config.num_attention_heads</td><td>64</td><td>4</td></tr><tr><td>text_config.num_experts_per_tok</td><td>8</td><td>2</td></tr><tr><td>text_config.num_hidden_layers</td><td>45</td><td>3</td></tr><tr><td>text_config.num_key_value_heads</td><td>64</td><td>4</td></tr><tr><td>text_config.q_lora_rank</td><td>1536</td><td>64</td></tr><tr><td>text_config.qk_nope_head_dim</td><td>256</td><td>32</td></tr><tr><td>text_config.v_head_dim</td><td>256</td><td>32</td></tr><tr><td>vision_config.depth</td><td>24</td><td>2</td></tr><tr><td>vision_config.hidden_size</td><td>1024</td><td>64</td></tr><tr><td>vision_config.image_size</td><td>448</td><td>56</td></tr><tr><td>vision_config.intermediate_size</td><td>4096</td><td>128</td></tr><tr><td>vision_config.num_heads</td><td>16</td><td>4</td></tr><tr><td>vision_config.out_hidden_size</td><td>4096</td><td>128</td></tr><tr><td>vision_config.projection_intermediate_size</td><td>10240</td><td>128</td></tr></table>
36
  </main></body></html>
shrink/GLM-5.3-Flash-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/GLM-5.3-Flash-mic@53f3e5bd10a67d2c75a170a3c2a28009e338c256", "findings": ["vision_config: image_size 448 became 56"]}
shrink/GLM-5.3-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/GLM-5.3-mic@1088c4f5a18bf3ba097240a67f6c391be8072006 ← zai-org/GLM-5.3@ba32f8c19c8738e2c57c71798c601d477a7bf844 Β· GlmMoeDsaForCausalLM Β· random weights</p>
33
- <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">743.4B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">41.0M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">1</span><span class="k">to review</span></div></div><section class="findings warn"><h2>⚠ To review</h2><ul><li>model: max_position_embeddings 1048576 became 512</li></ul></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 78</span><div class="cells"><i class="c0" title="layer 0: Indexer, LayerNorm"></i><i class="c0" title="layer 1: Indexer, LayerNorm"></i><i class="c0" title="layer 2: Indexer, LayerNorm"></i><i class="c1" title="layer 3: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 4: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 5: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 6: Experts, Indexer, MoE …"></i><i class="c1" title="layer 7: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 8: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 9: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 10: Experts, Indexer, MoE …"></i><i class="c1" title="layer 11: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 12: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 13: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 14: Experts, Indexer, MoE …"></i><i class="c1" title="layer 15: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 16: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 17: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 18: Experts, Indexer, MoE …"></i><i class="c1" title="layer 19: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 20: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 21: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 22: Experts, Indexer, MoE …"></i><i class="c1" title="layer 23: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 24: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 25: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 26: Experts, Indexer, MoE …"></i><i class="c1" title="layer 27: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 28: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 29: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 30: Experts, Indexer, MoE …"></i><i class="c1" title="layer 31: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 32: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 33: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 34: Experts, Indexer, MoE …"></i><i class="c1" title="layer 35: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 36: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 37: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 38: Experts, Indexer, MoE …"></i><i class="c1" title="layer 39: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 40: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 41: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 42: Experts, Indexer, MoE …"></i><i class="c1" title="layer 43: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 44: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 45: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 46: Experts, Indexer, MoE …"></i><i class="c1" title="layer 47: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 48: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 49: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 50: Experts, Indexer, MoE …"></i><i class="c1" title="layer 51: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 52: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 53: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 54: Experts, Indexer, MoE …"></i><i class="c1" title="layer 55: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 56: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 57: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 58: Experts, Indexer, MoE …"></i><i class="c1" title="layer 59: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 60: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 61: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 62: Experts, Indexer, MoE …"></i><i class="c1" title="layer 63: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 64: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 65: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 66: Experts, Indexer, MoE …"></i><i class="c1" title="layer 67: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 68: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 69: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 70: Experts, Indexer, MoE …"></i><i class="c1" title="layer 71: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 72: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 73: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 74: Experts, Indexer, MoE …"></i><i class="c1" title="layer 75: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 76: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 77: Experts, MoE, TopkRouter"></i></div></div><div class="row"><span class="lab">Fixture Β· 7</span><div class="cells"><i class="c0" title="layer 0: Indexer, LayerNorm"></i><i class="c0" title="layer 1: Indexer, LayerNorm"></i><i class="c0" title="layer 2: Indexer, LayerNorm"></i><i class="c1" title="layer 3: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 4: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 5: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 6: Experts, Indexer, MoE …"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>Indexer, LayerNorm</td><td>3</td><td>3</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>Experts, MoE, TopkRouter</td><td>57</td><td>3</td><td>βœ“ kept</td></tr><tr><td><i class="c2"></i>Experts, Indexer, MoE …</td><td>18</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.997440" title="model.layers: 741.5B (99.7%)"></i><i class="c1" style="flex:0.001280" title="lm_head: 951.6M (0.1%)"></i><i class="c2" style="flex:0.001280" title="model.embed_tokens: 951.6M (0.1%)"></i><i class="c3" style="flex:0.000000" title="model.norm: 6.1K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.033438" title="model.layers: 1.4M (3.3%)"></i><i class="c1" style="flex:0.483279" title="lm_head: 19.8M (48.3%)"></i><i class="c2" style="flex:0.483279" title="model.embed_tokens: 19.8M (48.3%)"></i><i class="c3" style="flex:0.000003" title="model.norm: 128 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>741.5B</td><td>99.7%</td><td>1.4M</td><td>3.3%</td></tr><tr><td><i class="c1"></i>lm_head</td><td>951.6M</td><td>0.1%</td><td>19.8M</td><td>48.3%</td></tr><tr><td><i class="c2"></i>model.embed_tokens</td><td>951.6M</td><td>0.1%</td><td>19.8M</td><td>48.3%</td></tr><tr><td><i class="c3"></i>model.norm</td><td>6.1K</td><td>0.0%</td><td>128</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>MHA (64 heads, 64 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-8 of 256 experts</td><td>top-2 of 8 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>glm_moe_dsa</td><td>glm_moe_dsa</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>154880</td><td>154880</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class="bad"><td>model</td><td>max_position_embeddings</td><td>1048576</td><td>512</td><td>⚠ changed</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix GlmMoeDsa omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>688</td><td>69</td></tr><tr class=""><td>RMSNorm</td><td>313</td><td>29</td></tr><tr class=""><td>SiLUActivation</td><td>153</td><td>11</td></tr><tr class=""><td>Attention</td><td>78</td><td>7</td></tr><tr class=""><td>DecoderLayer</td><td>78</td><td>7</td></tr><tr class=""><td>MLP</td><td>78</td><td>7</td></tr><tr class=""><td>Experts</td><td>75</td><td>4</td></tr><tr class=""><td>MoE</td><td>75</td><td>4</td></tr><tr class=""><td>TopkRouter</td><td>75</td><td>4</td></tr><tr class=""><td>Indexer</td><td>21</td><td>4</td></tr><tr class=""><td>LayerNorm</td><td>21</td><td>4</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">23 fields changed, 42 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>first_k_dense_replace</td><td>3</td><td>1</td></tr><tr><td>head_dim</td><td>64</td><td>8</td></tr><tr><td>hidden_size</td><td>6144</td><td>128</td></tr><tr><td>index_head_dim</td><td>128</td><td>16</td></tr><tr><td>index_n_heads</td><td>32</td><td>2</td></tr><tr><td>index_topk</td><td>2048</td><td>8</td></tr><tr><td>indexer_types</td><td>78 entries: 57Γ— shared, 21Γ— full</td><td>7 entries: 4Γ— full, 3Γ— shared</td></tr><tr><td>intermediate_size</td><td>12288</td><td>128</td></tr><tr><td>kv_lora_rank</td><td>512</td><td>32</td></tr><tr><td>layer_types</td><td>78 entries: 78Γ— indexed_attention</td><td>7 entries: 7Γ— indexed_attention</td></tr><tr><td>max_position_embeddings</td><td>1048576</td><td>512</td></tr><tr><td>mlp_layer_types</td><td>78 entries: 75Γ— sparse, 3Γ— dense</td><td>7 entries: 4Γ— sparse, 3Γ— dense</td></tr><tr><td>moe_intermediate_size</td><td>2048</td><td>64</td></tr><tr><td>n_routed_experts</td><td>256</td><td>8</td></tr><tr><td>num_attention_heads</td><td>64</td><td>4</td></tr><tr><td>num_experts_per_tok</td><td>8</td><td>2</td></tr><tr><td>num_hidden_layers</td><td>78</td><td>7</td></tr><tr><td>num_key_value_heads</td><td>64</td><td>4</td></tr><tr><td>q_lora_rank</td><td>2048</td><td>64</td></tr><tr><td>qk_nope_head_dim</td><td>192</td><td>24</td></tr><tr><td>qk_rope_head_dim</td><td>64</td><td>8</td></tr><tr><td>quantization_config</td><td>{&quot;activation_scheme&quot;: &quot;dynamic&quot;, &quot;fmt&quot;: &quot;e4m3&quot;, &quot;quant_method&quot;: &quot;fp8&quot;, &quot;weigh…</td><td>β€”</td></tr><tr><td>v_head_dim</td><td>256</td><td>32</td></tr></table>
 
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>zai-org/GLM-5.3: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>zai-org/GLM-5.3</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/GLM-5.3-mic/tree/1088c4f5a18bf3ba097240a67f6c391be8072006" title="huggingface-mic/GLM-5.3-mic@1088c4f5a18bf3ba097240a67f6c391be8072006">huggingface-mic/GLM-5.3-mic@1088c4f5a18b</a> ← <a href="https://huggingface.co/zai-org/GLM-5.3/tree/ba32f8c19c8738e2c57c71798c601d477a7bf844" title="zai-org/GLM-5.3@ba32f8c19c8738e2c57c71798c601d477a7bf844">zai-org/GLM-5.3@ba32f8c19c87</a> Β· GlmMoeDsaForCausalLM Β· random weights</p>
35
+ <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">743.4B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">41.0M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">1</span><span class="k">to review</span></div></div><section class="findings warn"><h2>⚠ To review</h2><ul><li>model: max_position_embeddings 1048576 became 512</li></ul></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 78</span><div class="cells"><i class="c0" title="layer 0: Indexer, LayerNorm"></i><i class="c0" title="layer 1: Indexer, LayerNorm"></i><i class="c0" title="layer 2: Indexer, LayerNorm"></i><i class="c1" title="layer 3: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 4: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 5: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 6: Experts, Indexer, MoE …"></i><i class="c1" title="layer 7: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 8: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 9: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 10: Experts, Indexer, MoE …"></i><i class="c1" title="layer 11: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 12: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 13: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 14: Experts, Indexer, MoE …"></i><i class="c1" title="layer 15: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 16: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 17: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 18: Experts, Indexer, MoE …"></i><i class="c1" title="layer 19: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 20: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 21: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 22: Experts, Indexer, MoE …"></i><i class="c1" title="layer 23: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 24: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 25: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 26: Experts, Indexer, MoE …"></i><i class="c1" title="layer 27: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 28: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 29: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 30: Experts, Indexer, MoE …"></i><i class="c1" title="layer 31: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 32: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 33: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 34: Experts, Indexer, MoE …"></i><i class="c1" title="layer 35: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 36: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 37: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 38: Experts, Indexer, MoE …"></i><i class="c1" title="layer 39: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 40: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 41: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 42: Experts, Indexer, MoE …"></i><i class="c1" title="layer 43: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 44: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 45: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 46: Experts, Indexer, MoE …"></i><i class="c1" title="layer 47: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 48: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 49: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 50: Experts, Indexer, MoE …"></i><i class="c1" title="layer 51: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 52: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 53: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 54: Experts, Indexer, MoE …"></i><i class="c1" title="layer 55: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 56: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 57: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 58: Experts, Indexer, MoE …"></i><i class="c1" title="layer 59: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 60: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 61: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 62: Experts, Indexer, MoE …"></i><i class="c1" title="layer 63: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 64: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 65: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 66: Experts, Indexer, MoE …"></i><i class="c1" title="layer 67: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 68: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 69: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 70: Experts, Indexer, MoE …"></i><i class="c1" title="layer 71: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 72: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 73: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 74: Experts, Indexer, MoE …"></i><i class="c1" title="layer 75: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 76: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 77: Experts, MoE, TopkRouter"></i></div></div><div class="row"><span class="lab">Fixture Β· 7</span><div class="cells"><i class="c0" title="layer 0: Indexer, LayerNorm"></i><i class="c0" title="layer 1: Indexer, LayerNorm"></i><i class="c0" title="layer 2: Indexer, LayerNorm"></i><i class="c1" title="layer 3: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 4: Experts, MoE, TopkRouter"></i><i class="c1" title="layer 5: Experts, MoE, TopkRouter"></i><i class="c2" title="layer 6: Experts, Indexer, MoE …"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>Indexer, LayerNorm</td><td>3</td><td>3</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>Experts, MoE, TopkRouter</td><td>57</td><td>3</td><td>βœ“ kept</td></tr><tr><td><i class="c2"></i>Experts, Indexer, MoE …</td><td>18</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.997440" title="model.layers: 741.5B (99.7%)"></i><i class="c1" style="flex:0.001280" title="model.embed_tokens: 951.6M (0.1%)"></i><i class="c2" style="flex:0.001280" title="lm_head: 951.6M (0.1%)"></i><i class="c3" style="flex:0.000000" title="model.norm: 6.1K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.033438" title="model.layers: 1.4M (3.3%)"></i><i class="c1" style="flex:0.483279" title="model.embed_tokens: 19.8M (48.3%)"></i><i class="c2" style="flex:0.483279" title="lm_head: 19.8M (48.3%)"></i><i class="c3" style="flex:0.000003" title="model.norm: 128 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>741.5B</td><td>99.7%</td><td>1.4M</td><td>3.3%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>951.6M</td><td>0.1%</td><td>19.8M</td><td>48.3%</td></tr><tr><td><i class="c2"></i>lm_head</td><td>951.6M</td><td>0.1%</td><td>19.8M</td><td>48.3%</td></tr><tr><td><i class="c3"></i>model.norm</td><td>6.1K</td><td>0.0%</td><td>128</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>MHA (64 heads, 64 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-8 of 256 experts</td><td>top-2 of 8 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>glm_moe_dsa</td><td>glm_moe_dsa</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>154880</td><td>154880</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class="bad"><td>model</td><td>max_position_embeddings</td><td>1048576</td><td>512</td><td>⚠ changed</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix GlmMoeDsa omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>688</td><td>69</td></tr><tr class=""><td>RMSNorm</td><td>313</td><td>29</td></tr><tr class=""><td>SiLUActivation</td><td>153</td><td>11</td></tr><tr class=""><td>Attention</td><td>78</td><td>7</td></tr><tr class=""><td>DecoderLayer</td><td>78</td><td>7</td></tr><tr class=""><td>MLP</td><td>78</td><td>7</td></tr><tr class=""><td>Experts</td><td>75</td><td>4</td></tr><tr class=""><td>MoE</td><td>75</td><td>4</td></tr><tr class=""><td>TopkRouter</td><td>75</td><td>4</td></tr><tr class=""><td>Indexer</td><td>21</td><td>4</td></tr><tr class=""><td>LayerNorm</td><td>21</td><td>4</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">23 fields changed, 42 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>first_k_dense_replace</td><td>3</td><td>1</td></tr><tr><td>head_dim</td><td>64</td><td>8</td></tr><tr><td>hidden_size</td><td>6144</td><td>128</td></tr><tr><td>index_head_dim</td><td>128</td><td>16</td></tr><tr><td>index_n_heads</td><td>32</td><td>2</td></tr><tr><td>index_topk</td><td>2048</td><td>8</td></tr><tr><td>indexer_types</td><td>78 entries: 57Γ— shared, 21Γ— full</td><td>7 entries: 4Γ— full, 3Γ— shared</td></tr><tr><td>intermediate_size</td><td>12288</td><td>128</td></tr><tr><td>kv_lora_rank</td><td>512</td><td>32</td></tr><tr><td>layer_types</td><td>78 entries: 78Γ— indexed_attention</td><td>7 entries: 7Γ— indexed_attention</td></tr><tr><td>max_position_embeddings</td><td>1048576</td><td>512</td></tr><tr><td>mlp_layer_types</td><td>78 entries: 75Γ— sparse, 3Γ— dense</td><td>7 entries: 4Γ— sparse, 3Γ— dense</td></tr><tr><td>moe_intermediate_size</td><td>2048</td><td>64</td></tr><tr><td>n_routed_experts</td><td>256</td><td>8</td></tr><tr><td>num_attention_heads</td><td>64</td><td>4</td></tr><tr><td>num_experts_per_tok</td><td>8</td><td>2</td></tr><tr><td>num_hidden_layers</td><td>78</td><td>7</td></tr><tr><td>num_key_value_heads</td><td>64</td><td>4</td></tr><tr><td>q_lora_rank</td><td>2048</td><td>64</td></tr><tr><td>qk_nope_head_dim</td><td>192</td><td>24</td></tr><tr><td>qk_rope_head_dim</td><td>64</td><td>8</td></tr><tr><td>quantization_config</td><td>{&quot;activation_scheme&quot;: &quot;dynamic&quot;, &quot;fmt&quot;: &quot;e4m3&quot;, &quot;quant_method&quot;: &quot;fp8&quot;, &quot;weigh…</td><td>β€”</td></tr><tr><td>v_head_dim</td><td>256</td><td>32</td></tr></table>
36
  </main></body></html>
shrink/GLM-5.3-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/GLM-5.3-mic@1088c4f5a18bf3ba097240a67f6c391be8072006", "findings": ["model: max_position_embeddings 1048576 became 512"]}
shrink/GLM-OCR-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/GLM-OCR-mic@08eddf7ae838d5d3b6b135b984c0e3ec53c28441 ← zai-org/GLM-OCR@2e85a62840ccac27daa451df36c736c4636b8628 Β· GlmOcrForConditionalGeneration Β· random weights</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">1.1B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">8.8M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">2</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.visual.blocks</h3><div class="row"><span class="lab">Original Β· 24</span><div class="cells"><i class="c0" title="layer 0: VisionBlock"></i><i class="c0" title="layer 1: VisionBlock"></i><i class="c0" title="layer 2: VisionBlock"></i><i class="c0" title="layer 3: VisionBlock"></i><i class="c0" title="layer 4: VisionBlock"></i><i class="c0" title="layer 5: VisionBlock"></i><i class="c0" title="layer 6: VisionBlock"></i><i class="c0" title="layer 7: VisionBlock"></i><i class="c0" title="layer 8: VisionBlock"></i><i class="c0" title="layer 9: VisionBlock"></i><i class="c0" title="layer 10: VisionBlock"></i><i class="c0" title="layer 11: VisionBlock"></i><i class="c0" title="layer 12: VisionBlock"></i><i class="c0" title="layer 13: VisionBlock"></i><i class="c0" title="layer 14: VisionBlock"></i><i class="c0" title="layer 15: VisionBlock"></i><i class="c0" title="layer 16: VisionBlock"></i><i class="c0" title="layer 17: VisionBlock"></i><i class="c0" title="layer 18: VisionBlock"></i><i class="c0" title="layer 19: VisionBlock"></i><i class="c0" title="layer 20: VisionBlock"></i><i class="c0" title="layer 21: VisionBlock"></i><i class="c0" title="layer 22: VisionBlock"></i><i class="c0" title="layer 23: VisionBlock"></i></div></div><div class="row"><span class="lab">Fixture Β· 24</span><div class="cells"><i class="c0" title="layer 0: VisionBlock"></i><i class="c0" title="layer 1: VisionBlock"></i><i class="c0" title="layer 2: VisionBlock"></i><i class="c0" title="layer 3: VisionBlock"></i><i class="c0" title="layer 4: VisionBlock"></i><i class="c0" title="layer 5: VisionBlock"></i><i class="c0" title="layer 6: VisionBlock"></i><i class="c0" title="layer 7: VisionBlock"></i><i class="c0" title="layer 8: VisionBlock"></i><i class="c0" title="layer 9: VisionBlock"></i><i class="c0" title="layer 10: VisionBlock"></i><i class="c0" title="layer 11: VisionBlock"></i><i class="c0" title="layer 12: VisionBlock"></i><i class="c0" title="layer 13: VisionBlock"></i><i class="c0" title="layer 14: VisionBlock"></i><i class="c0" title="layer 15: VisionBlock"></i><i class="c0" title="layer 16: VisionBlock"></i><i class="c0" title="layer 17: VisionBlock"></i><i class="c0" title="layer 18: VisionBlock"></i><i class="c0" title="layer 19: VisionBlock"></i><i class="c0" title="layer 20: VisionBlock"></i><i class="c0" title="layer 21: VisionBlock"></i><i class="c0" title="layer 22: VisionBlock"></i><i class="c0" title="layer 23: VisionBlock"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>VisionBlock</td><td>24</td><td>24</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.language_model.layers</h3><div class="row"><span class="lab">Original Β· 16</span><div class="cells"><i class="c0" title="layer 0: TextDecoderLayer"></i><i class="c0" title="layer 1: TextDecoderLayer"></i><i class="c0" title="layer 2: TextDecoderLayer"></i><i class="c0" title="layer 3: TextDecoderLayer"></i><i class="c0" title="layer 4: TextDecoderLayer"></i><i class="c0" title="layer 5: TextDecoderLayer"></i><i class="c0" title="layer 6: TextDecoderLayer"></i><i class="c0" title="layer 7: TextDecoderLayer"></i><i class="c0" title="layer 8: TextDecoderLayer"></i><i class="c0" title="layer 9: TextDecoderLayer"></i><i class="c0" title="layer 10: TextDecoderLayer"></i><i class="c0" title="layer 11: TextDecoderLayer"></i><i class="c0" title="layer 12: TextDecoderLayer"></i><i class="c0" title="layer 13: TextDecoderLayer"></i><i class="c0" title="layer 14: TextDecoderLayer"></i><i class="c0" title="layer 15: TextDecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: TextDecoderLayer"></i><i class="c0" title="layer 1: TextDecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>TextDecoderLayer</td><td>16</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.443227" title="model.language_model.layers: 490.8M (44.3%)"></i><i class="c1" style="flex:0.363936" title="model.visual.blocks: 403.0M (36.4%)"></i><i class="c2" style="flex:0.082380" title="model.language_model: 91.2M (8.2%)"></i><i class="c3" style="flex:0.082378" title="lm_head: 91.2M (8.2%)"></i><i class="c4" style="flex:0.028079" title="model.visual: 31.1M (2.8%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.008427" title="model.language_model.layers: 74.2K (0.8%)"></i><i class="c1" style="flex:0.113586" title="model.visual.blocks: 1.0M (11.4%)"></i><i class="c2" style="flex:0.431453" title="model.language_model: 3.8M (43.1%)"></i><i class="c3" style="flex:0.431446" title="lm_head: 3.8M (43.1%)"></i><i class="c4" style="flex:0.015088" title="model.visual: 132.9K (1.5%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.language_model.layers</td><td>490.8M</td><td>44.3%</td><td>74.2K</td><td>0.8%</td></tr><tr><td><i class="c1"></i>model.visual.blocks</td><td>403.0M</td><td>36.4%</td><td>1.0M</td><td>11.4%</td></tr><tr><td><i class="c2"></i>model.language_model</td><td>91.2M</td><td>8.2%</td><td>3.8M</td><td>43.1%</td></tr><tr><td><i class="c3"></i>lm_head</td><td>91.2M</td><td>8.2%</td><td>3.8M</td><td>43.1%</td></tr><tr><td><i class="c4"></i>model.visual</td><td>31.1M</td><td>2.8%</td><td>132.9K</td><td>1.5%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>model_type</td><td>glm_ocr</td><td>glm_ocr</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>attention style</td><td>GQA (16 heads, 8 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>model_type</td><td>glm_ocr_text</td><td>glm_ocr_text</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>vocab_size</td><td>59392</td><td>59392</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>max_position_embeddings</td><td>131072</td><td>131072</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>RoPE type</td><td>axial</td><td>axial</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>model_type</td><td>glm_ocr_vision</td><td>glm_ocr_vision</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>image_size</td><td>336</td><td>336</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>patch_size</td><td>14</td><td>14</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix GlmOcr omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>221</td><td>137</td></tr><tr class=""><td>RMSNorm</td><td>162</td><td>106</td></tr><tr class=""><td>SiLUActivation</td><td>41</td><td>27</td></tr><tr class=""><td>VisionAttention</td><td>24</td><td>24</td></tr><tr class=""><td>VisionBlock</td><td>24</td><td>24</td></tr><tr class=""><td>VisionMlp</td><td>24</td><td>24</td></tr><tr class=""><td>TextAttention</td><td>16</td><td>2</td></tr><tr class=""><td>TextDecoderLayer</td><td>16</td><td>2</td></tr><tr class=""><td>TextMLP</td><td>16</td><td>2</td></tr><tr class=""><td>ModuleList</td><td>2</td><td>2</td></tr><tr class=""><td>Conv2d</td><td>1</td><td>1</td></tr><tr class=""><td>Conv3d</td><td>1</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>GELU</td><td>1</td><td>1</td></tr><tr class=""><td>ForConditionalGeneration</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>TextModel</td><td>1</td><td>1</td></tr><tr class=""><td>TextRotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>VisionModel</td><td>1</td><td>1</td></tr><tr class=""><td>VisionPatchEmbed</td><td>1</td><td>1</td></tr><tr class=""><td>VisionPatchMerger</td><td>1</td><td>1</td></tr><tr class=""><td>VisionRotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>LayerNorm</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">11 fields changed, 59 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>text_config.head_dim</td><td>128</td><td>16</td></tr><tr><td>text_config.hidden_size</td><td>1536</td><td>64</td></tr><tr><td>text_config.intermediate_size</td><td>4608</td><td>128</td></tr><tr><td>text_config.num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>text_config.num_hidden_layers</td><td>16</td><td>2</td></tr><tr><td>text_config.num_key_value_heads</td><td>8</td><td>2</td></tr><tr><td>text_config.rope_parameters</td><td>{&quot;mrope_section&quot;: [16, 24, 24], &quot;partial_rotary_factor&quot;: 1.0, &quot;rope_theta&quot;: 1…</td><td>{&quot;mrope_section&quot;: [2, 3, 3], &quot;partial_rotary_factor&quot;: 1.0, &quot;rope_theta&quot;: 1000…</td></tr><tr><td>vision_config.hidden_size</td><td>1024</td><td>64</td></tr><tr><td>vision_config.intermediate_size</td><td>4096</td><td>128</td></tr><tr><td>vision_config.num_heads</td><td>16</td><td>4</td></tr><tr><td>vision_config.out_hidden_size</td><td>1536</td><td>64</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>zai-org/GLM-OCR: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>zai-org/GLM-OCR</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/GLM-OCR-mic/tree/08eddf7ae838d5d3b6b135b984c0e3ec53c28441" title="huggingface-mic/GLM-OCR-mic@08eddf7ae838d5d3b6b135b984c0e3ec53c28441">huggingface-mic/GLM-OCR-mic@08eddf7ae838</a> ← <a href="https://huggingface.co/zai-org/GLM-OCR/tree/2e85a62840ccac27daa451df36c736c4636b8628" title="zai-org/GLM-OCR@2e85a62840ccac27daa451df36c736c4636b8628">zai-org/GLM-OCR@2e85a62840cc</a> Β· GlmOcrForConditionalGeneration Β· random weights</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">1.1B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">8.8M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">2</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.visual.blocks</h3><div class="row"><span class="lab">Original Β· 24</span><div class="cells"><i class="c0" title="layer 0: VisionBlock"></i><i class="c0" title="layer 1: VisionBlock"></i><i class="c0" title="layer 2: VisionBlock"></i><i class="c0" title="layer 3: VisionBlock"></i><i class="c0" title="layer 4: VisionBlock"></i><i class="c0" title="layer 5: VisionBlock"></i><i class="c0" title="layer 6: VisionBlock"></i><i class="c0" title="layer 7: VisionBlock"></i><i class="c0" title="layer 8: VisionBlock"></i><i class="c0" title="layer 9: VisionBlock"></i><i class="c0" title="layer 10: VisionBlock"></i><i class="c0" title="layer 11: VisionBlock"></i><i class="c0" title="layer 12: VisionBlock"></i><i class="c0" title="layer 13: VisionBlock"></i><i class="c0" title="layer 14: VisionBlock"></i><i class="c0" title="layer 15: VisionBlock"></i><i class="c0" title="layer 16: VisionBlock"></i><i class="c0" title="layer 17: VisionBlock"></i><i class="c0" title="layer 18: VisionBlock"></i><i class="c0" title="layer 19: VisionBlock"></i><i class="c0" title="layer 20: VisionBlock"></i><i class="c0" title="layer 21: VisionBlock"></i><i class="c0" title="layer 22: VisionBlock"></i><i class="c0" title="layer 23: VisionBlock"></i></div></div><div class="row"><span class="lab">Fixture Β· 24</span><div class="cells"><i class="c0" title="layer 0: VisionBlock"></i><i class="c0" title="layer 1: VisionBlock"></i><i class="c0" title="layer 2: VisionBlock"></i><i class="c0" title="layer 3: VisionBlock"></i><i class="c0" title="layer 4: VisionBlock"></i><i class="c0" title="layer 5: VisionBlock"></i><i class="c0" title="layer 6: VisionBlock"></i><i class="c0" title="layer 7: VisionBlock"></i><i class="c0" title="layer 8: VisionBlock"></i><i class="c0" title="layer 9: VisionBlock"></i><i class="c0" title="layer 10: VisionBlock"></i><i class="c0" title="layer 11: VisionBlock"></i><i class="c0" title="layer 12: VisionBlock"></i><i class="c0" title="layer 13: VisionBlock"></i><i class="c0" title="layer 14: VisionBlock"></i><i class="c0" title="layer 15: VisionBlock"></i><i class="c0" title="layer 16: VisionBlock"></i><i class="c0" title="layer 17: VisionBlock"></i><i class="c0" title="layer 18: VisionBlock"></i><i class="c0" title="layer 19: VisionBlock"></i><i class="c0" title="layer 20: VisionBlock"></i><i class="c0" title="layer 21: VisionBlock"></i><i class="c0" title="layer 22: VisionBlock"></i><i class="c0" title="layer 23: VisionBlock"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>VisionBlock</td><td>24</td><td>24</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.language_model.layers</h3><div class="row"><span class="lab">Original Β· 16</span><div class="cells"><i class="c0" title="layer 0: TextDecoderLayer"></i><i class="c0" title="layer 1: TextDecoderLayer"></i><i class="c0" title="layer 2: TextDecoderLayer"></i><i class="c0" title="layer 3: TextDecoderLayer"></i><i class="c0" title="layer 4: TextDecoderLayer"></i><i class="c0" title="layer 5: TextDecoderLayer"></i><i class="c0" title="layer 6: TextDecoderLayer"></i><i class="c0" title="layer 7: TextDecoderLayer"></i><i class="c0" title="layer 8: TextDecoderLayer"></i><i class="c0" title="layer 9: TextDecoderLayer"></i><i class="c0" title="layer 10: TextDecoderLayer"></i><i class="c0" title="layer 11: TextDecoderLayer"></i><i class="c0" title="layer 12: TextDecoderLayer"></i><i class="c0" title="layer 13: TextDecoderLayer"></i><i class="c0" title="layer 14: TextDecoderLayer"></i><i class="c0" title="layer 15: TextDecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: TextDecoderLayer"></i><i class="c0" title="layer 1: TextDecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>TextDecoderLayer</td><td>16</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.443227" title="model.language_model.layers: 490.8M (44.3%)"></i><i class="c1" style="flex:0.363936" title="model.visual.blocks: 403.0M (36.4%)"></i><i class="c2" style="flex:0.082380" title="model.language_model: 91.2M (8.2%)"></i><i class="c3" style="flex:0.082378" title="lm_head: 91.2M (8.2%)"></i><i class="c4" style="flex:0.028079" title="model.visual: 31.1M (2.8%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.008427" title="model.language_model.layers: 74.2K (0.8%)"></i><i class="c1" style="flex:0.113586" title="model.visual.blocks: 1.0M (11.4%)"></i><i class="c2" style="flex:0.431453" title="model.language_model: 3.8M (43.1%)"></i><i class="c3" style="flex:0.431446" title="lm_head: 3.8M (43.1%)"></i><i class="c4" style="flex:0.015088" title="model.visual: 132.9K (1.5%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.language_model.layers</td><td>490.8M</td><td>44.3%</td><td>74.2K</td><td>0.8%</td></tr><tr><td><i class="c1"></i>model.visual.blocks</td><td>403.0M</td><td>36.4%</td><td>1.0M</td><td>11.4%</td></tr><tr><td><i class="c2"></i>model.language_model</td><td>91.2M</td><td>8.2%</td><td>3.8M</td><td>43.1%</td></tr><tr><td><i class="c3"></i>lm_head</td><td>91.2M</td><td>8.2%</td><td>3.8M</td><td>43.1%</td></tr><tr><td><i class="c4"></i>model.visual</td><td>31.1M</td><td>2.8%</td><td>132.9K</td><td>1.5%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>model_type</td><td>glm_ocr</td><td>glm_ocr</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>attention style</td><td>GQA (16 heads, 8 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>model_type</td><td>glm_ocr_text</td><td>glm_ocr_text</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>vocab_size</td><td>59392</td><td>59392</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>max_position_embeddings</td><td>131072</td><td>131072</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>RoPE type</td><td>axial</td><td>axial</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>model_type</td><td>glm_ocr_vision</td><td>glm_ocr_vision</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>image_size</td><td>336</td><td>336</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>patch_size</td><td>14</td><td>14</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix GlmOcr omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>221</td><td>137</td></tr><tr class=""><td>RMSNorm</td><td>162</td><td>106</td></tr><tr class=""><td>SiLUActivation</td><td>41</td><td>27</td></tr><tr class=""><td>VisionAttention</td><td>24</td><td>24</td></tr><tr class=""><td>VisionBlock</td><td>24</td><td>24</td></tr><tr class=""><td>VisionMlp</td><td>24</td><td>24</td></tr><tr class=""><td>TextAttention</td><td>16</td><td>2</td></tr><tr class=""><td>TextDecoderLayer</td><td>16</td><td>2</td></tr><tr class=""><td>TextMLP</td><td>16</td><td>2</td></tr><tr class=""><td>ModuleList</td><td>2</td><td>2</td></tr><tr class=""><td>Conv2d</td><td>1</td><td>1</td></tr><tr class=""><td>Conv3d</td><td>1</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>GELU</td><td>1</td><td>1</td></tr><tr class=""><td>ForConditionalGeneration</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>TextModel</td><td>1</td><td>1</td></tr><tr class=""><td>TextRotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>VisionModel</td><td>1</td><td>1</td></tr><tr class=""><td>VisionPatchEmbed</td><td>1</td><td>1</td></tr><tr class=""><td>VisionPatchMerger</td><td>1</td><td>1</td></tr><tr class=""><td>VisionRotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>LayerNorm</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">11 fields changed, 59 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>text_config.head_dim</td><td>128</td><td>16</td></tr><tr><td>text_config.hidden_size</td><td>1536</td><td>64</td></tr><tr><td>text_config.intermediate_size</td><td>4608</td><td>128</td></tr><tr><td>text_config.num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>text_config.num_hidden_layers</td><td>16</td><td>2</td></tr><tr><td>text_config.num_key_value_heads</td><td>8</td><td>2</td></tr><tr><td>text_config.rope_parameters</td><td>{&quot;mrope_section&quot;: [16, 24, 24], &quot;partial_rotary_factor&quot;: 1.0, &quot;rope_theta&quot;: 1…</td><td>{&quot;mrope_section&quot;: [2, 3, 3], &quot;partial_rotary_factor&quot;: 1.0, &quot;rope_theta&quot;: 1000…</td></tr><tr><td>vision_config.hidden_size</td><td>1024</td><td>64</td></tr><tr><td>vision_config.intermediate_size</td><td>4096</td><td>128</td></tr><tr><td>vision_config.num_heads</td><td>16</td><td>4</td></tr><tr><td>vision_config.out_hidden_size</td><td>1536</td><td>64</td></tr></table>
36
  </main></body></html>
shrink/GLM-OCR-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/GLM-OCR-mic@08eddf7ae838d5d3b6b135b984c0e3ec53c28441", "findings": []}
shrink/Hunyuan-0.5B-Instruct-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/Hunyuan-0.5B-Instruct-mic@d7fe47dc3288bba155076d3747959365159073b0 ← tencent/Hunyuan-0.5B-Instruct@2359fb220c010e9d6d62c62d466f0eda179c2cf3 Β· HunYuanDenseV1ForCausalLM Β· random weights</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">539.0M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">7.8M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 24</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i><i class="c0" title="layer 2: DecoderLayer"></i><i class="c0" title="layer 3: DecoderLayer"></i><i class="c0" title="layer 4: DecoderLayer"></i><i class="c0" title="layer 5: DecoderLayer"></i><i class="c0" title="layer 6: DecoderLayer"></i><i class="c0" title="layer 7: DecoderLayer"></i><i class="c0" title="layer 8: DecoderLayer"></i><i class="c0" title="layer 9: DecoderLayer"></i><i class="c0" title="layer 10: DecoderLayer"></i><i class="c0" title="layer 11: DecoderLayer"></i><i class="c0" title="layer 12: DecoderLayer"></i><i class="c0" title="layer 13: DecoderLayer"></i><i class="c0" title="layer 14: DecoderLayer"></i><i class="c0" title="layer 15: DecoderLayer"></i><i class="c0" title="layer 16: DecoderLayer"></i><i class="c0" title="layer 17: DecoderLayer"></i><i class="c0" title="layer 18: DecoderLayer"></i><i class="c0" title="layer 19: DecoderLayer"></i><i class="c0" title="layer 20: DecoderLayer"></i><i class="c0" title="layer 21: DecoderLayer"></i><i class="c0" title="layer 22: DecoderLayer"></i><i class="c0" title="layer 23: DecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>DecoderLayer</td><td>24</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.770471" title="model.layers: 415.3M (77.0%)"></i><i class="c1" style="flex:0.229528" title="model.embed_tokens: 123.7M (23.0%)"></i><i class="c2" style="flex:0.000002" title="model.norm: 1.0K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.009485" title="model.layers: 74.0K (0.9%)"></i><i class="c1" style="flex:0.990506" title="model.embed_tokens: 7.7M (99.1%)"></i><i class="c2" style="flex:0.000008" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>415.3M</td><td>77.0%</td><td>74.0K</td><td>0.9%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>123.7M</td><td>23.0%</td><td>7.7M</td><td>99.1%</td></tr><tr><td><i class="c2"></i>model.norm</td><td>1.0K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>GQA (16 heads, 8 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>dynamic</td><td>dynamic</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>hunyuan_v1_dense</td><td>hunyuan_v1_dense</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>120818</td><td>120818</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>262144</td><td>262144</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix HunYuanDenseV1 omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>169</td><td>15</td></tr><tr class=""><td>RMSNorm</td><td>97</td><td>9</td></tr><tr class=""><td>Attention</td><td>24</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>24</td><td>2</td></tr><tr class=""><td>MLP</td><td>24</td><td>2</td></tr><tr class=""><td>SiLUActivation</td><td>24</td><td>2</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">7 fields changed, 43 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>attention_head_dim</td><td>128</td><td>16</td></tr><tr><td>head_dim</td><td>128</td><td>16</td></tr><tr><td>hidden_size</td><td>1024</td><td>64</td></tr><tr><td>intermediate_size</td><td>3584</td><td>128</td></tr><tr><td>num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>num_hidden_layers</td><td>24</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>8</td><td>2</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>tencent/Hunyuan-0.5B-Instruct: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>tencent/Hunyuan-0.5B-Instruct</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/Hunyuan-0.5B-Instruct-mic/tree/d7fe47dc3288bba155076d3747959365159073b0" title="huggingface-mic/Hunyuan-0.5B-Instruct-mic@d7fe47dc3288bba155076d3747959365159073b0">huggingface-mic/Hunyuan-0.5B-Instruct-mic@d7fe47dc3288</a> ← <a href="https://huggingface.co/tencent/Hunyuan-0.5B-Instruct/tree/2359fb220c010e9d6d62c62d466f0eda179c2cf3" title="tencent/Hunyuan-0.5B-Instruct@2359fb220c010e9d6d62c62d466f0eda179c2cf3">tencent/Hunyuan-0.5B-Instruct@2359fb220c01</a> Β· HunYuanDenseV1ForCausalLM Β· random weights</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">539.0M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">7.8M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 24</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i><i class="c0" title="layer 2: DecoderLayer"></i><i class="c0" title="layer 3: DecoderLayer"></i><i class="c0" title="layer 4: DecoderLayer"></i><i class="c0" title="layer 5: DecoderLayer"></i><i class="c0" title="layer 6: DecoderLayer"></i><i class="c0" title="layer 7: DecoderLayer"></i><i class="c0" title="layer 8: DecoderLayer"></i><i class="c0" title="layer 9: DecoderLayer"></i><i class="c0" title="layer 10: DecoderLayer"></i><i class="c0" title="layer 11: DecoderLayer"></i><i class="c0" title="layer 12: DecoderLayer"></i><i class="c0" title="layer 13: DecoderLayer"></i><i class="c0" title="layer 14: DecoderLayer"></i><i class="c0" title="layer 15: DecoderLayer"></i><i class="c0" title="layer 16: DecoderLayer"></i><i class="c0" title="layer 17: DecoderLayer"></i><i class="c0" title="layer 18: DecoderLayer"></i><i class="c0" title="layer 19: DecoderLayer"></i><i class="c0" title="layer 20: DecoderLayer"></i><i class="c0" title="layer 21: DecoderLayer"></i><i class="c0" title="layer 22: DecoderLayer"></i><i class="c0" title="layer 23: DecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: DecoderLayer"></i><i class="c0" title="layer 1: DecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>DecoderLayer</td><td>24</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.770471" title="model.layers: 415.3M (77.0%)"></i><i class="c1" style="flex:0.229528" title="model.embed_tokens: 123.7M (23.0%)"></i><i class="c2" style="flex:0.000002" title="model.norm: 1.0K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.009485" title="model.layers: 74.0K (0.9%)"></i><i class="c1" style="flex:0.990506" title="model.embed_tokens: 7.7M (99.1%)"></i><i class="c2" style="flex:0.000008" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>415.3M</td><td>77.0%</td><td>74.0K</td><td>0.9%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>123.7M</td><td>23.0%</td><td>7.7M</td><td>99.1%</td></tr><tr><td><i class="c2"></i>model.norm</td><td>1.0K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>GQA (16 heads, 8 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>dynamic</td><td>dynamic</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>hunyuan_v1_dense</td><td>hunyuan_v1_dense</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>120818</td><td>120818</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>262144</td><td>262144</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix HunYuanDenseV1 omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>169</td><td>15</td></tr><tr class=""><td>RMSNorm</td><td>97</td><td>9</td></tr><tr class=""><td>Attention</td><td>24</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>24</td><td>2</td></tr><tr class=""><td>MLP</td><td>24</td><td>2</td></tr><tr class=""><td>SiLUActivation</td><td>24</td><td>2</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">7 fields changed, 43 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>attention_head_dim</td><td>128</td><td>16</td></tr><tr><td>head_dim</td><td>128</td><td>16</td></tr><tr><td>hidden_size</td><td>1024</td><td>64</td></tr><tr><td>intermediate_size</td><td>3584</td><td>128</td></tr><tr><td>num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>num_hidden_layers</td><td>24</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>8</td><td>2</td></tr></table>
36
  </main></body></html>
shrink/Hunyuan-0.5B-Instruct-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/Hunyuan-0.5B-Instruct-mic@d7fe47dc3288bba155076d3747959365159073b0", "findings": []}
shrink/HunyuanOCR-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/HunyuanOCR-mic@2b69a85316ef4335c59c1a82d624555a1dfd7e71 ← tencent/HunyuanOCR@47644ecc4fc854efa4f505155158831f36773ee4 Β· HunYuanVLForConditionalGeneration Β· random weights</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">996.2M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">9.1M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">2</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.language_model.layers</h3><div class="row"><span class="lab">Original Β· 24</span><div class="cells"><i class="c0" title="layer 0: DenseV1DecoderLayer"></i><i class="c0" title="layer 1: DenseV1DecoderLayer"></i><i class="c0" title="layer 2: DenseV1DecoderLayer"></i><i class="c0" title="layer 3: DenseV1DecoderLayer"></i><i class="c0" title="layer 4: DenseV1DecoderLayer"></i><i class="c0" title="layer 5: DenseV1DecoderLayer"></i><i class="c0" title="layer 6: DenseV1DecoderLayer"></i><i class="c0" title="layer 7: DenseV1DecoderLayer"></i><i class="c0" title="layer 8: DenseV1DecoderLayer"></i><i class="c0" title="layer 9: DenseV1DecoderLayer"></i><i class="c0" title="layer 10: DenseV1DecoderLayer"></i><i class="c0" title="layer 11: DenseV1DecoderLayer"></i><i class="c0" title="layer 12: DenseV1DecoderLayer"></i><i class="c0" title="layer 13: DenseV1DecoderLayer"></i><i class="c0" title="layer 14: DenseV1DecoderLayer"></i><i class="c0" title="layer 15: DenseV1DecoderLayer"></i><i class="c0" title="layer 16: DenseV1DecoderLayer"></i><i class="c0" title="layer 17: DenseV1DecoderLayer"></i><i class="c0" title="layer 18: DenseV1DecoderLayer"></i><i class="c0" title="layer 19: DenseV1DecoderLayer"></i><i class="c0" title="layer 20: DenseV1DecoderLayer"></i><i class="c0" title="layer 21: DenseV1DecoderLayer"></i><i class="c0" title="layer 22: DenseV1DecoderLayer"></i><i class="c0" title="layer 23: DenseV1DecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: DenseV1DecoderLayer"></i><i class="c0" title="layer 1: DenseV1DecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>DenseV1DecoderLayer</td><td>24</td><td>2</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.vision_tower.layers</h3><div class="row"><span class="lab">Original Β· 27</span><div class="cells"><i class="c0" title="layer 0: VisionBlock"></i><i class="c0" title="layer 1: VisionBlock"></i><i class="c0" title="layer 2: VisionBlock"></i><i class="c0" title="layer 3: VisionBlock"></i><i class="c0" title="layer 4: VisionBlock"></i><i class="c0" title="layer 5: VisionBlock"></i><i class="c0" title="layer 6: VisionBlock"></i><i class="c0" title="layer 7: VisionBlock"></i><i class="c0" title="layer 8: VisionBlock"></i><i class="c0" title="layer 9: VisionBlock"></i><i class="c0" title="layer 10: VisionBlock"></i><i class="c0" title="layer 11: VisionBlock"></i><i class="c0" title="layer 12: VisionBlock"></i><i class="c0" title="layer 13: VisionBlock"></i><i class="c0" title="layer 14: VisionBlock"></i><i class="c0" title="layer 15: VisionBlock"></i><i class="c0" title="layer 16: VisionBlock"></i><i class="c0" title="layer 17: VisionBlock"></i><i class="c0" title="layer 18: VisionBlock"></i><i class="c0" title="layer 19: VisionBlock"></i><i class="c0" title="layer 20: VisionBlock"></i><i class="c0" title="layer 21: VisionBlock"></i><i class="c0" title="layer 22: VisionBlock"></i><i class="c0" title="layer 23: VisionBlock"></i><i class="c0" title="layer 24: VisionBlock"></i><i class="c0" title="layer 25: VisionBlock"></i><i class="c0" title="layer 26: VisionBlock"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: VisionBlock"></i><i class="c0" title="layer 1: VisionBlock"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>VisionBlock</td><td>27</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.416872" title="model.language_model.layers: 415.3M (41.7%)"></i><i class="c1" style="flex:0.413033" title="model.vision_tower.layers: 411.5M (41.3%)"></i><i class="c2" style="flex:0.124190" title="model.language_model: 123.7M (12.4%)"></i><i class="c3" style="flex:0.045906" title="model.vision_tower: 45.7M (4.6%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.008178" title="model.language_model.layers: 74.0K (0.8%)"></i><i class="c1" style="flex:0.007394" title="model.vision_tower.layers: 66.9K (0.7%)"></i><i class="c2" style="flex:0.854014" title="model.language_model: 7.7M (85.4%)"></i><i class="c3" style="flex:0.130414" title="model.vision_tower: 1.2M (13.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.language_model.layers</td><td>415.3M</td><td>41.7%</td><td>74.0K</td><td>0.8%</td></tr><tr><td><i class="c1"></i>model.vision_tower.layers</td><td>411.5M</td><td>41.3%</td><td>66.9K</td><td>0.7%</td></tr><tr><td><i class="c2"></i>model.language_model</td><td>123.7M</td><td>12.4%</td><td>7.7M</td><td>85.4%</td></tr><tr><td><i class="c3"></i>model.vision_tower</td><td>45.7M</td><td>4.6%</td><td>1.2M</td><td>13.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>model_type</td><td>hunyuan_vl</td><td>hunyuan_vl</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>attention style</td><td>GQA (16 heads, 8 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>RoPE type</td><td>dynamic</td><td>dynamic</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>model_type</td><td>hunyuan_vl_text</td><td>hunyuan_vl_text</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>vocab_size</td><td>120818</td><td>120818</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>max_position_embeddings</td><td>131072</td><td>131072</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>attention style</td><td>MHA (16 heads, 16 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>model_type</td><td>hunyuan_vl_vision</td><td>hunyuan_vl_vision</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>hidden_act</td><td>gelu</td><td>gelu</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>patch_size</td><td>16</td><td>16</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>num_channels</td><td>3</td><td>3</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix HunYuanVL omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>332</td><td>28</td></tr><tr class=""><td>RMSNorm</td><td>99</td><td>11</td></tr><tr class=""><td>LayerNorm</td><td>54</td><td>4</td></tr><tr class=""><td>GELUActivation</td><td>28</td><td>3</td></tr><tr class=""><td>VisionAttention</td><td>27</td><td>2</td></tr><tr class=""><td>VisionBlock</td><td>27</td><td>2</td></tr><tr class=""><td>VisionMLP</td><td>27</td><td>2</td></tr><tr class=""><td>DenseV1Attention</td><td>24</td><td>2</td></tr><tr class=""><td>DenseV1DecoderLayer</td><td>24</td><td>2</td></tr><tr class=""><td>MLP</td><td>24</td><td>2</td></tr><tr class=""><td>SiLUActivation</td><td>24</td><td>2</td></tr><tr class=""><td>Conv2d</td><td>3</td><td>3</td></tr><tr class=""><td>Embedding</td><td>2</td><td>2</td></tr><tr class=""><td>ModuleList</td><td>2</td><td>2</td></tr><tr class=""><td>ForConditionalGeneration</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>TextModel</td><td>1</td><td>1</td></tr><tr class=""><td>VisionPatchEmbed</td><td>1</td><td>1</td></tr><tr class=""><td>VisionPatchMerger</td><td>1</td><td>1</td></tr><tr class=""><td>VisionTransformer</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">14 fields changed, 68 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>text_config.head_dim</td><td>128</td><td>16</td></tr><tr><td>text_config.hidden_size</td><td>1024</td><td>64</td></tr><tr><td>text_config.intermediate_size</td><td>3584</td><td>128</td></tr><tr><td>text_config.num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>text_config.num_hidden_layers</td><td>24</td><td>2</td></tr><tr><td>text_config.num_key_value_heads</td><td>8</td><td>2</td></tr><tr><td>text_config.rope_parameters</td><td>{&quot;alpha&quot;: 1000.0, &quot;beta_fast&quot;: 32, &quot;beta_slow&quot;: 1, &quot;factor&quot;: 1.0, &quot;mscale&quot;: 1…</td><td>{&quot;alpha&quot;: 1000.0, &quot;beta_fast&quot;: 32, &quot;beta_slow&quot;: 1, &quot;factor&quot;: 1.0, &quot;mrope_sect…</td></tr><tr><td>vision_config.hidden_size</td><td>1152</td><td>64</td></tr><tr><td>vision_config.intermediate_size</td><td>4304</td><td>128</td></tr><tr><td>vision_config.num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>vision_config.num_hidden_layers</td><td>27</td><td>2</td></tr><tr><td>vision_config.num_key_value_heads</td><td>16</td><td>4</td></tr><tr><td>vision_config.out_hidden_size</td><td>1024</td><td>64</td></tr><tr><td>vision_config.text_hidden_size</td><td>1024</td><td>64</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>tencent/HunyuanOCR: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>tencent/HunyuanOCR</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/HunyuanOCR-mic/tree/2b69a85316ef4335c59c1a82d624555a1dfd7e71" title="huggingface-mic/HunyuanOCR-mic@2b69a85316ef4335c59c1a82d624555a1dfd7e71">huggingface-mic/HunyuanOCR-mic@2b69a85316ef</a> ← <a href="https://huggingface.co/tencent/HunyuanOCR/tree/47644ecc4fc854efa4f505155158831f36773ee4" title="tencent/HunyuanOCR@47644ecc4fc854efa4f505155158831f36773ee4">tencent/HunyuanOCR@47644ecc4fc8</a> Β· HunYuanVLForConditionalGeneration Β· random weights</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">996.2M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">9.1M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">2</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.language_model.layers</h3><div class="row"><span class="lab">Original Β· 24</span><div class="cells"><i class="c0" title="layer 0: DenseV1DecoderLayer"></i><i class="c0" title="layer 1: DenseV1DecoderLayer"></i><i class="c0" title="layer 2: DenseV1DecoderLayer"></i><i class="c0" title="layer 3: DenseV1DecoderLayer"></i><i class="c0" title="layer 4: DenseV1DecoderLayer"></i><i class="c0" title="layer 5: DenseV1DecoderLayer"></i><i class="c0" title="layer 6: DenseV1DecoderLayer"></i><i class="c0" title="layer 7: DenseV1DecoderLayer"></i><i class="c0" title="layer 8: DenseV1DecoderLayer"></i><i class="c0" title="layer 9: DenseV1DecoderLayer"></i><i class="c0" title="layer 10: DenseV1DecoderLayer"></i><i class="c0" title="layer 11: DenseV1DecoderLayer"></i><i class="c0" title="layer 12: DenseV1DecoderLayer"></i><i class="c0" title="layer 13: DenseV1DecoderLayer"></i><i class="c0" title="layer 14: DenseV1DecoderLayer"></i><i class="c0" title="layer 15: DenseV1DecoderLayer"></i><i class="c0" title="layer 16: DenseV1DecoderLayer"></i><i class="c0" title="layer 17: DenseV1DecoderLayer"></i><i class="c0" title="layer 18: DenseV1DecoderLayer"></i><i class="c0" title="layer 19: DenseV1DecoderLayer"></i><i class="c0" title="layer 20: DenseV1DecoderLayer"></i><i class="c0" title="layer 21: DenseV1DecoderLayer"></i><i class="c0" title="layer 22: DenseV1DecoderLayer"></i><i class="c0" title="layer 23: DenseV1DecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: DenseV1DecoderLayer"></i><i class="c0" title="layer 1: DenseV1DecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>DenseV1DecoderLayer</td><td>24</td><td>2</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.vision_tower.layers</h3><div class="row"><span class="lab">Original Β· 27</span><div class="cells"><i class="c0" title="layer 0: VisionBlock"></i><i class="c0" title="layer 1: VisionBlock"></i><i class="c0" title="layer 2: VisionBlock"></i><i class="c0" title="layer 3: VisionBlock"></i><i class="c0" title="layer 4: VisionBlock"></i><i class="c0" title="layer 5: VisionBlock"></i><i class="c0" title="layer 6: VisionBlock"></i><i class="c0" title="layer 7: VisionBlock"></i><i class="c0" title="layer 8: VisionBlock"></i><i class="c0" title="layer 9: VisionBlock"></i><i class="c0" title="layer 10: VisionBlock"></i><i class="c0" title="layer 11: VisionBlock"></i><i class="c0" title="layer 12: VisionBlock"></i><i class="c0" title="layer 13: VisionBlock"></i><i class="c0" title="layer 14: VisionBlock"></i><i class="c0" title="layer 15: VisionBlock"></i><i class="c0" title="layer 16: VisionBlock"></i><i class="c0" title="layer 17: VisionBlock"></i><i class="c0" title="layer 18: VisionBlock"></i><i class="c0" title="layer 19: VisionBlock"></i><i class="c0" title="layer 20: VisionBlock"></i><i class="c0" title="layer 21: VisionBlock"></i><i class="c0" title="layer 22: VisionBlock"></i><i class="c0" title="layer 23: VisionBlock"></i><i class="c0" title="layer 24: VisionBlock"></i><i class="c0" title="layer 25: VisionBlock"></i><i class="c0" title="layer 26: VisionBlock"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: VisionBlock"></i><i class="c0" title="layer 1: VisionBlock"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>VisionBlock</td><td>27</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.416872" title="model.language_model.layers: 415.3M (41.7%)"></i><i class="c1" style="flex:0.413033" title="model.vision_tower.layers: 411.5M (41.3%)"></i><i class="c2" style="flex:0.124190" title="model.language_model: 123.7M (12.4%)"></i><i class="c3" style="flex:0.045906" title="model.vision_tower: 45.7M (4.6%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.008178" title="model.language_model.layers: 74.0K (0.8%)"></i><i class="c1" style="flex:0.007394" title="model.vision_tower.layers: 66.9K (0.7%)"></i><i class="c2" style="flex:0.854014" title="model.language_model: 7.7M (85.4%)"></i><i class="c3" style="flex:0.130414" title="model.vision_tower: 1.2M (13.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.language_model.layers</td><td>415.3M</td><td>41.7%</td><td>74.0K</td><td>0.8%</td></tr><tr><td><i class="c1"></i>model.vision_tower.layers</td><td>411.5M</td><td>41.3%</td><td>66.9K</td><td>0.7%</td></tr><tr><td><i class="c2"></i>model.language_model</td><td>123.7M</td><td>12.4%</td><td>7.7M</td><td>85.4%</td></tr><tr><td><i class="c3"></i>model.vision_tower</td><td>45.7M</td><td>4.6%</td><td>1.2M</td><td>13.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>model_type</td><td>hunyuan_vl</td><td>hunyuan_vl</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>attention style</td><td>GQA (16 heads, 8 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>RoPE type</td><td>dynamic</td><td>dynamic</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>model_type</td><td>hunyuan_vl_text</td><td>hunyuan_vl_text</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>vocab_size</td><td>120818</td><td>120818</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>max_position_embeddings</td><td>131072</td><td>131072</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>attention style</td><td>MHA (16 heads, 16 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>model_type</td><td>hunyuan_vl_vision</td><td>hunyuan_vl_vision</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>hidden_act</td><td>gelu</td><td>gelu</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>patch_size</td><td>16</td><td>16</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>num_channels</td><td>3</td><td>3</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix HunYuanVL omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>332</td><td>28</td></tr><tr class=""><td>RMSNorm</td><td>99</td><td>11</td></tr><tr class=""><td>LayerNorm</td><td>54</td><td>4</td></tr><tr class=""><td>GELUActivation</td><td>28</td><td>3</td></tr><tr class=""><td>VisionAttention</td><td>27</td><td>2</td></tr><tr class=""><td>VisionBlock</td><td>27</td><td>2</td></tr><tr class=""><td>VisionMLP</td><td>27</td><td>2</td></tr><tr class=""><td>DenseV1Attention</td><td>24</td><td>2</td></tr><tr class=""><td>DenseV1DecoderLayer</td><td>24</td><td>2</td></tr><tr class=""><td>MLP</td><td>24</td><td>2</td></tr><tr class=""><td>SiLUActivation</td><td>24</td><td>2</td></tr><tr class=""><td>Conv2d</td><td>3</td><td>3</td></tr><tr class=""><td>Embedding</td><td>2</td><td>2</td></tr><tr class=""><td>ModuleList</td><td>2</td><td>2</td></tr><tr class=""><td>ForConditionalGeneration</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>TextModel</td><td>1</td><td>1</td></tr><tr class=""><td>VisionPatchEmbed</td><td>1</td><td>1</td></tr><tr class=""><td>VisionPatchMerger</td><td>1</td><td>1</td></tr><tr class=""><td>VisionTransformer</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">14 fields changed, 68 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>text_config.head_dim</td><td>128</td><td>16</td></tr><tr><td>text_config.hidden_size</td><td>1024</td><td>64</td></tr><tr><td>text_config.intermediate_size</td><td>3584</td><td>128</td></tr><tr><td>text_config.num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>text_config.num_hidden_layers</td><td>24</td><td>2</td></tr><tr><td>text_config.num_key_value_heads</td><td>8</td><td>2</td></tr><tr><td>text_config.rope_parameters</td><td>{&quot;alpha&quot;: 1000.0, &quot;beta_fast&quot;: 32, &quot;beta_slow&quot;: 1, &quot;factor&quot;: 1.0, &quot;mscale&quot;: 1…</td><td>{&quot;alpha&quot;: 1000.0, &quot;beta_fast&quot;: 32, &quot;beta_slow&quot;: 1, &quot;factor&quot;: 1.0, &quot;mrope_sect…</td></tr><tr><td>vision_config.hidden_size</td><td>1152</td><td>64</td></tr><tr><td>vision_config.intermediate_size</td><td>4304</td><td>128</td></tr><tr><td>vision_config.num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>vision_config.num_hidden_layers</td><td>27</td><td>2</td></tr><tr><td>vision_config.num_key_value_heads</td><td>16</td><td>4</td></tr><tr><td>vision_config.out_hidden_size</td><td>1024</td><td>64</td></tr><tr><td>vision_config.text_hidden_size</td><td>1024</td><td>64</td></tr></table>
36
  </main></body></html>
shrink/HunyuanOCR-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/HunyuanOCR-mic@2b69a85316ef4335c59c1a82d624555a1dfd7e71", "findings": []}
shrink/Hy-MT2-30B-A3B-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/Hy-MT2-30B-A3B-mic@8357a369b5aa965d0f486cbe4fe63329d0a1d00e ← tencent/Hy-MT2-30B-A3B@d3ead4dba61c09aac60a261a96ad1df3e705febb Β· HYV3ForCausalLM Β· random weights</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">30.1B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">15.5M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 48</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 2: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 3: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 4: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 5: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 6: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 7: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 8: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 9: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 10: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 11: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 12: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 13: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 14: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 15: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 16: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 17: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 18: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 19: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 20: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 21: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 22: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 23: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 24: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 25: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 26: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 27: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 28: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 29: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 30: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 31: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 32: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 33: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 34: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 35: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 36: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 37: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 38: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 39: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 40: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 41: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 42: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 43: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 44: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 45: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 46: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 47: Experts, MoE, TopKRouter"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, MoE, TopKRouter"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>no Experts</td><td>1</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>Experts, MoE, TopKRouter</td><td>47</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.983538" title="model.layers: 29.6B (98.4%)"></i><i class="c1" style="flex:0.008231" title="lm_head: 247.5M (0.8%)"></i><i class="c2" style="flex:0.008231" title="model.embed_tokens: 247.5M (0.8%)"></i><i class="c3" style="flex:0.000000" title="model.norm: 2.0K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.005175" title="model.layers: 80.4K (0.5%)"></i><i class="c1" style="flex:0.497411" title="lm_head: 7.7M (49.7%)"></i><i class="c2" style="flex:0.497411" title="model.embed_tokens: 7.7M (49.7%)"></i><i class="c3" style="flex:0.000004" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>29.6B</td><td>98.4%</td><td>80.4K</td><td>0.5%</td></tr><tr><td><i class="c1"></i>lm_head</td><td>247.5M</td><td>0.8%</td><td>7.7M</td><td>49.7%</td></tr><tr><td><i class="c2"></i>model.embed_tokens</td><td>247.5M</td><td>0.8%</td><td>7.7M</td><td>49.7%</td></tr><tr><td><i class="c3"></i>model.norm</td><td>2.0K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>GQA (32 heads, 4 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-8 of 128 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>hy_v3</td><td>hy_v3</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>120832</td><td>120832</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>262144</td><td>262144</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix HYV3 omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>337</td><td>15</td></tr><tr class=""><td>RMSNorm</td><td>193</td><td>9</td></tr><tr class=""><td>SiLUActivation</td><td>95</td><td>3</td></tr><tr class=""><td>Attention</td><td>48</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>48</td><td>2</td></tr><tr class=""><td>MLP</td><td>48</td><td>2</td></tr><tr class=""><td>Experts</td><td>47</td><td>1</td></tr><tr class=""><td>MoE</td><td>47</td><td>1</td></tr><tr class=""><td>TopKRouter</td><td>47</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">10 fields changed, 39 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>head_dim</td><td>128</td><td>16</td></tr><tr><td>hidden_size</td><td>2048</td><td>64</td></tr><tr><td>intermediate_size</td><td>6912</td><td>128</td></tr><tr><td>mlp_layer_types</td><td>48 entries: 47Γ— sparse, 1Γ— dense</td><td>[&quot;dense&quot;, &quot;sparse&quot;]</td></tr><tr><td>moe_intermediate_size</td><td>768</td><td>32</td></tr><tr><td>num_attention_heads</td><td>32</td><td>4</td></tr><tr><td>num_experts</td><td>128</td><td>4</td></tr><tr><td>num_experts_per_tok</td><td>8</td><td>2</td></tr><tr><td>num_hidden_layers</td><td>48</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>4</td><td>2</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>tencent/Hy-MT2-30B-A3B: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>tencent/Hy-MT2-30B-A3B</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/Hy-MT2-30B-A3B-mic/tree/8357a369b5aa965d0f486cbe4fe63329d0a1d00e" title="huggingface-mic/Hy-MT2-30B-A3B-mic@8357a369b5aa965d0f486cbe4fe63329d0a1d00e">huggingface-mic/Hy-MT2-30B-A3B-mic@8357a369b5aa</a> ← <a href="https://huggingface.co/tencent/Hy-MT2-30B-A3B/tree/d3ead4dba61c09aac60a261a96ad1df3e705febb" title="tencent/Hy-MT2-30B-A3B@d3ead4dba61c09aac60a261a96ad1df3e705febb">tencent/Hy-MT2-30B-A3B@d3ead4dba61c</a> Β· HYV3ForCausalLM Β· random weights</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">30.1B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">15.5M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 48</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 2: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 3: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 4: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 5: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 6: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 7: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 8: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 9: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 10: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 11: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 12: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 13: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 14: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 15: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 16: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 17: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 18: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 19: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 20: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 21: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 22: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 23: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 24: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 25: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 26: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 27: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 28: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 29: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 30: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 31: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 32: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 33: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 34: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 35: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 36: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 37: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 38: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 39: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 40: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 41: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 42: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 43: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 44: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 45: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 46: Experts, MoE, TopKRouter"></i><i class="c1" title="layer 47: Experts, MoE, TopKRouter"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: no Experts"></i><i class="c1" title="layer 1: Experts, MoE, TopKRouter"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>no Experts</td><td>1</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>Experts, MoE, TopKRouter</td><td>47</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.983538" title="model.layers: 29.6B (98.4%)"></i><i class="c1" style="flex:0.008231" title="lm_head: 247.5M (0.8%)"></i><i class="c2" style="flex:0.008231" title="model.embed_tokens: 247.5M (0.8%)"></i><i class="c3" style="flex:0.000000" title="model.norm: 2.0K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.005175" title="model.layers: 80.4K (0.5%)"></i><i class="c1" style="flex:0.497411" title="lm_head: 7.7M (49.7%)"></i><i class="c2" style="flex:0.497411" title="model.embed_tokens: 7.7M (49.7%)"></i><i class="c3" style="flex:0.000004" title="model.norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>29.6B</td><td>98.4%</td><td>80.4K</td><td>0.5%</td></tr><tr><td><i class="c1"></i>lm_head</td><td>247.5M</td><td>0.8%</td><td>7.7M</td><td>49.7%</td></tr><tr><td><i class="c2"></i>model.embed_tokens</td><td>247.5M</td><td>0.8%</td><td>7.7M</td><td>49.7%</td></tr><tr><td><i class="c3"></i>model.norm</td><td>2.0K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>GQA (32 heads, 4 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-8 of 128 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>hy_v3</td><td>hy_v3</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>120832</td><td>120832</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>262144</td><td>262144</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix HYV3 omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>337</td><td>15</td></tr><tr class=""><td>RMSNorm</td><td>193</td><td>9</td></tr><tr class=""><td>SiLUActivation</td><td>95</td><td>3</td></tr><tr class=""><td>Attention</td><td>48</td><td>2</td></tr><tr class=""><td>DecoderLayer</td><td>48</td><td>2</td></tr><tr class=""><td>MLP</td><td>48</td><td>2</td></tr><tr class=""><td>Experts</td><td>47</td><td>1</td></tr><tr class=""><td>MoE</td><td>47</td><td>1</td></tr><tr class=""><td>TopKRouter</td><td>47</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">10 fields changed, 39 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>head_dim</td><td>128</td><td>16</td></tr><tr><td>hidden_size</td><td>2048</td><td>64</td></tr><tr><td>intermediate_size</td><td>6912</td><td>128</td></tr><tr><td>mlp_layer_types</td><td>48 entries: 47Γ— sparse, 1Γ— dense</td><td>[&quot;dense&quot;, &quot;sparse&quot;]</td></tr><tr><td>moe_intermediate_size</td><td>768</td><td>32</td></tr><tr><td>num_attention_heads</td><td>32</td><td>4</td></tr><tr><td>num_experts</td><td>128</td><td>4</td></tr><tr><td>num_experts_per_tok</td><td>8</td><td>2</td></tr><tr><td>num_hidden_layers</td><td>48</td><td>2</td></tr><tr><td>num_key_value_heads</td><td>4</td><td>2</td></tr></table>
36
  </main></body></html>
shrink/Hy-MT2-30B-A3B-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/Hy-MT2-30B-A3B-mic@8357a369b5aa965d0f486cbe4fe63329d0a1d00e", "findings": []}
shrink/Inkling-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/Inkling-mic@a7c61810959524051418bc1a49a70ac559307c0b ← thinkingmachines/Inkling@828496eeae4c243ff1a22f7f28ff83694f2f7bc9 Β· InklingForConditionalGeneration Β· random weights</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">947.1B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">33.8M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">2</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.language_model.layers</h3><div class="row"><span class="lab">Original Β· 66</span><div class="cells"><i class="c0" title="layer 0: hybrid_sliding, MLP"></i><i class="c0" title="layer 1: hybrid_sliding, MLP"></i><i class="c1" title="layer 2: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 3: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 4: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 5: hybrid, Experts, MoE …"></i><i class="c1" title="layer 6: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 7: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 8: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 9: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 10: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 11: hybrid, Experts, MoE …"></i><i class="c1" title="layer 12: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 13: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 14: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 15: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 16: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 17: hybrid, Experts, MoE …"></i><i class="c1" title="layer 18: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 19: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 20: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 21: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 22: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 23: hybrid, Experts, MoE …"></i><i class="c1" title="layer 24: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 25: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 26: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 27: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 28: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 29: hybrid, Experts, MoE …"></i><i class="c1" title="layer 30: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 31: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 32: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 33: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 34: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 35: hybrid, Experts, MoE …"></i><i class="c1" title="layer 36: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 37: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 38: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 39: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 40: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 41: hybrid, Experts, MoE …"></i><i class="c1" title="layer 42: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 43: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 44: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 45: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 46: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 47: hybrid, Experts, MoE …"></i><i class="c1" title="layer 48: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 49: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 50: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 51: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 52: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 53: hybrid, Experts, MoE …"></i><i class="c1" title="layer 54: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 55: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 56: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 57: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 58: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 59: hybrid, Experts, MoE …"></i><i class="c1" title="layer 60: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 61: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 62: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 63: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 64: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 65: hybrid, Experts, MoE …"></i></div></div><div class="row"><span class="lab">Fixture Β· 6</span><div class="cells"><i class="c0" title="layer 0: hybrid_sliding, MLP"></i><i class="c0" title="layer 1: hybrid_sliding, MLP"></i><i class="c1" title="layer 2: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 3: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 4: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 5: hybrid, Experts, MoE …"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>hybrid_sliding, MLP</td><td>2</td><td>2</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>hybrid_sliding, Experts, MoE …</td><td>53</td><td>3</td><td>βœ“ kept</td></tr><tr><td><i class="c2"></i>hybrid, Experts, MoE …</td><td>11</td><td>1</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.vision_tower.encoder_layers</h3><div class="row"><span class="lab">Original Β· 4</span><div class="cells"><i class="c0" title="layer 0: RMSNorm"></i><i class="c0" title="layer 1: RMSNorm"></i><i class="c0" title="layer 2: RMSNorm"></i><i class="c1" title="layer 3: no RMSNorm"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: RMSNorm"></i><i class="c1" title="layer 1: no RMSNorm"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>RMSNorm</td><td>3</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>no RMSNorm</td><td>1</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.997295" title="model.language_model.layers: 944.6B (99.7%)"></i><i class="c1" style="flex:0.001304" title="model.language_model: 1.2B (0.1%)"></i><i class="c2" style="flex:0.001304" title="lm_head: 1.2B (0.1%)"></i><i class="c3" style="flex:0.000088" title="model.vision_tower.encoder_layers: 83.7M (0.0%)"></i><i class="c4" style="flex:0.000008" title="model.audio_tower: 7.9M (0.0%)"></i><i class="c5" style="flex:0.000000" title="model.vision_tower: 6.1K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.214181" title="model.language_model.layers: 7.2M (21.4%)"></i><i class="c1" style="flex:0.380580" title="model.language_model: 12.9M (38.1%)"></i><i class="c2" style="flex:0.380576" title="lm_head: 12.9M (38.1%)"></i><i class="c3" style="flex:0.022235" title="model.vision_tower.encoder_layers: 751.7K (2.2%)"></i><i class="c4" style="flex:0.002425" title="model.audio_tower: 82.0K (0.2%)"></i><i class="c5" style="flex:0.000002" title="model.vision_tower: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.language_model.layers</td><td>944.6B</td><td>99.7%</td><td>7.2M</td><td>21.4%</td></tr><tr><td><i class="c1"></i>model.language_model</td><td>1.2B</td><td>0.1%</td><td>12.9M</td><td>38.1%</td></tr><tr><td><i class="c2"></i>lm_head</td><td>1.2B</td><td>0.1%</td><td>12.9M</td><td>38.1%</td></tr><tr><td><i class="c3"></i>model.vision_tower.encoder_layers</td><td>83.7M</td><td>0.0%</td><td>751.7K</td><td>2.2%</td></tr><tr><td><i class="c4"></i>model.audio_tower</td><td>7.9M</td><td>0.0%</td><td>82.0K</td><td>0.2%</td></tr><tr><td><i class="c5"></i>model.vision_tower</td><td>6.1K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>model_type</td><td>inkling_mm_model</td><td>inkling_mm_model</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>attention style</td><td>GQA (64 heads, 8 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>MoE routing</td><td>top-6 of 256 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>model_type</td><td>inkling_text</td><td>inkling_text</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>vocab_size</td><td>201024</td><td>201024</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>max_position_embeddings</td><td>1048576</td><td>1048576</td><td>βœ“ kept</td></tr><tr class=""><td>audio_config</td><td>model_type</td><td>inkling_audio</td><td>inkling_audio</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>model_type</td><td>inkling_vision</td><td>inkling_vision</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>patch_size</td><td>40</td><td>40</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>num_channels</td><td>3</td><td>3</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Inkling omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>341</td><td>39</td></tr><tr class=""><td>RMSNorm</td><td>271</td><td>29</td></tr><tr class=""><td>Conv1d</td><td>264</td><td>24</td></tr><tr class=""><td>ShortConvolution</td><td>264</td><td>24</td></tr><tr class=""><td>SiLUActivation</td><td>130</td><td>10</td></tr><tr class=""><td>Attention</td><td>66</td><td>6</td></tr><tr class=""><td>DecoderLayer</td><td>66</td><td>6</td></tr><tr class=""><td>RelativeLogits</td><td>66</td><td>6</td></tr><tr class=""><td>Experts</td><td>64</td><td>4</td></tr><tr class=""><td>MoE</td><td>64</td><td>4</td></tr><tr class=""><td>SharedExperts</td><td>64</td><td>4</td></tr><tr class=""><td>TopkRouter</td><td>64</td><td>4</td></tr><tr class=""><td>VisionEncoderLayer</td><td>4</td><td>2</td></tr><tr class=""><td>MLP</td><td>2</td><td>2</td></tr><tr class=""><td>ModuleList</td><td>2</td><td>2</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>AudioModel</td><td>1</td><td>1</td></tr><tr class=""><td>AudioModelEmbeddings</td><td>1</td><td>1</td></tr><tr class=""><td>ForConditionalGeneration</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>NormedEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>TextModel</td><td>1</td><td>1</td></tr><tr class=""><td>VisionModel</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">16 fields changed, 102 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>audio_config.text_hidden_size</td><td>6144</td><td>64</td></tr><tr><td>text_config.head_dim</td><td>128</td><td>16</td></tr><tr><td>text_config.hidden_size</td><td>6144</td><td>64</td></tr><tr><td>text_config.intermediate_size</td><td>24576</td><td>128</td></tr><tr><td>text_config.layer_types</td><td>66 entries: 55Γ— hybrid_sliding, 11Γ— hybrid</td><td>[&quot;hybrid_sliding&quot;, &quot;hybrid_sliding&quot;, &quot;hybrid_sliding&quot;, &quot;hybrid_sliding&quot;, &quot;hyb…</td></tr><tr><td>text_config.mlp_layer_types</td><td>66 entries: 64Γ— sparse, 2Γ— dense</td><td>[&quot;dense&quot;, &quot;dense&quot;, &quot;sparse&quot;, &quot;sparse&quot;, &quot;sparse&quot;, &quot;sparse&quot;]</td></tr><tr><td>text_config.moe_intermediate_size</td><td>3072</td><td>32</td></tr><tr><td>text_config.n_routed_experts</td><td>256</td><td>4</td></tr><tr><td>text_config.num_attention_heads</td><td>64</td><td>4</td></tr><tr><td>text_config.num_experts_per_tok</td><td>6</td><td>2</td></tr><tr><td>text_config.num_hidden_layers</td><td>66</td><td>6</td></tr><tr><td>text_config.num_key_value_heads</td><td>8</td><td>2</td></tr><tr><td>vision_config.hidden_size</td><td>1024</td><td>64</td></tr><tr><td>vision_config.n_layers</td><td>4</td><td>2</td></tr><tr><td>vision_config.num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>vision_config.text_hidden_size</td><td>6144</td><td>64</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>thinkingmachines/Inkling: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>thinkingmachines/Inkling</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/Inkling-mic/tree/a7c61810959524051418bc1a49a70ac559307c0b" title="huggingface-mic/Inkling-mic@a7c61810959524051418bc1a49a70ac559307c0b">huggingface-mic/Inkling-mic@a7c618109595</a> ← <a href="https://huggingface.co/thinkingmachines/Inkling/tree/828496eeae4c243ff1a22f7f28ff83694f2f7bc9" title="thinkingmachines/Inkling@828496eeae4c243ff1a22f7f28ff83694f2f7bc9">thinkingmachines/Inkling@828496eeae4c</a> Β· InklingForConditionalGeneration Β· random weights</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">947.1B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">33.8M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">2</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.language_model.layers</h3><div class="row"><span class="lab">Original Β· 66</span><div class="cells"><i class="c0" title="layer 0: hybrid_sliding, MLP"></i><i class="c0" title="layer 1: hybrid_sliding, MLP"></i><i class="c1" title="layer 2: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 3: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 4: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 5: hybrid, Experts, MoE …"></i><i class="c1" title="layer 6: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 7: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 8: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 9: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 10: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 11: hybrid, Experts, MoE …"></i><i class="c1" title="layer 12: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 13: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 14: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 15: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 16: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 17: hybrid, Experts, MoE …"></i><i class="c1" title="layer 18: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 19: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 20: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 21: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 22: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 23: hybrid, Experts, MoE …"></i><i class="c1" title="layer 24: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 25: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 26: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 27: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 28: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 29: hybrid, Experts, MoE …"></i><i class="c1" title="layer 30: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 31: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 32: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 33: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 34: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 35: hybrid, Experts, MoE …"></i><i class="c1" title="layer 36: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 37: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 38: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 39: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 40: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 41: hybrid, Experts, MoE …"></i><i class="c1" title="layer 42: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 43: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 44: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 45: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 46: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 47: hybrid, Experts, MoE …"></i><i class="c1" title="layer 48: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 49: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 50: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 51: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 52: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 53: hybrid, Experts, MoE …"></i><i class="c1" title="layer 54: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 55: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 56: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 57: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 58: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 59: hybrid, Experts, MoE …"></i><i class="c1" title="layer 60: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 61: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 62: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 63: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 64: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 65: hybrid, Experts, MoE …"></i></div></div><div class="row"><span class="lab">Fixture Β· 6</span><div class="cells"><i class="c0" title="layer 0: hybrid_sliding, MLP"></i><i class="c0" title="layer 1: hybrid_sliding, MLP"></i><i class="c1" title="layer 2: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 3: hybrid_sliding, Experts, MoE …"></i><i class="c1" title="layer 4: hybrid_sliding, Experts, MoE …"></i><i class="c2" title="layer 5: hybrid, Experts, MoE …"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>hybrid_sliding, MLP</td><td>2</td><td>2</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>hybrid_sliding, Experts, MoE …</td><td>53</td><td>3</td><td>βœ“ kept</td></tr><tr><td><i class="c2"></i>hybrid, Experts, MoE …</td><td>11</td><td>1</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.vision_tower.encoder_layers</h3><div class="row"><span class="lab">Original Β· 4</span><div class="cells"><i class="c0" title="layer 0: RMSNorm"></i><i class="c0" title="layer 1: RMSNorm"></i><i class="c0" title="layer 2: RMSNorm"></i><i class="c1" title="layer 3: no RMSNorm"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: RMSNorm"></i><i class="c1" title="layer 1: no RMSNorm"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>RMSNorm</td><td>3</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>no RMSNorm</td><td>1</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.997295" title="model.language_model.layers: 944.6B (99.7%)"></i><i class="c1" style="flex:0.001304" title="model.language_model: 1.2B (0.1%)"></i><i class="c2" style="flex:0.001304" title="lm_head: 1.2B (0.1%)"></i><i class="c3" style="flex:0.000088" title="model.vision_tower.encoder_layers: 83.7M (0.0%)"></i><i class="c4" style="flex:0.000008" title="model.audio_tower: 7.9M (0.0%)"></i><i class="c5" style="flex:0.000000" title="model.vision_tower: 6.1K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.214181" title="model.language_model.layers: 7.2M (21.4%)"></i><i class="c1" style="flex:0.380580" title="model.language_model: 12.9M (38.1%)"></i><i class="c2" style="flex:0.380576" title="lm_head: 12.9M (38.1%)"></i><i class="c3" style="flex:0.022235" title="model.vision_tower.encoder_layers: 751.7K (2.2%)"></i><i class="c4" style="flex:0.002425" title="model.audio_tower: 82.0K (0.2%)"></i><i class="c5" style="flex:0.000002" title="model.vision_tower: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.language_model.layers</td><td>944.6B</td><td>99.7%</td><td>7.2M</td><td>21.4%</td></tr><tr><td><i class="c1"></i>model.language_model</td><td>1.2B</td><td>0.1%</td><td>12.9M</td><td>38.1%</td></tr><tr><td><i class="c2"></i>lm_head</td><td>1.2B</td><td>0.1%</td><td>12.9M</td><td>38.1%</td></tr><tr><td><i class="c3"></i>model.vision_tower.encoder_layers</td><td>83.7M</td><td>0.0%</td><td>751.7K</td><td>2.2%</td></tr><tr><td><i class="c4"></i>model.audio_tower</td><td>7.9M</td><td>0.0%</td><td>82.0K</td><td>0.2%</td></tr><tr><td><i class="c5"></i>model.vision_tower</td><td>6.1K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>model_type</td><td>inkling_mm_model</td><td>inkling_mm_model</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>attention style</td><td>GQA (64 heads, 8 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>MoE routing</td><td>top-6 of 256 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>model_type</td><td>inkling_text</td><td>inkling_text</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>vocab_size</td><td>201024</td><td>201024</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>max_position_embeddings</td><td>1048576</td><td>1048576</td><td>βœ“ kept</td></tr><tr class=""><td>audio_config</td><td>model_type</td><td>inkling_audio</td><td>inkling_audio</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>model_type</td><td>inkling_vision</td><td>inkling_vision</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>patch_size</td><td>40</td><td>40</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>num_channels</td><td>3</td><td>3</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Inkling omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>341</td><td>39</td></tr><tr class=""><td>RMSNorm</td><td>271</td><td>29</td></tr><tr class=""><td>Conv1d</td><td>264</td><td>24</td></tr><tr class=""><td>ShortConvolution</td><td>264</td><td>24</td></tr><tr class=""><td>SiLUActivation</td><td>130</td><td>10</td></tr><tr class=""><td>Attention</td><td>66</td><td>6</td></tr><tr class=""><td>DecoderLayer</td><td>66</td><td>6</td></tr><tr class=""><td>RelativeLogits</td><td>66</td><td>6</td></tr><tr class=""><td>Experts</td><td>64</td><td>4</td></tr><tr class=""><td>MoE</td><td>64</td><td>4</td></tr><tr class=""><td>SharedExperts</td><td>64</td><td>4</td></tr><tr class=""><td>TopkRouter</td><td>64</td><td>4</td></tr><tr class=""><td>VisionEncoderLayer</td><td>4</td><td>2</td></tr><tr class=""><td>MLP</td><td>2</td><td>2</td></tr><tr class=""><td>ModuleList</td><td>2</td><td>2</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>AudioModel</td><td>1</td><td>1</td></tr><tr class=""><td>AudioModelEmbeddings</td><td>1</td><td>1</td></tr><tr class=""><td>ForConditionalGeneration</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>NormedEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>TextModel</td><td>1</td><td>1</td></tr><tr class=""><td>VisionModel</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">16 fields changed, 102 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>audio_config.text_hidden_size</td><td>6144</td><td>64</td></tr><tr><td>text_config.head_dim</td><td>128</td><td>16</td></tr><tr><td>text_config.hidden_size</td><td>6144</td><td>64</td></tr><tr><td>text_config.intermediate_size</td><td>24576</td><td>128</td></tr><tr><td>text_config.layer_types</td><td>66 entries: 55Γ— hybrid_sliding, 11Γ— hybrid</td><td>[&quot;hybrid_sliding&quot;, &quot;hybrid_sliding&quot;, &quot;hybrid_sliding&quot;, &quot;hybrid_sliding&quot;, &quot;hyb…</td></tr><tr><td>text_config.mlp_layer_types</td><td>66 entries: 64Γ— sparse, 2Γ— dense</td><td>[&quot;dense&quot;, &quot;dense&quot;, &quot;sparse&quot;, &quot;sparse&quot;, &quot;sparse&quot;, &quot;sparse&quot;]</td></tr><tr><td>text_config.moe_intermediate_size</td><td>3072</td><td>32</td></tr><tr><td>text_config.n_routed_experts</td><td>256</td><td>4</td></tr><tr><td>text_config.num_attention_heads</td><td>64</td><td>4</td></tr><tr><td>text_config.num_experts_per_tok</td><td>6</td><td>2</td></tr><tr><td>text_config.num_hidden_layers</td><td>66</td><td>6</td></tr><tr><td>text_config.num_key_value_heads</td><td>8</td><td>2</td></tr><tr><td>vision_config.hidden_size</td><td>1024</td><td>64</td></tr><tr><td>vision_config.n_layers</td><td>4</td><td>2</td></tr><tr><td>vision_config.num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>vision_config.text_hidden_size</td><td>6144</td><td>64</td></tr></table>
36
  </main></body></html>
shrink/Inkling-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/Inkling-mic@a7c61810959524051418bc1a49a70ac559307c0b", "findings": []}
shrink/Jamba-tiny-dev-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/Jamba-tiny-dev-mic@d03ef8cc4d1f81b8af2ae68030ecaa9bc1d51a6d ← ai21labs/Jamba-tiny-dev@ed303361004ac875426a61675edecf8e9d976882 Β· JambaForCausalLM Β· random weights</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">318.7M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">8.8M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 16</span><div class="cells"><i class="c0" title="layer 0: Conv1d, MLP, MambaDecoderLayer …"></i><i class="c1" title="layer 1: Conv1d, Experts, MambaDecoderLayer …"></i><i class="c0" title="layer 2: Conv1d, MLP, MambaDecoderLayer …"></i><i class="c1" title="layer 3: Conv1d, Experts, MambaDecoderLayer …"></i><i class="c2" title="layer 4: is_causal=True, Attention, AttentionDecoderLayer …"></i><i class="c1" title="layer 5: Conv1d, Experts, MambaDecoderLayer …"></i><i class="c0" title="layer 6: Conv1d, MLP, MambaDecoderLayer …"></i><i class="c1" title="layer 7: Conv1d, Experts, MambaDecoderLayer …"></i><i class="c0" title="layer 8: Conv1d, MLP, MambaDecoderLayer …"></i><i class="c1" title="layer 9: Conv1d, Experts, MambaDecoderLayer …"></i><i class="c0" title="layer 10: Conv1d, MLP, MambaDecoderLayer …"></i><i class="c1" title="layer 11: Conv1d, Experts, MambaDecoderLayer …"></i><i class="c2" title="layer 12: is_causal=True, Attention, AttentionDecoderLayer …"></i><i class="c1" title="layer 13: Conv1d, Experts, MambaDecoderLayer …"></i><i class="c0" title="layer 14: Conv1d, MLP, MambaDecoderLayer …"></i><i class="c1" title="layer 15: Conv1d, Experts, MambaDecoderLayer …"></i></div></div><div class="row"><span class="lab">Fixture Β· 5</span><div class="cells"><i class="c0" title="layer 0: Conv1d, MLP, MambaDecoderLayer …"></i><i class="c1" title="layer 1: Conv1d, Experts, MambaDecoderLayer …"></i><i class="c0" title="layer 2: Conv1d, MLP, MambaDecoderLayer …"></i><i class="c1" title="layer 3: Conv1d, Experts, MambaDecoderLayer …"></i><i class="c2" title="layer 4: is_causal=True, Attention, AttentionDecoderLayer …"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>Conv1d, MLP, MambaDecoderLayer …</td><td>6</td><td>2</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>Conv1d, Experts, MambaDecoderLayer …</td><td>8</td><td>2</td><td>βœ“ kept</td></tr><tr><td><i class="c2"></i>is_causal=True, Attention, AttentionDecoderLayer …</td><td>2</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.789420" title="model.layers: 251.6M (78.9%)"></i><i class="c1" style="flex:0.105289" title="lm_head: 33.6M (10.5%)"></i><i class="c2" style="flex:0.105289" title="model.embed_tokens: 33.6M (10.5%)"></i><i class="c3" style="flex:0.000002" title="model.final_layernorm: 512 (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.050189" title="model.layers: 443.3K (5.0%)"></i><i class="c1" style="flex:0.474902" title="lm_head: 4.2M (47.5%)"></i><i class="c2" style="flex:0.474902" title="model.embed_tokens: 4.2M (47.5%)"></i><i class="c3" style="flex:0.000007" title="model.final_layernorm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>251.6M</td><td>78.9%</td><td>443.3K</td><td>5.0%</td></tr><tr><td><i class="c1"></i>lm_head</td><td>33.6M</td><td>10.5%</td><td>4.2M</td><td>47.5%</td></tr><tr><td><i class="c2"></i>model.embed_tokens</td><td>33.6M</td><td>10.5%</td><td>4.2M</td><td>47.5%</td></tr><tr><td><i class="c3"></i>model.final_layernorm</td><td>512</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>GQA (8 heads, 2 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-2 of 8 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>jamba</td><td>jamba</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>65536</td><td>65536</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>262144</td><td>262144</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Jamba omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>89</td><td>30</td></tr><tr class=""><td>RMSNorm</td><td>75</td><td>23</td></tr><tr class=""><td>SiLUActivation</td><td>30</td><td>9</td></tr><tr class=""><td>Conv1d</td><td>14</td><td>4</td></tr><tr class=""><td>MambaDecoderLayer</td><td>14</td><td>4</td></tr><tr class=""><td>MambaMixer</td><td>14</td><td>4</td></tr><tr class=""><td>Experts</td><td>8</td><td>2</td></tr><tr class=""><td>MLP</td><td>8</td><td>3</td></tr><tr class=""><td>SparseMoeBlock</td><td>8</td><td>2</td></tr><tr class=""><td>TopKRouter</td><td>8</td><td>2</td></tr><tr class=""><td>Attention</td><td>2</td><td>1</td></tr><tr class=""><td>AttentionDecoderLayer</td><td>2</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">5 fields changed, 39 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>hidden_size</td><td>512</td><td>64</td></tr><tr><td>intermediate_size</td><td>2048</td><td>128</td></tr><tr><td>num_attention_heads</td><td>8</td><td>4</td></tr><tr><td>num_experts</td><td>8</td><td>4</td></tr><tr><td>num_hidden_layers</td><td>16</td><td>5</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>ai21labs/Jamba-tiny-dev: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>ai21labs/Jamba-tiny-dev</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/Jamba-tiny-dev-mic/tree/d03ef8cc4d1f81b8af2ae68030ecaa9bc1d51a6d" title="huggingface-mic/Jamba-tiny-dev-mic@d03ef8cc4d1f81b8af2ae68030ecaa9bc1d51a6d">huggingface-mic/Jamba-tiny-dev-mic@d03ef8cc4d1f</a> ← <a href="https://huggingface.co/ai21labs/Jamba-tiny-dev/tree/ed303361004ac875426a61675edecf8e9d976882" title="ai21labs/Jamba-tiny-dev@ed303361004ac875426a61675edecf8e9d976882">ai21labs/Jamba-tiny-dev@ed303361004a</a> Β· JambaForCausalLM Β· random weights</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">318.7M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">8.8M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 16</span><div class="cells"><i class="c0" title="layer 0: Conv1d, MLP, MambaDecoderLayer …"></i><i class="c1" title="layer 1: Conv1d, Experts, MambaDecoderLayer …"></i><i class="c0" title="layer 2: Conv1d, MLP, MambaDecoderLayer …"></i><i class="c1" title="layer 3: Conv1d, Experts, MambaDecoderLayer …"></i><i class="c2" title="layer 4: is_causal=True, Attention, AttentionDecoderLayer …"></i><i class="c1" title="layer 5: Conv1d, Experts, MambaDecoderLayer …"></i><i class="c0" title="layer 6: Conv1d, MLP, MambaDecoderLayer …"></i><i class="c1" title="layer 7: Conv1d, Experts, MambaDecoderLayer …"></i><i class="c0" title="layer 8: Conv1d, MLP, MambaDecoderLayer …"></i><i class="c1" title="layer 9: Conv1d, Experts, MambaDecoderLayer …"></i><i class="c0" title="layer 10: Conv1d, MLP, MambaDecoderLayer …"></i><i class="c1" title="layer 11: Conv1d, Experts, MambaDecoderLayer …"></i><i class="c2" title="layer 12: is_causal=True, Attention, AttentionDecoderLayer …"></i><i class="c1" title="layer 13: Conv1d, Experts, MambaDecoderLayer …"></i><i class="c0" title="layer 14: Conv1d, MLP, MambaDecoderLayer …"></i><i class="c1" title="layer 15: Conv1d, Experts, MambaDecoderLayer …"></i></div></div><div class="row"><span class="lab">Fixture Β· 5</span><div class="cells"><i class="c0" title="layer 0: Conv1d, MLP, MambaDecoderLayer …"></i><i class="c1" title="layer 1: Conv1d, Experts, MambaDecoderLayer …"></i><i class="c0" title="layer 2: Conv1d, MLP, MambaDecoderLayer …"></i><i class="c1" title="layer 3: Conv1d, Experts, MambaDecoderLayer …"></i><i class="c2" title="layer 4: is_causal=True, Attention, AttentionDecoderLayer …"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>Conv1d, MLP, MambaDecoderLayer …</td><td>6</td><td>2</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>Conv1d, Experts, MambaDecoderLayer …</td><td>8</td><td>2</td><td>βœ“ kept</td></tr><tr><td><i class="c2"></i>is_causal=True, Attention, AttentionDecoderLayer …</td><td>2</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.789420" title="model.layers: 251.6M (78.9%)"></i><i class="c1" style="flex:0.105289" title="lm_head: 33.6M (10.5%)"></i><i class="c2" style="flex:0.105289" title="model.embed_tokens: 33.6M (10.5%)"></i><i class="c3" style="flex:0.000002" title="model.final_layernorm: 512 (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.050189" title="model.layers: 443.3K (5.0%)"></i><i class="c1" style="flex:0.474902" title="lm_head: 4.2M (47.5%)"></i><i class="c2" style="flex:0.474902" title="model.embed_tokens: 4.2M (47.5%)"></i><i class="c3" style="flex:0.000007" title="model.final_layernorm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>251.6M</td><td>78.9%</td><td>443.3K</td><td>5.0%</td></tr><tr><td><i class="c1"></i>lm_head</td><td>33.6M</td><td>10.5%</td><td>4.2M</td><td>47.5%</td></tr><tr><td><i class="c2"></i>model.embed_tokens</td><td>33.6M</td><td>10.5%</td><td>4.2M</td><td>47.5%</td></tr><tr><td><i class="c3"></i>model.final_layernorm</td><td>512</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>GQA (8 heads, 2 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-2 of 8 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>jamba</td><td>jamba</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>65536</td><td>65536</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>262144</td><td>262144</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Jamba omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>89</td><td>30</td></tr><tr class=""><td>RMSNorm</td><td>75</td><td>23</td></tr><tr class=""><td>SiLUActivation</td><td>30</td><td>9</td></tr><tr class=""><td>Conv1d</td><td>14</td><td>4</td></tr><tr class=""><td>MambaDecoderLayer</td><td>14</td><td>4</td></tr><tr class=""><td>MambaMixer</td><td>14</td><td>4</td></tr><tr class=""><td>Experts</td><td>8</td><td>2</td></tr><tr class=""><td>MLP</td><td>8</td><td>3</td></tr><tr class=""><td>SparseMoeBlock</td><td>8</td><td>2</td></tr><tr class=""><td>TopKRouter</td><td>8</td><td>2</td></tr><tr class=""><td>Attention</td><td>2</td><td>1</td></tr><tr class=""><td>AttentionDecoderLayer</td><td>2</td><td>1</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">5 fields changed, 39 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>hidden_size</td><td>512</td><td>64</td></tr><tr><td>intermediate_size</td><td>2048</td><td>128</td></tr><tr><td>num_attention_heads</td><td>8</td><td>4</td></tr><tr><td>num_experts</td><td>8</td><td>4</td></tr><tr><td>num_hidden_layers</td><td>16</td><td>5</td></tr></table>
36
  </main></body></html>
shrink/Jamba-tiny-dev-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/Jamba-tiny-dev-mic@d03ef8cc4d1f81b8af2ae68030ecaa9bc1d51a6d", "findings": []}
shrink/Janus-Pro-1B-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/Janus-Pro-1B-mic@5d0bfc05499ea83be331a434ea79583ca970581a ← deepseek-community/Janus-Pro-1B@1655280bb75959cc1cb85529a2a8b26e7016072e Β· JanusForConditionalGeneration Β· random weights Β· built before the shrink rule (no recorded recipe)</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">1.9B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">81.9M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">4</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.vision_model.encoder.layers</h3><div class="row"><span class="lab">Original Β· 24</span><div class="cells"><i class="c0" title="layer 0: VisionEncoderLayer"></i><i class="c0" title="layer 1: VisionEncoderLayer"></i><i class="c0" title="layer 2: VisionEncoderLayer"></i><i class="c0" title="layer 3: VisionEncoderLayer"></i><i class="c0" title="layer 4: VisionEncoderLayer"></i><i class="c0" title="layer 5: VisionEncoderLayer"></i><i class="c0" title="layer 6: VisionEncoderLayer"></i><i class="c0" title="layer 7: VisionEncoderLayer"></i><i class="c0" title="layer 8: VisionEncoderLayer"></i><i class="c0" title="layer 9: VisionEncoderLayer"></i><i class="c0" title="layer 10: VisionEncoderLayer"></i><i class="c0" title="layer 11: VisionEncoderLayer"></i><i class="c0" title="layer 12: VisionEncoderLayer"></i><i class="c0" title="layer 13: VisionEncoderLayer"></i><i class="c0" title="layer 14: VisionEncoderLayer"></i><i class="c0" title="layer 15: VisionEncoderLayer"></i><i class="c0" title="layer 16: VisionEncoderLayer"></i><i class="c0" title="layer 17: VisionEncoderLayer"></i><i class="c0" title="layer 18: VisionEncoderLayer"></i><i class="c0" title="layer 19: VisionEncoderLayer"></i><i class="c0" title="layer 20: VisionEncoderLayer"></i><i class="c0" title="layer 21: VisionEncoderLayer"></i><i class="c0" title="layer 22: VisionEncoderLayer"></i><i class="c0" title="layer 23: VisionEncoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: VisionEncoderLayer"></i><i class="c0" title="layer 1: VisionEncoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>VisionEncoderLayer</td><td>24</td><td>2</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.vqmodel.encoder.down</h3><div class="row"><span class="lab">Original Β· 5</span><div class="cells"><i class="c0" title="layer 0: VQVAEConvDownsample"></i><i class="c0" title="layer 1: VQVAEConvDownsample"></i><i class="c0" title="layer 2: VQVAEConvDownsample"></i><i class="c0" title="layer 3: VQVAEConvDownsample"></i><i class="c1" title="layer 4: VQVAEAttnBlock"></i></div></div><div class="row"><span class="lab">Fixture Β· 5</span><div class="cells"><i class="c0" title="layer 0: VQVAEConvDownsample"></i><i class="c0" title="layer 1: VQVAEConvDownsample"></i><i class="c0" title="layer 2: VQVAEConvDownsample"></i><i class="c0" title="layer 3: VQVAEConvDownsample"></i><i class="c1" title="layer 4: VQVAEAttnBlock"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>VQVAEConvDownsample</td><td>4</td><td>4</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>VQVAEAttnBlock</td><td>1</td><td>1</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.vqmodel.decoder.up</h3><div class="row"><span class="lab">Original Β· 5</span><div class="cells"><i class="c0" title="layer 0: VQVAEAttnBlock, VQVAEConvUpsample"></i><i class="c1" title="layer 1: VQVAEConvUpsample"></i><i class="c1" title="layer 2: VQVAEConvUpsample"></i><i class="c1" title="layer 3: VQVAEConvUpsample"></i><i class="c2" title="layer 4: no VQVAEAttnBlock"></i></div></div><div class="row"><span class="lab">Fixture Β· 5</span><div class="cells"><i class="c0" title="layer 0: VQVAEAttnBlock, VQVAEConvUpsample"></i><i class="c1" title="layer 1: VQVAEConvUpsample"></i><i class="c1" title="layer 2: VQVAEConvUpsample"></i><i class="c1" title="layer 3: VQVAEConvUpsample"></i><i class="c2" title="layer 4: no VQVAEAttnBlock"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>VQVAEAttnBlock, VQVAEConvUpsample</td><td>1</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>VQVAEConvUpsample</td><td>3</td><td>3</td><td>βœ“ kept</td></tr><tr><td><i class="c2"></i>no VQVAEAttnBlock</td><td>1</td><td>1</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.language_model.layers</h3><div class="row"><span class="lab">Original Β· 24</span><div class="cells"><i class="c0" title="layer 0: LlamaDecoderLayer"></i><i class="c0" title="layer 1: LlamaDecoderLayer"></i><i class="c0" title="layer 2: LlamaDecoderLayer"></i><i class="c0" title="layer 3: LlamaDecoderLayer"></i><i class="c0" title="layer 4: LlamaDecoderLayer"></i><i class="c0" title="layer 5: LlamaDecoderLayer"></i><i class="c0" title="layer 6: LlamaDecoderLayer"></i><i class="c0" title="layer 7: LlamaDecoderLayer"></i><i class="c0" title="layer 8: LlamaDecoderLayer"></i><i class="c0" title="layer 9: LlamaDecoderLayer"></i><i class="c0" title="layer 10: LlamaDecoderLayer"></i><i class="c0" title="layer 11: LlamaDecoderLayer"></i><i class="c0" title="layer 12: LlamaDecoderLayer"></i><i class="c0" title="layer 13: LlamaDecoderLayer"></i><i class="c0" title="layer 14: LlamaDecoderLayer"></i><i class="c0" title="layer 15: LlamaDecoderLayer"></i><i class="c0" title="layer 16: LlamaDecoderLayer"></i><i class="c0" title="layer 17: LlamaDecoderLayer"></i><i class="c0" title="layer 18: LlamaDecoderLayer"></i><i class="c0" title="layer 19: LlamaDecoderLayer"></i><i class="c0" title="layer 20: LlamaDecoderLayer"></i><i class="c0" title="layer 21: LlamaDecoderLayer"></i><i class="c0" title="layer 22: LlamaDecoderLayer"></i><i class="c0" title="layer 23: LlamaDecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: LlamaDecoderLayer"></i><i class="c0" title="layer 1: LlamaDecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>LlamaDecoderLayer</td><td>24</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.660565" title="model.language_model.layers: 1.2B (66.1%)"></i><i class="c1" style="flex:0.161929" title="model.vision_model.encoder.layers: 302.3M (16.2%)"></i><i class="c2" style="flex:0.112333" title="model.language_model: 209.7M (11.2%)"></i><i class="c3" style="flex:0.020230" title="model.generation_head: 37.8M (2.0%)"></i><i class="c4" style="flex:0.016482" title="model.vqmodel.decoder.up: 30.8M (1.6%)"></i><i class="c5" style="flex:0.012584" title="model.vqmodel: 23.5M (1.3%)"></i><i class="c6" style="flex:0.009437" title="model.vqmodel.encoder.down: 17.6M (0.9%)"></i><i class="c7" style="flex:0.006439" title="4 others: 12.0M (0.6%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.001003" title="model.language_model.layers: 82.2K (0.1%)"></i><i class="c1" style="flex:0.001220" title="model.vision_model.encoder.layers: 100.0K (0.1%)"></i><i class="c2" style="flex:0.080008" title="model.language_model: 6.6M (8.0%)"></i><i class="c3" style="flex:0.014602" title="model.generation_head: 1.2M (1.5%)"></i><i class="c4" style="flex:0.375654" title="model.vqmodel.decoder.up: 30.8M (37.6%)"></i><i class="c5" style="flex:0.298366" title="model.vqmodel: 24.4M (29.8%)"></i><i class="c6" style="flex:0.215090" title="model.vqmodel.encoder.down: 17.6M (21.5%)"></i><i class="c7" style="flex:0.014057" title="4 others: 1.2M (1.4%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.language_model.layers</td><td>1.2B</td><td>66.1%</td><td>82.2K</td><td>0.1%</td></tr><tr><td><i class="c1"></i>model.vision_model.encoder.layers</td><td>302.3M</td><td>16.2%</td><td>100.0K</td><td>0.1%</td></tr><tr><td><i class="c2"></i>model.language_model</td><td>209.7M</td><td>11.2%</td><td>6.6M</td><td>8.0%</td></tr><tr><td><i class="c3"></i>model.generation_head</td><td>37.8M</td><td>2.0%</td><td>1.2M</td><td>1.5%</td></tr><tr><td><i class="c4"></i>model.vqmodel.decoder.up</td><td>30.8M</td><td>1.6%</td><td>30.8M</td><td>37.6%</td></tr><tr><td><i class="c5"></i>model.vqmodel</td><td>23.5M</td><td>1.3%</td><td>24.4M</td><td>29.8%</td></tr><tr><td><i class="c6"></i>model.vqmodel.encoder.down</td><td>17.6M</td><td>0.9%</td><td>17.6M</td><td>21.5%</td></tr><tr><td><i class="c7"></i>4 others</td><td>12.0M</td><td>0.6%</td><td>1.2M</td><td>1.4%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>model_type</td><td>janus</td><td>janus</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>attention style</td><td>MHA (16 heads, 16 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>model_type</td><td>llama</td><td>llama</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>vocab_size</td><td>102400</td><td>102400</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>max_position_embeddings</td><td>16384</td><td>16384</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>model_type</td><td>janus_vision_model</td><td>janus_vision_model</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>hidden_act</td><td>gelu</td><td>gelu</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>image_size</td><td>384</td><td>384</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>patch_size</td><td>16</td><td>16</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>num_channels</td><td>3</td><td>3</td><td>βœ“ kept</td></tr><tr class=""><td>vq_config</td><td>model_type</td><td>janus_vqgan</td><td>janus_vqgan</td><td>βœ“ kept</td></tr><tr class=""><td>vq_config</td><td>hidden_act</td><td>gelu</td><td>gelu</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Janus omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>319</td><td>33</td></tr><tr class=""><td>Conv2d</td><td>105</td><td>105</td></tr><tr class=""><td>Dropout</td><td>77</td><td>33</td></tr><tr class=""><td>Identity</td><td>72</td><td>6</td></tr><tr class=""><td>GroupNorm</td><td>67</td><td>67</td></tr><tr class=""><td>LayerNorm</td><td>49</td><td>5</td></tr><tr class=""><td>LlamaRMSNorm</td><td>49</td><td>5</td></tr><tr class=""><td>VQVAEResnetBlock</td><td>29</td><td>29</td></tr><tr class=""><td>GELUActivation</td><td>27</td><td>5</td></tr><tr class=""><td>ModuleList</td><td>26</td><td>26</td></tr><tr class=""><td>VisionAttention</td><td>24</td><td>2</td></tr><tr class=""><td>VisionEncoderLayer</td><td>24</td><td>2</td></tr><tr class=""><td>VisionMLP</td><td>24</td><td>2</td></tr><tr class=""><td>LlamaAttention</td><td>24</td><td>2</td></tr><tr class=""><td>LlamaDecoderLayer</td><td>24</td><td>2</td></tr><tr class=""><td>LlamaMLP</td><td>24</td><td>2</td></tr><tr class=""><td>SiLUActivation</td><td>24</td><td>2</td></tr><tr class=""><td>Module</td><td>10</td><td>10</td></tr><tr class=""><td>VQVAEAttnBlock</td><td>7</td><td>7</td></tr><tr class=""><td>Embedding</td><td>4</td><td>4</td></tr><tr class=""><td>VQVAEConvDownsample</td><td>4</td><td>4</td></tr><tr class=""><td>VQVAEConvUpsample</td><td>4</td><td>4</td></tr><tr class=""><td>VQVAEMidBlock</td><td>2</td><td>2</td></tr><tr class=""><td>ForConditionalGeneration</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>VQVAE</td><td>1</td><td>1</td></tr><tr class=""><td>VQVAEAlignerMLP</td><td>1</td><td>1</td></tr><tr class=""><td>VQVAEDecoder</td><td>1</td><td>1</td></tr><tr class=""><td>VQVAEEncoder</td><td>1</td><td>1</td></tr><tr class=""><td>VQVAEHead</td><td>1</td><td>1</td></tr><tr class=""><td>VQVAEVectorQuantizer</td><td>1</td><td>1</td></tr><tr class=""><td>VisionAlignerMLP</td><td>1</td><td>1</td></tr><tr class=""><td>VisionEmbeddings</td><td>1</td><td>1</td></tr><tr class=""><td>VisionEncoder</td><td>1</td><td>1</td></tr><tr class=""><td>VisionModel</td><td>1</td><td>1</td></tr><tr class=""><td>LlamaModel</td><td>1</td><td>1</td></tr><tr class=""><td>LlamaRotaryEmbedding</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">12 fields changed, 82 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>text_config.head_dim</td><td>128</td><td>16</td></tr><tr><td>text_config.hidden_size</td><td>2048</td><td>64</td></tr><tr><td>text_config.intermediate_size</td><td>5632</td><td>128</td></tr><tr><td>text_config.num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>text_config.num_hidden_layers</td><td>24</td><td>2</td></tr><tr><td>text_config.num_key_value_heads</td><td>16</td><td>4</td></tr><tr><td>vision_config.hidden_size</td><td>1024</td><td>64</td></tr><tr><td>vision_config.num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>vision_config.num_hidden_layers</td><td>24</td><td>2</td></tr><tr><td>vision_config.projection_dim</td><td>2048</td><td>64</td></tr><tr><td>vq_config.embed_dim</td><td>8</td><td>64</td></tr><tr><td>vq_config.projection_dim</td><td>2048</td><td>64</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>deepseek-community/Janus-Pro-1B: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>deepseek-community/Janus-Pro-1B</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/Janus-Pro-1B-mic/tree/5d0bfc05499ea83be331a434ea79583ca970581a" title="huggingface-mic/Janus-Pro-1B-mic@5d0bfc05499ea83be331a434ea79583ca970581a">huggingface-mic/Janus-Pro-1B-mic@5d0bfc05499e</a> ← <a href="https://huggingface.co/deepseek-community/Janus-Pro-1B/tree/1655280bb75959cc1cb85529a2a8b26e7016072e" title="deepseek-community/Janus-Pro-1B@1655280bb75959cc1cb85529a2a8b26e7016072e">deepseek-community/Janus-Pro-1B@1655280bb759</a> Β· JanusForConditionalGeneration Β· random weights Β· built before the shrink rule (no recorded recipe)</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">1.9B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">81.9M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">4</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.vision_model.encoder.layers</h3><div class="row"><span class="lab">Original Β· 24</span><div class="cells"><i class="c0" title="layer 0: VisionEncoderLayer"></i><i class="c0" title="layer 1: VisionEncoderLayer"></i><i class="c0" title="layer 2: VisionEncoderLayer"></i><i class="c0" title="layer 3: VisionEncoderLayer"></i><i class="c0" title="layer 4: VisionEncoderLayer"></i><i class="c0" title="layer 5: VisionEncoderLayer"></i><i class="c0" title="layer 6: VisionEncoderLayer"></i><i class="c0" title="layer 7: VisionEncoderLayer"></i><i class="c0" title="layer 8: VisionEncoderLayer"></i><i class="c0" title="layer 9: VisionEncoderLayer"></i><i class="c0" title="layer 10: VisionEncoderLayer"></i><i class="c0" title="layer 11: VisionEncoderLayer"></i><i class="c0" title="layer 12: VisionEncoderLayer"></i><i class="c0" title="layer 13: VisionEncoderLayer"></i><i class="c0" title="layer 14: VisionEncoderLayer"></i><i class="c0" title="layer 15: VisionEncoderLayer"></i><i class="c0" title="layer 16: VisionEncoderLayer"></i><i class="c0" title="layer 17: VisionEncoderLayer"></i><i class="c0" title="layer 18: VisionEncoderLayer"></i><i class="c0" title="layer 19: VisionEncoderLayer"></i><i class="c0" title="layer 20: VisionEncoderLayer"></i><i class="c0" title="layer 21: VisionEncoderLayer"></i><i class="c0" title="layer 22: VisionEncoderLayer"></i><i class="c0" title="layer 23: VisionEncoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: VisionEncoderLayer"></i><i class="c0" title="layer 1: VisionEncoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>VisionEncoderLayer</td><td>24</td><td>2</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.vqmodel.encoder.down</h3><div class="row"><span class="lab">Original Β· 5</span><div class="cells"><i class="c0" title="layer 0: VQVAEConvDownsample"></i><i class="c0" title="layer 1: VQVAEConvDownsample"></i><i class="c0" title="layer 2: VQVAEConvDownsample"></i><i class="c0" title="layer 3: VQVAEConvDownsample"></i><i class="c1" title="layer 4: VQVAEAttnBlock"></i></div></div><div class="row"><span class="lab">Fixture Β· 5</span><div class="cells"><i class="c0" title="layer 0: VQVAEConvDownsample"></i><i class="c0" title="layer 1: VQVAEConvDownsample"></i><i class="c0" title="layer 2: VQVAEConvDownsample"></i><i class="c0" title="layer 3: VQVAEConvDownsample"></i><i class="c1" title="layer 4: VQVAEAttnBlock"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>VQVAEConvDownsample</td><td>4</td><td>4</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>VQVAEAttnBlock</td><td>1</td><td>1</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.vqmodel.decoder.up</h3><div class="row"><span class="lab">Original Β· 5</span><div class="cells"><i class="c0" title="layer 0: VQVAEAttnBlock, VQVAEConvUpsample"></i><i class="c1" title="layer 1: VQVAEConvUpsample"></i><i class="c1" title="layer 2: VQVAEConvUpsample"></i><i class="c1" title="layer 3: VQVAEConvUpsample"></i><i class="c2" title="layer 4: no VQVAEAttnBlock"></i></div></div><div class="row"><span class="lab">Fixture Β· 5</span><div class="cells"><i class="c0" title="layer 0: VQVAEAttnBlock, VQVAEConvUpsample"></i><i class="c1" title="layer 1: VQVAEConvUpsample"></i><i class="c1" title="layer 2: VQVAEConvUpsample"></i><i class="c1" title="layer 3: VQVAEConvUpsample"></i><i class="c2" title="layer 4: no VQVAEAttnBlock"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>VQVAEAttnBlock, VQVAEConvUpsample</td><td>1</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>VQVAEConvUpsample</td><td>3</td><td>3</td><td>βœ“ kept</td></tr><tr><td><i class="c2"></i>no VQVAEAttnBlock</td><td>1</td><td>1</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.language_model.layers</h3><div class="row"><span class="lab">Original Β· 24</span><div class="cells"><i class="c0" title="layer 0: LlamaDecoderLayer"></i><i class="c0" title="layer 1: LlamaDecoderLayer"></i><i class="c0" title="layer 2: LlamaDecoderLayer"></i><i class="c0" title="layer 3: LlamaDecoderLayer"></i><i class="c0" title="layer 4: LlamaDecoderLayer"></i><i class="c0" title="layer 5: LlamaDecoderLayer"></i><i class="c0" title="layer 6: LlamaDecoderLayer"></i><i class="c0" title="layer 7: LlamaDecoderLayer"></i><i class="c0" title="layer 8: LlamaDecoderLayer"></i><i class="c0" title="layer 9: LlamaDecoderLayer"></i><i class="c0" title="layer 10: LlamaDecoderLayer"></i><i class="c0" title="layer 11: LlamaDecoderLayer"></i><i class="c0" title="layer 12: LlamaDecoderLayer"></i><i class="c0" title="layer 13: LlamaDecoderLayer"></i><i class="c0" title="layer 14: LlamaDecoderLayer"></i><i class="c0" title="layer 15: LlamaDecoderLayer"></i><i class="c0" title="layer 16: LlamaDecoderLayer"></i><i class="c0" title="layer 17: LlamaDecoderLayer"></i><i class="c0" title="layer 18: LlamaDecoderLayer"></i><i class="c0" title="layer 19: LlamaDecoderLayer"></i><i class="c0" title="layer 20: LlamaDecoderLayer"></i><i class="c0" title="layer 21: LlamaDecoderLayer"></i><i class="c0" title="layer 22: LlamaDecoderLayer"></i><i class="c0" title="layer 23: LlamaDecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: LlamaDecoderLayer"></i><i class="c0" title="layer 1: LlamaDecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>LlamaDecoderLayer</td><td>24</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.660565" title="model.language_model.layers: 1.2B (66.1%)"></i><i class="c1" style="flex:0.161929" title="model.vision_model.encoder.layers: 302.3M (16.2%)"></i><i class="c2" style="flex:0.112333" title="model.language_model: 209.7M (11.2%)"></i><i class="c3" style="flex:0.020230" title="model.generation_head: 37.8M (2.0%)"></i><i class="c4" style="flex:0.016482" title="model.vqmodel.decoder.up: 30.8M (1.6%)"></i><i class="c5" style="flex:0.012584" title="model.vqmodel: 23.5M (1.3%)"></i><i class="c6" style="flex:0.009437" title="model.vqmodel.encoder.down: 17.6M (0.9%)"></i><i class="c7" style="flex:0.006439" title="4 others: 12.0M (0.6%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.001003" title="model.language_model.layers: 82.2K (0.1%)"></i><i class="c1" style="flex:0.001220" title="model.vision_model.encoder.layers: 100.0K (0.1%)"></i><i class="c2" style="flex:0.080008" title="model.language_model: 6.6M (8.0%)"></i><i class="c3" style="flex:0.014602" title="model.generation_head: 1.2M (1.5%)"></i><i class="c4" style="flex:0.375654" title="model.vqmodel.decoder.up: 30.8M (37.6%)"></i><i class="c5" style="flex:0.298366" title="model.vqmodel: 24.4M (29.8%)"></i><i class="c6" style="flex:0.215090" title="model.vqmodel.encoder.down: 17.6M (21.5%)"></i><i class="c7" style="flex:0.014057" title="4 others: 1.2M (1.4%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.language_model.layers</td><td>1.2B</td><td>66.1%</td><td>82.2K</td><td>0.1%</td></tr><tr><td><i class="c1"></i>model.vision_model.encoder.layers</td><td>302.3M</td><td>16.2%</td><td>100.0K</td><td>0.1%</td></tr><tr><td><i class="c2"></i>model.language_model</td><td>209.7M</td><td>11.2%</td><td>6.6M</td><td>8.0%</td></tr><tr><td><i class="c3"></i>model.generation_head</td><td>37.8M</td><td>2.0%</td><td>1.2M</td><td>1.5%</td></tr><tr><td><i class="c4"></i>model.vqmodel.decoder.up</td><td>30.8M</td><td>1.6%</td><td>30.8M</td><td>37.6%</td></tr><tr><td><i class="c5"></i>model.vqmodel</td><td>23.5M</td><td>1.3%</td><td>24.4M</td><td>29.8%</td></tr><tr><td><i class="c6"></i>model.vqmodel.encoder.down</td><td>17.6M</td><td>0.9%</td><td>17.6M</td><td>21.5%</td></tr><tr><td><i class="c7"></i>4 others</td><td>12.0M</td><td>0.6%</td><td>1.2M</td><td>1.4%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>model_type</td><td>janus</td><td>janus</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>attention style</td><td>MHA (16 heads, 16 KV)</td><td>MHA (4 heads, 4 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>model_type</td><td>llama</td><td>llama</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>vocab_size</td><td>102400</td><td>102400</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>max_position_embeddings</td><td>16384</td><td>16384</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>model_type</td><td>janus_vision_model</td><td>janus_vision_model</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>hidden_act</td><td>gelu</td><td>gelu</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>image_size</td><td>384</td><td>384</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>patch_size</td><td>16</td><td>16</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>num_channels</td><td>3</td><td>3</td><td>βœ“ kept</td></tr><tr class=""><td>vq_config</td><td>model_type</td><td>janus_vqgan</td><td>janus_vqgan</td><td>βœ“ kept</td></tr><tr class=""><td>vq_config</td><td>hidden_act</td><td>gelu</td><td>gelu</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Janus omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>319</td><td>33</td></tr><tr class=""><td>Conv2d</td><td>105</td><td>105</td></tr><tr class=""><td>Dropout</td><td>77</td><td>33</td></tr><tr class=""><td>Identity</td><td>72</td><td>6</td></tr><tr class=""><td>GroupNorm</td><td>67</td><td>67</td></tr><tr class=""><td>LayerNorm</td><td>49</td><td>5</td></tr><tr class=""><td>LlamaRMSNorm</td><td>49</td><td>5</td></tr><tr class=""><td>VQVAEResnetBlock</td><td>29</td><td>29</td></tr><tr class=""><td>GELUActivation</td><td>27</td><td>5</td></tr><tr class=""><td>ModuleList</td><td>26</td><td>26</td></tr><tr class=""><td>VisionAttention</td><td>24</td><td>2</td></tr><tr class=""><td>VisionEncoderLayer</td><td>24</td><td>2</td></tr><tr class=""><td>VisionMLP</td><td>24</td><td>2</td></tr><tr class=""><td>LlamaAttention</td><td>24</td><td>2</td></tr><tr class=""><td>LlamaDecoderLayer</td><td>24</td><td>2</td></tr><tr class=""><td>LlamaMLP</td><td>24</td><td>2</td></tr><tr class=""><td>SiLUActivation</td><td>24</td><td>2</td></tr><tr class=""><td>Module</td><td>10</td><td>10</td></tr><tr class=""><td>VQVAEAttnBlock</td><td>7</td><td>7</td></tr><tr class=""><td>Embedding</td><td>4</td><td>4</td></tr><tr class=""><td>VQVAEConvDownsample</td><td>4</td><td>4</td></tr><tr class=""><td>VQVAEConvUpsample</td><td>4</td><td>4</td></tr><tr class=""><td>VQVAEMidBlock</td><td>2</td><td>2</td></tr><tr class=""><td>ForConditionalGeneration</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>VQVAE</td><td>1</td><td>1</td></tr><tr class=""><td>VQVAEAlignerMLP</td><td>1</td><td>1</td></tr><tr class=""><td>VQVAEDecoder</td><td>1</td><td>1</td></tr><tr class=""><td>VQVAEEncoder</td><td>1</td><td>1</td></tr><tr class=""><td>VQVAEHead</td><td>1</td><td>1</td></tr><tr class=""><td>VQVAEVectorQuantizer</td><td>1</td><td>1</td></tr><tr class=""><td>VisionAlignerMLP</td><td>1</td><td>1</td></tr><tr class=""><td>VisionEmbeddings</td><td>1</td><td>1</td></tr><tr class=""><td>VisionEncoder</td><td>1</td><td>1</td></tr><tr class=""><td>VisionModel</td><td>1</td><td>1</td></tr><tr class=""><td>LlamaModel</td><td>1</td><td>1</td></tr><tr class=""><td>LlamaRotaryEmbedding</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">12 fields changed, 82 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>text_config.head_dim</td><td>128</td><td>16</td></tr><tr><td>text_config.hidden_size</td><td>2048</td><td>64</td></tr><tr><td>text_config.intermediate_size</td><td>5632</td><td>128</td></tr><tr><td>text_config.num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>text_config.num_hidden_layers</td><td>24</td><td>2</td></tr><tr><td>text_config.num_key_value_heads</td><td>16</td><td>4</td></tr><tr><td>vision_config.hidden_size</td><td>1024</td><td>64</td></tr><tr><td>vision_config.num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>vision_config.num_hidden_layers</td><td>24</td><td>2</td></tr><tr><td>vision_config.projection_dim</td><td>2048</td><td>64</td></tr><tr><td>vq_config.embed_dim</td><td>8</td><td>64</td></tr><tr><td>vq_config.projection_dim</td><td>2048</td><td>64</td></tr></table>
36
  </main></body></html>
shrink/Janus-Pro-1B-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/Janus-Pro-1B-mic@5d0bfc05499ea83be331a434ea79583ca970581a", "findings": []}
shrink/LCO-Embedding-Omni-3B-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/LCO-Embedding-Omni-3B-mic@1e770f7015e573f2c4287893af7c6eb0c1d45c33 ← LCO-Embedding/LCO-Embedding-Omni-3B@73434565c33d032551639362f7fc91fa3c6f065f Β· Qwen2_5OmniThinkerForConditionalGeneration Β· random weights</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">4.7B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">21.1M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">3</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.audio_tower.layers</h3><div class="row"><span class="lab">Original Β· 32</span><div class="cells"><i class="c0" title="layer 0: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 1: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 2: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 3: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 4: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 5: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 6: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 7: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 8: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 9: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 10: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 11: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 12: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 13: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 14: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 15: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 16: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 17: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 18: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 19: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 20: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 21: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 22: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 23: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 24: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 25: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 26: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 27: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 28: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 29: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 30: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 31: Qwen2_5OmniAudioEncoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 1: Qwen2_5OmniAudioEncoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>Qwen2_5OmniAudioEncoderLayer</td><td>32</td><td>2</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.visual.blocks</h3><div class="row"><span class="lab">Original Β· 32</span><div class="cells"><i class="c0" title="layer 0: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 1: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 2: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 3: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 4: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 5: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 6: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 7: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 8: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 9: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 10: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 11: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 12: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 13: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 14: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 15: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 16: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 17: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 18: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 19: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 20: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 21: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 22: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 23: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 24: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 25: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 26: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 27: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 28: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 29: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 30: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 31: Qwen2_5OmniVisionBlock"></i></div></div><div class="row"><span class="lab">Fixture Β· 32</span><div class="cells"><i class="c0" title="layer 0: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 1: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 2: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 3: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 4: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 5: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 6: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 7: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 8: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 9: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 10: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 11: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 12: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 13: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 14: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 15: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 16: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 17: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 18: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 19: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 20: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 21: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 22: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 23: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 24: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 25: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 26: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 27: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 28: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 29: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 30: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 31: Qwen2_5OmniVisionBlock"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>Qwen2_5OmniVisionBlock</td><td>32</td><td>32</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.language_model.layers</h3><div class="row"><span class="lab">Original Β· 36</span><div class="cells"><i class="c0" title="layer 0: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 1: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 2: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 3: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 4: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 5: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 6: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 7: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 8: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 9: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 10: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 11: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 12: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 13: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 14: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 15: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 16: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 17: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 18: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 19: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 20: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 21: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 22: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 23: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 24: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 25: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 26: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 27: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 28: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 29: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 30: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 31: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 32: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 33: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 34: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 35: Qwen2_5OmniDecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 1: Qwen2_5OmniDecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>Qwen2_5OmniDecoderLayer</td><td>36</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.589942" title="model.language_model.layers: 2.8B (59.0%)"></i><i class="c1" style="flex:0.134044" title="model.visual.blocks: 630.5M (13.4%)"></i><i class="c2" style="flex:0.133867" title="model.audio_tower.layers: 629.6M (13.4%)"></i><i class="c3" style="flex:0.066157" title="model.language_model: 311.2M (6.6%)"></i><i class="c4" style="flex:0.066157" title="lm_head: 311.2M (6.6%)"></i><i class="c5" style="flex:0.008125" title="model.visual: 38.2M (0.8%)"></i><i class="c6" style="flex:0.001709" title="model.audio_tower: 8.0M (0.2%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.003515" title="model.language_model.layers: 74.2K (0.4%)"></i><i class="c1" style="flex:0.063124" title="model.visual.blocks: 1.3M (6.3%)"></i><i class="c2" style="flex:0.003163" title="model.audio_tower.layers: 66.8K (0.3%)"></i><i class="c3" style="flex:0.460390" title="model.language_model: 9.7M (46.0%)"></i><i class="c4" style="flex:0.460387" title="lm_head: 9.7M (46.0%)"></i><i class="c5" style="flex:0.007460" title="model.visual: 157.6K (0.7%)"></i><i class="c6" style="flex:0.001960" title="model.audio_tower: 41.4K (0.2%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.language_model.layers</td><td>2.8B</td><td>59.0%</td><td>74.2K</td><td>0.4%</td></tr><tr><td><i class="c1"></i>model.visual.blocks</td><td>630.5M</td><td>13.4%</td><td>1.3M</td><td>6.3%</td></tr><tr><td><i class="c2"></i>model.audio_tower.layers</td><td>629.6M</td><td>13.4%</td><td>66.8K</td><td>0.3%</td></tr><tr><td><i class="c3"></i>model.language_model</td><td>311.2M</td><td>6.6%</td><td>9.7M</td><td>46.0%</td></tr><tr><td><i class="c4"></i>lm_head</td><td>311.2M</td><td>6.6%</td><td>9.7M</td><td>46.0%</td></tr><tr><td><i class="c5"></i>model.visual</td><td>38.2M</td><td>0.8%</td><td>157.6K</td><td>0.7%</td></tr><tr><td><i class="c6"></i>model.audio_tower</td><td>8.0M</td><td>0.2%</td><td>41.4K</td><td>0.2%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>model_type</td><td>qwen2_5_omni_thinker</td><td>qwen2_5_omni_thinker</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>audio_config</td><td>model_type</td><td>qwen2_5_omni_audio_encoder</td><td>qwen2_5_omni_audio_encoder</td><td>βœ“ kept</td></tr><tr class=""><td>audio_config</td><td>num_mel_bins</td><td>128</td><td>128</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>RoPE type</td><td>axial</td><td>axial</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>model_type</td><td>qwen2_5_omni_vision_encoder</td><td>qwen2_5_omni_vision_encoder</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>patch_size</td><td>14</td><td>14</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>attention style</td><td>GQA (16 heads, 2 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>model_type</td><td>qwen2_5_omni_text</td><td>qwen2_5_omni_text</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>vocab_size</td><td>151936</td><td>151936</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>max_position_embeddings</td><td>32768</td><td>32768</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Qwen2_5OmniThinker omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>672</td><td>254</td></tr><tr class=""><td>Qwen2_5OmniRMSNorm</td><td>138</td><td>70</td></tr><tr class=""><td>SiLUActivation</td><td>68</td><td>34</td></tr><tr class=""><td>LayerNorm</td><td>65</td><td>5</td></tr><tr class=""><td>Qwen2MLP</td><td>36</td><td>2</td></tr><tr class=""><td>Qwen2_5OmniAttention</td><td>36</td><td>2</td></tr><tr class=""><td>Qwen2_5OmniDecoderLayer</td><td>36</td><td>2</td></tr><tr class=""><td>GELUActivation</td><td>32</td><td>2</td></tr><tr class=""><td>Qwen2_5OmniAudioAttention</td><td>32</td><td>2</td></tr><tr class=""><td>Qwen2_5OmniAudioEncoderLayer</td><td>32</td><td>2</td></tr><tr class=""><td>Qwen2_5OmniMLP</td><td>32</td><td>32</td></tr><tr class=""><td>Qwen2_5OmniVisionAttention</td><td>32</td><td>32</td></tr><tr class=""><td>Qwen2_5OmniVisionBlock</td><td>32</td><td>32</td></tr><tr class=""><td>ModuleList</td><td>3</td><td>3</td></tr><tr class=""><td>Conv1d</td><td>2</td><td>2</td></tr><tr class=""><td>Embedding</td><td>2</td><td>2</td></tr><tr class=""><td>AvgPool1d</td><td>1</td><td>1</td></tr><tr class=""><td>Conv3d</td><td>1</td><td>1</td></tr><tr class=""><td>GELU</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen2_5OmniAudioEncoder</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen2_5OmniPatchMerger</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen2_5OmniRotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForConditionalGeneration</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>TextModel</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen2_5OmniVisionEncoder</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen2_5OmniVisionRotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen2_5_VisionPatchEmbed</td><td>1</td><td>1</td></tr><tr class=""><td>Sequential</td><td>1</td><td>1</td></tr><tr class=""><td>SinusoidsPositionEmbedding</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">16 fields changed, 102 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>audio_config.d_model</td><td>1280</td><td>64</td></tr><tr><td>audio_config.encoder_attention_heads</td><td>20</td><td>4</td></tr><tr><td>audio_config.encoder_ffn_dim</td><td>5120</td><td>128</td></tr><tr><td>audio_config.encoder_layers</td><td>32</td><td>2</td></tr><tr><td>audio_config.output_dim</td><td>2048</td><td>64</td></tr><tr><td>text_config.hidden_size</td><td>2048</td><td>64</td></tr><tr><td>text_config.intermediate_size</td><td>11008</td><td>128</td></tr><tr><td>text_config.layer_types</td><td>36 entries: 36Γ— full_attention</td><td>[&quot;full_attention&quot;, &quot;full_attention&quot;]</td></tr><tr><td>text_config.num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>text_config.num_hidden_layers</td><td>36</td><td>2</td></tr><tr><td>text_config.rope_parameters</td><td>{&quot;mrope_section&quot;: [16, 24, 24], &quot;rope_type&quot;: &quot;default&quot;, &quot;type&quot;: &quot;default&quot;, &quot;r…</td><td>{&quot;mrope_section&quot;: [2, 3, 3], &quot;rope_theta&quot;: 1000000.0, &quot;rope_type&quot;: &quot;default&quot;,…</td></tr><tr><td>vision_config.embed_dim</td><td>1280</td><td>64</td></tr><tr><td>vision_config.hidden_size</td><td>1280</td><td>64</td></tr><tr><td>vision_config.intermediate_size</td><td>3420</td><td>128</td></tr><tr><td>vision_config.num_heads</td><td>16</td><td>4</td></tr><tr><td>vision_config.out_hidden_size</td><td>2048</td><td>64</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>LCO-Embedding/LCO-Embedding-Omni-3B: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>LCO-Embedding/LCO-Embedding-Omni-3B</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/LCO-Embedding-Omni-3B-mic/tree/1e770f7015e573f2c4287893af7c6eb0c1d45c33" title="huggingface-mic/LCO-Embedding-Omni-3B-mic@1e770f7015e573f2c4287893af7c6eb0c1d45c33">huggingface-mic/LCO-Embedding-Omni-3B-mic@1e770f7015e5</a> ← <a href="https://huggingface.co/LCO-Embedding/LCO-Embedding-Omni-3B/tree/73434565c33d032551639362f7fc91fa3c6f065f" title="LCO-Embedding/LCO-Embedding-Omni-3B@73434565c33d032551639362f7fc91fa3c6f065f">LCO-Embedding/LCO-Embedding-Omni-3B@73434565c33d</a> Β· Qwen2_5OmniThinkerForConditionalGeneration Β· random weights</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">4.7B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">21.1M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">3</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.audio_tower.layers</h3><div class="row"><span class="lab">Original Β· 32</span><div class="cells"><i class="c0" title="layer 0: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 1: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 2: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 3: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 4: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 5: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 6: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 7: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 8: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 9: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 10: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 11: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 12: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 13: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 14: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 15: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 16: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 17: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 18: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 19: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 20: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 21: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 22: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 23: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 24: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 25: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 26: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 27: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 28: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 29: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 30: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 31: Qwen2_5OmniAudioEncoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: Qwen2_5OmniAudioEncoderLayer"></i><i class="c0" title="layer 1: Qwen2_5OmniAudioEncoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>Qwen2_5OmniAudioEncoderLayer</td><td>32</td><td>2</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.visual.blocks</h3><div class="row"><span class="lab">Original Β· 32</span><div class="cells"><i class="c0" title="layer 0: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 1: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 2: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 3: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 4: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 5: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 6: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 7: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 8: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 9: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 10: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 11: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 12: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 13: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 14: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 15: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 16: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 17: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 18: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 19: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 20: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 21: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 22: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 23: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 24: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 25: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 26: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 27: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 28: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 29: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 30: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 31: Qwen2_5OmniVisionBlock"></i></div></div><div class="row"><span class="lab">Fixture Β· 32</span><div class="cells"><i class="c0" title="layer 0: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 1: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 2: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 3: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 4: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 5: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 6: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 7: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 8: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 9: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 10: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 11: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 12: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 13: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 14: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 15: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 16: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 17: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 18: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 19: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 20: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 21: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 22: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 23: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 24: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 25: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 26: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 27: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 28: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 29: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 30: Qwen2_5OmniVisionBlock"></i><i class="c0" title="layer 31: Qwen2_5OmniVisionBlock"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>Qwen2_5OmniVisionBlock</td><td>32</td><td>32</td><td>βœ“ kept</td></tr></table></div><div class="stack"><h3>model.language_model.layers</h3><div class="row"><span class="lab">Original Β· 36</span><div class="cells"><i class="c0" title="layer 0: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 1: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 2: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 3: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 4: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 5: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 6: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 7: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 8: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 9: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 10: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 11: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 12: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 13: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 14: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 15: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 16: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 17: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 18: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 19: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 20: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 21: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 22: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 23: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 24: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 25: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 26: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 27: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 28: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 29: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 30: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 31: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 32: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 33: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 34: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 35: Qwen2_5OmniDecoderLayer"></i></div></div><div class="row"><span class="lab">Fixture Β· 2</span><div class="cells"><i class="c0" title="layer 0: Qwen2_5OmniDecoderLayer"></i><i class="c0" title="layer 1: Qwen2_5OmniDecoderLayer"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>Qwen2_5OmniDecoderLayer</td><td>36</td><td>2</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.589942" title="model.language_model.layers: 2.8B (59.0%)"></i><i class="c1" style="flex:0.134044" title="model.visual.blocks: 630.5M (13.4%)"></i><i class="c2" style="flex:0.133867" title="model.audio_tower.layers: 629.6M (13.4%)"></i><i class="c3" style="flex:0.066157" title="model.language_model: 311.2M (6.6%)"></i><i class="c4" style="flex:0.066157" title="lm_head: 311.2M (6.6%)"></i><i class="c5" style="flex:0.008125" title="model.visual: 38.2M (0.8%)"></i><i class="c6" style="flex:0.001709" title="model.audio_tower: 8.0M (0.2%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.003515" title="model.language_model.layers: 74.2K (0.4%)"></i><i class="c1" style="flex:0.063124" title="model.visual.blocks: 1.3M (6.3%)"></i><i class="c2" style="flex:0.003163" title="model.audio_tower.layers: 66.8K (0.3%)"></i><i class="c3" style="flex:0.460390" title="model.language_model: 9.7M (46.0%)"></i><i class="c4" style="flex:0.460387" title="lm_head: 9.7M (46.0%)"></i><i class="c5" style="flex:0.007460" title="model.visual: 157.6K (0.7%)"></i><i class="c6" style="flex:0.001960" title="model.audio_tower: 41.4K (0.2%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.language_model.layers</td><td>2.8B</td><td>59.0%</td><td>74.2K</td><td>0.4%</td></tr><tr><td><i class="c1"></i>model.visual.blocks</td><td>630.5M</td><td>13.4%</td><td>1.3M</td><td>6.3%</td></tr><tr><td><i class="c2"></i>model.audio_tower.layers</td><td>629.6M</td><td>13.4%</td><td>66.8K</td><td>0.3%</td></tr><tr><td><i class="c3"></i>model.language_model</td><td>311.2M</td><td>6.6%</td><td>9.7M</td><td>46.0%</td></tr><tr><td><i class="c4"></i>lm_head</td><td>311.2M</td><td>6.6%</td><td>9.7M</td><td>46.0%</td></tr><tr><td><i class="c5"></i>model.visual</td><td>38.2M</td><td>0.8%</td><td>157.6K</td><td>0.7%</td></tr><tr><td><i class="c6"></i>model.audio_tower</td><td>8.0M</td><td>0.2%</td><td>41.4K</td><td>0.2%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>model_type</td><td>qwen2_5_omni_thinker</td><td>qwen2_5_omni_thinker</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>False</td><td>False</td><td>βœ“ kept</td></tr><tr class=""><td>audio_config</td><td>model_type</td><td>qwen2_5_omni_audio_encoder</td><td>qwen2_5_omni_audio_encoder</td><td>βœ“ kept</td></tr><tr class=""><td>audio_config</td><td>num_mel_bins</td><td>128</td><td>128</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>RoPE type</td><td>axial</td><td>axial</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>model_type</td><td>qwen2_5_omni_vision_encoder</td><td>qwen2_5_omni_vision_encoder</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>vision_config</td><td>patch_size</td><td>14</td><td>14</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>attention style</td><td>GQA (16 heads, 2 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>model_type</td><td>qwen2_5_omni_text</td><td>qwen2_5_omni_text</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>vocab_size</td><td>151936</td><td>151936</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>hidden_act</td><td>silu</td><td>silu</td><td>βœ“ kept</td></tr><tr class=""><td>text_config</td><td>max_position_embeddings</td><td>32768</td><td>32768</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Qwen2_5OmniThinker omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>672</td><td>254</td></tr><tr class=""><td>Qwen2_5OmniRMSNorm</td><td>138</td><td>70</td></tr><tr class=""><td>SiLUActivation</td><td>68</td><td>34</td></tr><tr class=""><td>LayerNorm</td><td>65</td><td>5</td></tr><tr class=""><td>Qwen2MLP</td><td>36</td><td>2</td></tr><tr class=""><td>Qwen2_5OmniAttention</td><td>36</td><td>2</td></tr><tr class=""><td>Qwen2_5OmniDecoderLayer</td><td>36</td><td>2</td></tr><tr class=""><td>GELUActivation</td><td>32</td><td>2</td></tr><tr class=""><td>Qwen2_5OmniAudioAttention</td><td>32</td><td>2</td></tr><tr class=""><td>Qwen2_5OmniAudioEncoderLayer</td><td>32</td><td>2</td></tr><tr class=""><td>Qwen2_5OmniMLP</td><td>32</td><td>32</td></tr><tr class=""><td>Qwen2_5OmniVisionAttention</td><td>32</td><td>32</td></tr><tr class=""><td>Qwen2_5OmniVisionBlock</td><td>32</td><td>32</td></tr><tr class=""><td>ModuleList</td><td>3</td><td>3</td></tr><tr class=""><td>Conv1d</td><td>2</td><td>2</td></tr><tr class=""><td>Embedding</td><td>2</td><td>2</td></tr><tr class=""><td>AvgPool1d</td><td>1</td><td>1</td></tr><tr class=""><td>Conv3d</td><td>1</td><td>1</td></tr><tr class=""><td>GELU</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen2_5OmniAudioEncoder</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen2_5OmniPatchMerger</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen2_5OmniRotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForConditionalGeneration</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>TextModel</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen2_5OmniVisionEncoder</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen2_5OmniVisionRotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>Qwen2_5_VisionPatchEmbed</td><td>1</td><td>1</td></tr><tr class=""><td>Sequential</td><td>1</td><td>1</td></tr><tr class=""><td>SinusoidsPositionEmbedding</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">16 fields changed, 102 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>audio_config.d_model</td><td>1280</td><td>64</td></tr><tr><td>audio_config.encoder_attention_heads</td><td>20</td><td>4</td></tr><tr><td>audio_config.encoder_ffn_dim</td><td>5120</td><td>128</td></tr><tr><td>audio_config.encoder_layers</td><td>32</td><td>2</td></tr><tr><td>audio_config.output_dim</td><td>2048</td><td>64</td></tr><tr><td>text_config.hidden_size</td><td>2048</td><td>64</td></tr><tr><td>text_config.intermediate_size</td><td>11008</td><td>128</td></tr><tr><td>text_config.layer_types</td><td>36 entries: 36Γ— full_attention</td><td>[&quot;full_attention&quot;, &quot;full_attention&quot;]</td></tr><tr><td>text_config.num_attention_heads</td><td>16</td><td>4</td></tr><tr><td>text_config.num_hidden_layers</td><td>36</td><td>2</td></tr><tr><td>text_config.rope_parameters</td><td>{&quot;mrope_section&quot;: [16, 24, 24], &quot;rope_type&quot;: &quot;default&quot;, &quot;type&quot;: &quot;default&quot;, &quot;r…</td><td>{&quot;mrope_section&quot;: [2, 3, 3], &quot;rope_theta&quot;: 1000000.0, &quot;rope_type&quot;: &quot;default&quot;,…</td></tr><tr><td>vision_config.embed_dim</td><td>1280</td><td>64</td></tr><tr><td>vision_config.hidden_size</td><td>1280</td><td>64</td></tr><tr><td>vision_config.intermediate_size</td><td>3420</td><td>128</td></tr><tr><td>vision_config.num_heads</td><td>16</td><td>4</td></tr><tr><td>vision_config.out_hidden_size</td><td>2048</td><td>64</td></tr></table>
36
  </main></body></html>
shrink/LCO-Embedding-Omni-3B-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/LCO-Embedding-Omni-3B-mic@1e770f7015e573f2c4287893af7c6eb0c1d45c33", "findings": []}
shrink/LFM2-8B-A1B-mic.html CHANGED
@@ -1,6 +1,6 @@
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
- <title>Shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
@@ -9,6 +9,7 @@ body{margin:0;background:var(--bg);color:var(--fg);font:15px/1.5 system-ui,sans-
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
 
12
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
13
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
14
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
@@ -29,6 +30,7 @@ td i{display:inline-block;width:10px;height:10px;border-radius:2px;margin-right:
29
  tr.bad td{color:var(--warn)}
30
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
31
  </style></head><body><main>
32
- <h1>Shrunk architecture</h1><p class="meta">huggingface-mic/LFM2-8B-A1B-mic@02493cd540778260b98790f375fb2070e8acd824 ← LiquidAI/LFM2-8B-A1B@c1c44ff9fc00db3ebf4516970563f5f383d23670 Β· Lfm2MoeForCausalLM Β· random weights</p>
 
33
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">8.3B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">4.4M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 24</span><div class="cells"><i class="c0" title="layer 0: conv, Conv1d, MLP …"></i><i class="c0" title="layer 1: conv, Conv1d, MLP …"></i><i class="c1" title="layer 2: is_causal=True, Attention, Experts …"></i><i class="c2" title="layer 3: conv, Conv1d, Experts …"></i><i class="c2" title="layer 4: conv, Conv1d, Experts …"></i><i class="c2" title="layer 5: conv, Conv1d, Experts …"></i><i class="c1" title="layer 6: is_causal=True, Attention, Experts …"></i><i class="c2" title="layer 7: conv, Conv1d, Experts …"></i><i class="c2" title="layer 8: conv, Conv1d, Experts …"></i><i class="c2" title="layer 9: conv, Conv1d, Experts …"></i><i class="c1" title="layer 10: is_causal=True, Attention, Experts …"></i><i class="c2" title="layer 11: conv, Conv1d, Experts …"></i><i class="c2" title="layer 12: conv, Conv1d, Experts …"></i><i class="c2" title="layer 13: conv, Conv1d, Experts …"></i><i class="c1" title="layer 14: is_causal=True, Attention, Experts …"></i><i class="c2" title="layer 15: conv, Conv1d, Experts …"></i><i class="c2" title="layer 16: conv, Conv1d, Experts …"></i><i class="c2" title="layer 17: conv, Conv1d, Experts …"></i><i class="c1" title="layer 18: is_causal=True, Attention, Experts …"></i><i class="c2" title="layer 19: conv, Conv1d, Experts …"></i><i class="c2" title="layer 20: conv, Conv1d, Experts …"></i><i class="c1" title="layer 21: is_causal=True, Attention, Experts …"></i><i class="c2" title="layer 22: conv, Conv1d, Experts …"></i><i class="c2" title="layer 23: conv, Conv1d, Experts …"></i></div></div><div class="row"><span class="lab">Fixture Β· 4</span><div class="cells"><i class="c0" title="layer 0: conv, Conv1d, MLP …"></i><i class="c0" title="layer 1: conv, Conv1d, MLP …"></i><i class="c1" title="layer 2: is_causal=True, Attention, Experts …"></i><i class="c2" title="layer 3: conv, Conv1d, Experts …"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>conv, Conv1d, MLP …</td><td>2</td><td>2</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>is_causal=True, Attention, Experts …</td><td>6</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c2"></i>conv, Conv1d, Experts …</td><td>16</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.983906" title="model.layers: 8.2B (98.4%)"></i><i class="c1" style="flex:0.016093" title="model.embed_tokens: 134.2M (1.6%)"></i><i class="c2" style="flex:0.000000" title="model.embedding_norm: 2.0K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.037049" title="model.layers: 161.4K (3.7%)"></i><i class="c1" style="flex:0.962936" title="model.embed_tokens: 4.2M (96.3%)"></i><i class="c2" style="flex:0.000015" title="model.embedding_norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>8.2B</td><td>98.4%</td><td>161.4K</td><td>3.7%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>134.2M</td><td>1.6%</td><td>4.2M</td><td>96.3%</td></tr><tr><td><i class="c2"></i>model.embedding_norm</td><td>2.0K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>GQA (32 heads, 8 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-4 of 32 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>lfm2_moe</td><td>lfm2_moe</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>65536</td><td>65536</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>128000</td><td>128000</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Lfm2Moe omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>67</td><td>17</td></tr><tr class=""><td>RMSNorm</td><td>61</td><td>11</td></tr><tr class=""><td>DecoderLayer</td><td>24</td><td>4</td></tr><tr class=""><td>Experts</td><td>22</td><td>2</td></tr><tr class=""><td>SparseMoeBlock</td><td>22</td><td>2</td></tr><tr class=""><td>TopKRouter</td><td>22</td><td>2</td></tr><tr class=""><td>Conv1d</td><td>18</td><td>3</td></tr><tr class=""><td>ShortConv</td><td>18</td><td>3</td></tr><tr class=""><td>Attention</td><td>6</td><td>1</td></tr><tr class=""><td>MLP</td><td>2</td><td>2</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">9 fields changed, 26 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>hidden_size</td><td>2048</td><td>64</td></tr><tr><td>intermediate_size</td><td>7168</td><td>128</td></tr><tr><td>layer_types</td><td>24 entries: 18Γ— conv, 6Γ— full_attention</td><td>[&quot;conv&quot;, &quot;conv&quot;, &quot;full_attention&quot;, &quot;conv&quot;]</td></tr><tr><td>moe_intermediate_size</td><td>1792</td><td>32</td></tr><tr><td>num_attention_heads</td><td>32</td><td>4</td></tr><tr><td>num_experts</td><td>32</td><td>4</td></tr><tr><td>num_experts_per_tok</td><td>4</td><td>2</td></tr><tr><td>num_hidden_layers</td><td>24</td><td>4</td></tr><tr><td>num_key_value_heads</td><td>8</td><td>2</td></tr></table>
34
  </main></body></html>
 
1
  <!doctype html>
2
  <html lang="en"><head><meta charset="utf-8"><meta name="viewport" content="width=device-width,initial-scale=1">
3
+ <title>LiquidAI/LFM2-8B-A1B: shrunk architecture</title>
4
  <style>
5
  :root{--bg:#fcfcfb;--fg:#0b0b0b;--muted:#52514e;--line:#e4e3df;--ok:#1d7a43;--warn:#b35c00;--s0:#2a78d6;--s1:#eb6834;--s2:#1baf7a;--s3:#eda100;--s4:#e87ba4;--s5:#008300;--s6:#6250d6;--s7:#e34948;--s8:#8a8984;}
6
  @media (prefers-color-scheme:dark){:root:not([data-theme="light"]){--bg:#1a1a19;--fg:#f0efec;--muted:#c3c2b7;--line:#3a3936;--ok:#2f9e5a;--warn:#d9822b;--s0:#3987e5;--s1:#d95926;--s2:#199e70;--s3:#c98500;--s4:#d55181;--s5:#008300;--s6:#9085e9;--s7:#e66767;--s8:#6f6e69;}}
 
9
  main{max-width:1040px;margin:0 auto;padding:24px 16px 64px}
10
  h1{font-size:24px;margin:0 0 4px}h2{font-size:18px;margin:36px 0 8px}h3{font-size:14px;margin:0 0 8px;font-family:ui-monospace,monospace}
11
  .meta,.note{color:var(--muted);margin:0 0 12px;overflow-wrap:anywhere}
12
+ a{color:inherit}
13
  .stats{display:grid;grid-template-columns:repeat(auto-fit,minmax(150px,1fr));gap:12px;margin:20px 0}
14
  .stat{border:1px solid var(--line);border-radius:8px;padding:12px;display:flex;flex-direction:column}
15
  .stat .v{font-size:26px;font-weight:600}.stat .k{color:var(--muted);font-size:13px}
 
30
  tr.bad td{color:var(--warn)}
31
  .c0{background:var(--s0)}.c1{background:var(--s1)}.c2{background:var(--s2)}.c3{background:var(--s3)}.c4{background:var(--s4)}.c5{background:var(--s5)}.c6{background:var(--s6)}.c7{background:var(--s7)}.c8{background:var(--s8)}
32
  </style></head><body><main>
33
+ <p class="meta"><a href="../index.html">← MIC coverage of Transformers model types</a></p>
34
+ <h1>LiquidAI/LFM2-8B-A1B</h1><p class="meta"><a href="https://huggingface.co/huggingface-mic/LFM2-8B-A1B-mic/tree/02493cd540778260b98790f375fb2070e8acd824" title="huggingface-mic/LFM2-8B-A1B-mic@02493cd540778260b98790f375fb2070e8acd824">huggingface-mic/LFM2-8B-A1B-mic@02493cd54077</a> ← <a href="https://huggingface.co/LiquidAI/LFM2-8B-A1B/tree/c1c44ff9fc00db3ebf4516970563f5f383d23670" title="LiquidAI/LFM2-8B-A1B@c1c44ff9fc00db3ebf4516970563f5f383d23670">LiquidAI/LFM2-8B-A1B@c1c44ff9fc00</a> Β· Lfm2MoeForCausalLM Β· random weights</p>
35
  <div class="stats"><div class="stat"><span class="k">Original</span><span class="v">8.3B</span><span class="k">parameters</span></div><div class="stat"><span class="k">Fixture</span><span class="v">4.4M</span><span class="k">parameters</span></div><div class="stat"><span class="k">Layer stacks</span><span class="v">1</span><span class="k">compared</span></div><div class="stat"><span class="k">Findings</span><span class="v">0</span><span class="k">to review</span></div></div><section class="findings ok"><h2>βœ“ Same architecture, smaller</h2><p>Every module class, layer kind and checked property of the original is in the fixture.</p></section><h2>Layer stacks</h2><p class="note">One square per layer, colored by kind: the layer's module classes and type attributes. Hover a square for its index and kind.</p><div class="stack"><h3>model.layers</h3><div class="row"><span class="lab">Original Β· 24</span><div class="cells"><i class="c0" title="layer 0: conv, Conv1d, MLP …"></i><i class="c0" title="layer 1: conv, Conv1d, MLP …"></i><i class="c1" title="layer 2: is_causal=True, Attention, Experts …"></i><i class="c2" title="layer 3: conv, Conv1d, Experts …"></i><i class="c2" title="layer 4: conv, Conv1d, Experts …"></i><i class="c2" title="layer 5: conv, Conv1d, Experts …"></i><i class="c1" title="layer 6: is_causal=True, Attention, Experts …"></i><i class="c2" title="layer 7: conv, Conv1d, Experts …"></i><i class="c2" title="layer 8: conv, Conv1d, Experts …"></i><i class="c2" title="layer 9: conv, Conv1d, Experts …"></i><i class="c1" title="layer 10: is_causal=True, Attention, Experts …"></i><i class="c2" title="layer 11: conv, Conv1d, Experts …"></i><i class="c2" title="layer 12: conv, Conv1d, Experts …"></i><i class="c2" title="layer 13: conv, Conv1d, Experts …"></i><i class="c1" title="layer 14: is_causal=True, Attention, Experts …"></i><i class="c2" title="layer 15: conv, Conv1d, Experts …"></i><i class="c2" title="layer 16: conv, Conv1d, Experts …"></i><i class="c2" title="layer 17: conv, Conv1d, Experts …"></i><i class="c1" title="layer 18: is_causal=True, Attention, Experts …"></i><i class="c2" title="layer 19: conv, Conv1d, Experts …"></i><i class="c2" title="layer 20: conv, Conv1d, Experts …"></i><i class="c1" title="layer 21: is_causal=True, Attention, Experts …"></i><i class="c2" title="layer 22: conv, Conv1d, Experts …"></i><i class="c2" title="layer 23: conv, Conv1d, Experts …"></i></div></div><div class="row"><span class="lab">Fixture Β· 4</span><div class="cells"><i class="c0" title="layer 0: conv, Conv1d, MLP …"></i><i class="c0" title="layer 1: conv, Conv1d, MLP …"></i><i class="c1" title="layer 2: is_causal=True, Attention, Experts …"></i><i class="c2" title="layer 3: conv, Conv1d, Experts …"></i></div></div><table class="legend"><tr><th>Kind</th><th>Original</th><th>Fixture</th><th></th></tr><tr><td><i class="c0"></i>conv, Conv1d, MLP …</td><td>2</td><td>2</td><td>βœ“ kept</td></tr><tr><td><i class="c1"></i>is_causal=True, Attention, Experts …</td><td>6</td><td>1</td><td>βœ“ kept</td></tr><tr><td><i class="c2"></i>conv, Conv1d, Experts …</td><td>16</td><td>1</td><td>βœ“ kept</td></tr></table></div><h2>Where the parameters are</h2><p class="note">Share of each model's parameters per component. Vocabulary and patch sizes are kept, so embeddings usually take a larger share of the fixture.</p><div class="row"><span class="lab">Original</span><div class="bar"><i class="c0" style="flex:0.983906" title="model.layers: 8.2B (98.4%)"></i><i class="c1" style="flex:0.016093" title="model.embed_tokens: 134.2M (1.6%)"></i><i class="c2" style="flex:0.000000" title="model.embedding_norm: 2.0K (0.0%)"></i></div></div><div class="row"><span class="lab">Fixture</span><div class="bar"><i class="c0" style="flex:0.037049" title="model.layers: 161.4K (3.7%)"></i><i class="c1" style="flex:0.962936" title="model.embed_tokens: 4.2M (96.3%)"></i><i class="c2" style="flex:0.000015" title="model.embedding_norm: 64 (0.0%)"></i></div></div><table class="legend"><tr><th>Component</th><th>Original</th><th>Share</th><th>Fixture</th><th>Share</th></tr><tr><td><i class="c0"></i>model.layers</td><td>8.2B</td><td>98.4%</td><td>161.4K</td><td>3.7%</td></tr><tr><td><i class="c1"></i>model.embed_tokens</td><td>134.2M</td><td>1.6%</td><td>4.2M</td><td>96.3%</td></tr><tr><td><i class="c2"></i>model.embedding_norm</td><td>2.0K</td><td>0.0%</td><td>64</td><td>0.0%</td></tr></table><h2>Preserved properties</h2><table><tr><th>Config</th><th>Property</th><th>Original</th><th>Fixture</th><th></th></tr><tr class=""><td>model</td><td>attention style</td><td>GQA (32 heads, 8 KV)</td><td>GQA (4 heads, 2 KV)</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>MoE routing</td><td>top-4 of 32 experts</td><td>top-2 of 4 experts</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>RoPE type</td><td>default</td><td>default</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>model_type</td><td>lfm2_moe</td><td>lfm2_moe</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>vocab_size</td><td>65536</td><td>65536</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>tie_word_embeddings</td><td>True</td><td>True</td><td>βœ“ kept</td></tr><tr class=""><td>model</td><td>max_position_embeddings</td><td>128000</td><td>128000</td><td>βœ“ kept</td></tr></table><h2>Module classes</h2><p class="note">Instances of each class in the built model (prefix Lfm2Moe omitted).</p><table><tr><th>Class</th><th>Original</th><th>Fixture</th></tr><tr class=""><td>Linear</td><td>67</td><td>17</td></tr><tr class=""><td>RMSNorm</td><td>61</td><td>11</td></tr><tr class=""><td>DecoderLayer</td><td>24</td><td>4</td></tr><tr class=""><td>Experts</td><td>22</td><td>2</td></tr><tr class=""><td>SparseMoeBlock</td><td>22</td><td>2</td></tr><tr class=""><td>TopKRouter</td><td>22</td><td>2</td></tr><tr class=""><td>Conv1d</td><td>18</td><td>3</td></tr><tr class=""><td>ShortConv</td><td>18</td><td>3</td></tr><tr class=""><td>Attention</td><td>6</td><td>1</td></tr><tr class=""><td>MLP</td><td>2</td><td>2</td></tr><tr class=""><td>Embedding</td><td>1</td><td>1</td></tr><tr class=""><td>ForCausalLM</td><td>1</td><td>1</td></tr><tr class=""><td>Model</td><td>1</td><td>1</td></tr><tr class=""><td>RotaryEmbedding</td><td>1</td><td>1</td></tr><tr class=""><td>ModuleList</td><td>1</td><td>1</td></tr></table><h2>Config changes</h2><p class="note">9 fields changed, 26 unchanged.</p><table><tr><th>Field</th><th>Original</th><th>Fixture</th></tr><tr><td>hidden_size</td><td>2048</td><td>64</td></tr><tr><td>intermediate_size</td><td>7168</td><td>128</td></tr><tr><td>layer_types</td><td>24 entries: 18Γ— conv, 6Γ— full_attention</td><td>[&quot;conv&quot;, &quot;conv&quot;, &quot;full_attention&quot;, &quot;conv&quot;]</td></tr><tr><td>moe_intermediate_size</td><td>1792</td><td>32</td></tr><tr><td>num_attention_heads</td><td>32</td><td>4</td></tr><tr><td>num_experts</td><td>32</td><td>4</td></tr><tr><td>num_experts_per_tok</td><td>4</td><td>2</td></tr><tr><td>num_hidden_layers</td><td>24</td><td>4</td></tr><tr><td>num_key_value_heads</td><td>8</td><td>2</td></tr></table>
36
  </main></body></html>
shrink/LFM2-8B-A1B-mic.json ADDED
@@ -0,0 +1 @@
 
 
1
+ {"fixture": "huggingface-mic/LFM2-8B-A1B-mic@02493cd540778260b98790f375fb2070e8acd824", "findings": []}