Add README, KLD plots, measurement

.gitattributes

@@ -33,3 +33,6 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+kld_plot.png filter=lfs diff=lfs merge=lfs -text
+kld_spread_plot.png filter=lfs diff=lfs merge=lfs -text
+example_dashboard.jpg filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,186 @@
+---
+license: apache-2.0
+base_model: microsoft/AesCode-32B
+base_model_relation: quantized
+quantized_by: WeReCooking
+pipeline_tag: image-text-to-text
+tags:
+- exl3
+- qwen3_vl
+- html
+---
+
+<style>
+  .container-dark { font-family: -apple-system, BlinkMacSystemFont, "Segoe UI", Roboto, Arial, sans-serif; line-height: 1.6; color: #d4d4d4; }
+  a { color: #569cd6; text-decoration: none; font-weight: 600; }
+  a:hover { text-decoration: underline; }
+  .card-dark { background-color: #252526; border-radius: 12px; padding: 24px; margin-bottom: 20px; box-shadow: 0 4px 12px rgba(0,0,0,0.3); border: 1px solid #3c3c3c; }
+  .card-dark h1 { font-size: 2.2em; color: #ffffff; text-align: center; margin-bottom: 10px; }
+  .card-dark .subtitle { text-align: center; font-size: 1.1em; color: #a0a0a0; }
+  .card-dark h2 { font-size: 1.5em; margin-top: 0; padding-bottom: 10px; border-bottom: 1px solid #3c3c3c; color: #c586c0; }
+  .styled-table { display: table; border: none; width: 100%; font-size: 0.95em; }
+  .styled-table thead th { background-color: #333333; color: #c586c0; text-align: left; padding: 12px 15px; }
+  .styled-table td { padding: 0; border-bottom: 1px solid #3c3c3c; }
+  .styled-table tbody tr { transition: background-color 0.1s ease; }
+  .styled-table tbody tr:hover { background-color: #3a3a3a; }
+  .styled-table tr:last-child td { border-bottom: none; }
+  .styled-table td a { display: block; padding: 12px 15px; }
+  .styled-table td a.fake-link { text-decoration:none; color:inherit; }
+  details { margin-top: 20px; border: 1px solid #3c3c3c; border-radius: 8px; overflow: hidden; }
+  summary { cursor: pointer; padding: 12px 18px; background-color: #6A5ACD; font-weight: 600; display: flex; align-items: center; gap: 10px; justify-content: space-between; list-style: none; }
+  summary::-webkit-details-marker { display: none; }
+  summary:hover { filter: brightness(1.1); }
+  summary::after { content: ''; display: inline-block; width: 8px; height: 8px; border-bottom: 2px solid white; border-right: 2px solid white; transform: rotate(45deg); transition: transform 0.3s ease; }
+  details[open] > summary::after { transform: rotate(225deg); }
+  .details-content { padding: 18px; }
+</style>
+
+<div class="container-dark">
+
+  <div class="card-dark">
+    <h1>AesCode-32B EXL3</h1>
+    <p class="subtitle">
+      EXL3 quants of <a href="https://huggingface.co/microsoft/AesCode-32B">microsoft/AesCode-32B</a>
+      using <a href="https://github.com/turboderp-org/exllamav3/">exllamav3</a> v1.6.1
+    </p>
+  </div>
+
+  <div class="card-dark">
+    <h2>KL Divergence vs VRAM</h2>
+    <img src="kld_plot.png" alt="KLD plot" style="width:100%; border-radius: 8px;" />
+    <p class="subtitle">Reference: BF16 original (PPL 6.843). Noise floor KLD 0.000151. Measured with exllamav3 <code>eval/qbench.py</code> on wikitext-2 (20 rows, 2048 ctx). VRAM = decoder + lm_head weights.</p>
+    <details>
+      <summary>KLD spread (median, IQR, noise floor)</summary>
+      <div class="details-content">
+        <img src="kld_spread_plot.png" alt="KLD spread" style="width:100%; border-radius: 8px;" />
+      </div>
+    </details>
+  </div>
+
+  <div class="card-dark">
+    <h2>Quants</h2>
+    <table class="styled-table">
+      <thead>
+        <tr><th>Branch</th><th>BPW</th><th>Head</th><th>VRAM (GiB)</th><th>KLD mean</th><th>KLD median</th><th>PPL</th><th>Type</th></tr>
+      </thead>
+      <tbody>
+        <tr>
+          <td><a href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/3.0bpw_H6">3.0bpw_H6</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/3.0bpw_H6">3.0</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/3.0bpw_H6">6</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/3.0bpw_H6">11.5</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/3.0bpw_H6">0.0360</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/3.0bpw_H6">0.01522</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/3.0bpw_H6">6.987</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/3.0bpw_H6">base</a></td>
+        </tr>
+        <tr>
+          <td><a href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/3.73bpw_H6">3.73bpw_H6</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/3.73bpw_H6">3.73</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/3.73bpw_H6">6</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/3.73bpw_H6">14.1</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/3.73bpw_H6">0.0157</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/3.73bpw_H6">0.00616</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/3.73bpw_H6">6.902</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/3.73bpw_H6">optimized</a></td>
+        </tr>
+        <tr>
+          <td><a href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/4.0bpw_H6">4.0bpw_H6</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/4.0bpw_H6">4.0</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/4.0bpw_H6">6</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/4.0bpw_H6">15.1</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/4.0bpw_H6">0.0093</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/4.0bpw_H6">0.00392</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/4.0bpw_H6">6.876</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/4.0bpw_H6">base</a></td>
+        </tr>
+        <tr>
+          <td><a href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/4.54bpw_H6">4.54bpw_H6</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/4.54bpw_H6">4.54</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/4.54bpw_H6">6</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/4.54bpw_H6">17.0</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/4.54bpw_H6">0.0065</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/4.54bpw_H6">0.00260</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/4.54bpw_H6">6.863</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/4.54bpw_H6">optimized</a></td>
+        </tr>
+        <tr>
+          <td><a href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/5.0bpw_H6">5.0bpw_H6</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/5.0bpw_H6">5.0</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/5.0bpw_H6">6</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/5.0bpw_H6">18.7</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/5.0bpw_H6">0.0027</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/5.0bpw_H6">0.00122</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/5.0bpw_H6">6.871</a></td>
+          <td><a class="fake-link" href="https://huggingface.co/WeReCooking/AesCode-32B-EXL3/tree/5.0bpw_H6">base</a></td>
+        </tr>
+      </tbody>
+    </table>
+    <p class="subtitle">Bases are direct converts. Optimized = KLD-guided tensor mix of the bases + attention recompiled at 5bpw; on this model they land on the base size/KLD curve, so treat them as in-between sizes and pick by VRAM. Vision tower 6bpw in every branch.</p>
+  </div>
+
+  <div class="card-dark">
+    <h2>Download and run</h2>
+    <details>
+      <summary>Download</summary>
+      <div class="details-content">
+        <pre><code>pip install -U "huggingface_hub"
+hf download WeReCooking/AesCode-32B-EXL3 --revision 4.0bpw_H6 --local-dir ./AesCode-32B-EXL3</code></pre>
+      </div>
+    </details>
+    <details>
+      <summary>Run with exllamav3</summary>
+      <div class="details-content">
+        <b>Text chat, official exllamav3 CLI</b> (model card sampling). Install exllamav3 first (<a href="https://github.com/turboderp-org/exllamav3/releases">prebuilt wheels</a>):
+        <pre><code>git clone https://github.com/turboderp-org/exllamav3 && cd exllamav3
+pip install pyperclip prompt_toolkit
+python examples/chat.py -m ../AesCode-32B-EXL3 -mode chatml -cs 24576 -maxr 12000 -temp 0.8 -topp 0.95 -topk 20 -minp 0</code></pre>
+        <b>One shot to file:</b>
+        <pre><code>python examples/chat.py -m ../AesCode-32B-EXL3 -mode chatml -cs 24576 -maxr 12000 -temp 0.8 -topp 0.95 -topk 20 -minp 0 -prompt "Create a portrait poster infographic about coffee origins as one complete HTML page." -save page.md</code></pre>
+        <b>With a reference image</b> (how AesCode is meant to be used, based on <code>examples/multimodal.py</code>):
+        <pre><code>from exllamav3 import Cache, ComboSampler, Config, Generator, Model, Tokenizer
+from PIL import Image
+
+config = Config.from_directory("AesCode-32B-EXL3")
+vision = Model.from_config(config, component="vision"); vision.load()
+model = Model.from_config(config); cache = Cache(model, max_num_tokens=24576); model.load()
+tok = Tokenizer.from_config(config)
+emb = vision.get_image_embeddings(tokenizer=tok, image=Image.open("reference.png"))
+prompt = ("&lt;|im_start|&gt;user\n" + emb.text_alias + "Create a landscape dashboard ... as one complete HTML page."
+          "&lt;|im_end|&gt;\n&lt;|im_start|&gt;assistant\n")
+gen = Generator(model=model, cache=cache, tokenizer=tok)
+print(gen.generate(prompt=prompt, embeddings=[emb], encode_special_tokens=True, completion_only=True,
+                   max_new_tokens=12000, stop_conditions=["&lt;|im_end|&gt;"],
+                   sampler=ComboSampler(temperature=0.8, top_p=0.95, top_k=20)))</code></pre>
+        <p>The repo's training system prompt and spec template (<a href="https://github.com/microsoft/AesCode/blob/main/inference/prompts.py">inference/prompts.py</a>) give the best results.</p>
+      </div>
+    </details>
+  </div>
+
+  <div class="card-dark">
+    <h2>Example output (4.0bpw_H6)</h2>
+    <img src="example_dashboard.jpg" alt="Reference image vs EXL3 output" style="width:100%; border-radius: 8px;" />
+    <p class="subtitle">Repo example <code>hospital_dashboard</code> with the training prompt template and reference image. Exact text 11/11, chart data 5/5, table 0/1, 1 element outside the 1280x720 canvas. 5097 tokens at 58.0 tok/s on an RTX 5090.</p>
+  </div>
+
+  <div class="card-dark">
+    <h2>Build Details</h2>
+    <details>
+      <summary>How these were made</summary>
+      <div class="details-content">
+        <p><b>Base quants:</b> <code>convert.py -b &lt;bpw&gt;</code> (3.0, 4.0, 5.0), head 6bpw, vision 6bpw. Calibration: 250 rows x 2048 (default) for 3.0, 128 rows x 2048 for 4.0/5.0 (32 GB RAM build box)</p>
+        <p><b>KLD measurement:</b> <code>util/measure.py -r &lt;bf16&gt; -i &lt;3.0&gt; &lt;4.0&gt; &lt;5.0&gt;</code> (<code>measurement.json</code> on main)</p>
+        <p><b>Optimized:</b> <code>util/optimize.py -i &lt;3.0&gt; &lt;4.0&gt; &lt;5.0&gt; -m measurement.json -b &lt;target&gt;</code> then <code>util/recompile.py -or override.yaml</code> with <code>*.self_attn.*</code> from 5.0bpw</p>
+        <p><b>Eval:</b> <code>eval/qbench.py</code>, BF16 streamed as reference</p>
+        <p>Docs: <a href="https://github.com/turboderp-org/exllamav3/blob/master/doc/convert.md">exllamav3 convert.md</a></p>
+      </div>
+    </details>
+  </div>
+
+  <div class="card-dark">
+    <h2>Files</h2>
+    <p><code>main</code> branch: <code>measurement.json</code>, KLD plots, example image, README</p>
+    <p>Each bpw branch: quantized shards (text + vision) + config + tokenizer + processor configs</p>
+  </div>
+
+</div>

measurement.json

@@ -0,0 +1,2388 @@
+{
+    "base": {
+        "dir": "D:\\main\\program\\AI\\tmp\\exl3_aescode\\quants\\3.0bpw_H6",
+        "bpw": 3.004301534673219
+    },
+    "alts": [
+        {
+            "dir": "D:\\main\\program\\AI\\tmp\\exl3_aescode\\quants\\4.0bpw_H6",
+            "bpw": 4.004301534673219
+        },
+        {
+            "dir": "D:\\main\\program\\AI\\tmp\\exl3_aescode\\quants\\5.0bpw_H6",
+            "bpw": 5.004301534673219
+        }
+    ],
+    "groups": [
+        {
+            "idx": 0,
+            "layers": [
+                "model.language_model.layers.0.self_attn.q_proj",
+                "model.language_model.layers.0.self_attn.k_proj",
+                "model.language_model.layers.0.self_attn.v_proj",
+                "model.language_model.layers.0.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.005342455487698294,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 0.002878085989505036,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 1,
+            "layers": [
+                "model.language_model.layers.0.mlp.gate_proj",
+                "model.language_model.layers.0.mlp.up_proj",
+                "model.language_model.layers.0.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.0010677056852728128,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.050666560931131255,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 2,
+            "layers": [
+                "model.language_model.layers.1.self_attn.q_proj",
+                "model.language_model.layers.1.self_attn.k_proj",
+                "model.language_model.layers.1.self_attn.v_proj",
+                "model.language_model.layers.1.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.1397632632870227,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.15157142621465028,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 3,
+            "layers": [
+                "model.language_model.layers.1.mlp.gate_proj",
+                "model.language_model.layers.1.mlp.up_proj",
+                "model.language_model.layers.1.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.0028491628821939174,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": 0.0034188396297395174,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 4,
+            "layers": [
+                "model.language_model.layers.2.self_attn.q_proj",
+                "model.language_model.layers.2.self_attn.k_proj",
+                "model.language_model.layers.2.self_attn.v_proj",
+                "model.language_model.layers.2.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.1418308971915394,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.011705041117966192,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 5,
+            "layers": [
+                "model.language_model.layers.2.mlp.gate_proj",
+                "model.language_model.layers.2.mlp.up_proj",
+                "model.language_model.layers.2.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0067629903089255206,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00024901125580073513,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 6,
+            "layers": [
+                "model.language_model.layers.3.self_attn.q_proj",
+                "model.language_model.layers.3.self_attn.k_proj",
+                "model.language_model.layers.3.self_attn.v_proj",
+                "model.language_model.layers.3.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0072793466039002,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.1536645043641329,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 7,
+            "layers": [
+                "model.language_model.layers.3.mlp.gate_proj",
+                "model.language_model.layers.3.mlp.up_proj",
+                "model.language_model.layers.3.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.006463171308860205,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": 0.008568887179717405,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 8,
+            "layers": [
+                "model.language_model.layers.4.self_attn.q_proj",
+                "model.language_model.layers.4.self_attn.k_proj",
+                "model.language_model.layers.4.self_attn.v_proj",
+                "model.language_model.layers.4.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.004537224862724537,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 0.0033987248782068324,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 9,
+            "layers": [
+                "model.language_model.layers.4.mlp.gate_proj",
+                "model.language_model.layers.4.mlp.up_proj",
+                "model.language_model.layers.4.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.00795677597634492,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.1482118721585721,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 10,
+            "layers": [
+                "model.language_model.layers.5.self_attn.q_proj",
+                "model.language_model.layers.5.self_attn.k_proj",
+                "model.language_model.layers.5.self_attn.v_proj",
+                "model.language_model.layers.5.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.009429611032828689,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.014525620499625802,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 11,
+            "layers": [
+                "model.language_model.layers.5.mlp.gate_proj",
+                "model.language_model.layers.5.mlp.up_proj",
+                "model.language_model.layers.5.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.002557756146416068,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": 0.004029345652088512,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 12,
+            "layers": [
+                "model.language_model.layers.6.self_attn.q_proj",
+                "model.language_model.layers.6.self_attn.k_proj",
+                "model.language_model.layers.6.self_attn.v_proj",
+                "model.language_model.layers.6.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.05095413839444518,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.060551986517384665,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 13,
+            "layers": [
+                "model.language_model.layers.6.mlp.gate_proj",
+                "model.language_model.layers.6.mlp.up_proj",
+                "model.language_model.layers.6.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.0034258951433002727,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": 0.004030669433996076,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 14,
+            "layers": [
+                "model.language_model.layers.7.self_attn.q_proj",
+                "model.language_model.layers.7.self_attn.k_proj",
+                "model.language_model.layers.7.self_attn.v_proj",
+                "model.language_model.layers.7.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.00012257243506608173,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -2.733203582466759e-05,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 15,
+            "layers": [
+                "model.language_model.layers.7.mlp.gate_proj",
+                "model.language_model.layers.7.mlp.up_proj",
+                "model.language_model.layers.7.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00040053990669550177,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00045577390119433403,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 16,
+            "layers": [
+                "model.language_model.layers.8.self_attn.q_proj",
+                "model.language_model.layers.8.self_attn.k_proj",
+                "model.language_model.layers.8.self_attn.v_proj",
+                "model.language_model.layers.8.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 3.334633074700277e-05,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.00012314599007368088,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 17,
+            "layers": [
+                "model.language_model.layers.8.mlp.gate_proj",
+                "model.language_model.layers.8.mlp.up_proj",
+                "model.language_model.layers.8.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00010292534716427326,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -7.305359467865702e-05,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 18,
+            "layers": [
+                "model.language_model.layers.9.self_attn.q_proj",
+                "model.language_model.layers.9.self_attn.k_proj",
+                "model.language_model.layers.9.self_attn.v_proj",
+                "model.language_model.layers.9.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 2.7341675013281552e-05,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 9.749447926876709e-05,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 19,
+            "layers": [
+                "model.language_model.layers.9.mlp.gate_proj",
+                "model.language_model.layers.9.mlp.up_proj",
+                "model.language_model.layers.9.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.0003252701833844074,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": 0.00020022029057142898,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 20,
+            "layers": [
+                "model.language_model.layers.10.self_attn.q_proj",
+                "model.language_model.layers.10.self_attn.k_proj",
+                "model.language_model.layers.10.self_attn.v_proj",
+                "model.language_model.layers.10.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.00011542919091878101,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 7.19995703548082e-05,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 21,
+            "layers": [
+                "model.language_model.layers.10.mlp.gate_proj",
+                "model.language_model.layers.10.mlp.up_proj",
+                "model.language_model.layers.10.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0002729880623519587,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00010036383755507305,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 22,
+            "layers": [
+                "model.language_model.layers.11.self_attn.q_proj",
+                "model.language_model.layers.11.self_attn.k_proj",
+                "model.language_model.layers.11.self_attn.v_proj",
+                "model.language_model.layers.11.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 4.468513652680639e-05,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 5.694916471837841e-05,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 23,
+            "layers": [
+                "model.language_model.layers.11.mlp.gate_proj",
+                "model.language_model.layers.11.mlp.up_proj",
+                "model.language_model.layers.11.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.000621724314987665,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0003705095499754174,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 24,
+            "layers": [
+                "model.language_model.layers.12.self_attn.q_proj",
+                "model.language_model.layers.12.self_attn.k_proj",
+                "model.language_model.layers.12.self_attn.v_proj",
+                "model.language_model.layers.12.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 2.4668266996735744e-05,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -9.445403702557087e-05,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 25,
+            "layers": [
+                "model.language_model.layers.12.mlp.gate_proj",
+                "model.language_model.layers.12.mlp.up_proj",
+                "model.language_model.layers.12.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0003460992127657103,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00034626852720975876,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 26,
+            "layers": [
+                "model.language_model.layers.13.self_attn.q_proj",
+                "model.language_model.layers.13.self_attn.k_proj",
+                "model.language_model.layers.13.self_attn.v_proj",
+                "model.language_model.layers.13.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.000152776110917352,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.00025113103911281187,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 27,
+            "layers": [
+                "model.language_model.layers.13.mlp.gate_proj",
+                "model.language_model.layers.13.mlp.up_proj",
+                "model.language_model.layers.13.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0007007658015936724,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00030329353176059004,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 28,
+            "layers": [
+                "model.language_model.layers.14.self_attn.q_proj",
+                "model.language_model.layers.14.self_attn.k_proj",
+                "model.language_model.layers.14.self_attn.v_proj",
+                "model.language_model.layers.14.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 4.437807947396677e-05,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -2.7742749080073015e-05,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 29,
+            "layers": [
+                "model.language_model.layers.14.mlp.gate_proj",
+                "model.language_model.layers.14.mlp.up_proj",
+                "model.language_model.layers.14.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00022678198292852558,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0002609751652926373,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 30,
+            "layers": [
+                "model.language_model.layers.15.self_attn.q_proj",
+                "model.language_model.layers.15.self_attn.k_proj",
+                "model.language_model.layers.15.self_attn.v_proj",
+                "model.language_model.layers.15.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00016537196934224285,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.00027730665169656277,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 31,
+            "layers": [
+                "model.language_model.layers.15.mlp.gate_proj",
+                "model.language_model.layers.15.mlp.up_proj",
+                "model.language_model.layers.15.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0002389888744801505,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00022584153339266777,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 32,
+            "layers": [
+                "model.language_model.layers.16.self_attn.q_proj",
+                "model.language_model.layers.16.self_attn.k_proj",
+                "model.language_model.layers.16.self_attn.v_proj",
+                "model.language_model.layers.16.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 8.467026054859161e-05,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 6.083534099160515e-05,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 33,
+            "layers": [
+                "model.language_model.layers.16.mlp.gate_proj",
+                "model.language_model.layers.16.mlp.up_proj",
+                "model.language_model.layers.16.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00029788915999234833,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00023291488178076025,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 34,
+            "layers": [
+                "model.language_model.layers.17.self_attn.q_proj",
+                "model.language_model.layers.17.self_attn.k_proj",
+                "model.language_model.layers.17.self_attn.v_proj",
+                "model.language_model.layers.17.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00013766633346679047,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.0002778722904622777,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 35,
+            "layers": [
+                "model.language_model.layers.17.mlp.gate_proj",
+                "model.language_model.layers.17.mlp.up_proj",
+                "model.language_model.layers.17.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 3.100791946053505e-05,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -1.545818522574738e-05,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 36,
+            "layers": [
+                "model.language_model.layers.18.self_attn.q_proj",
+                "model.language_model.layers.18.self_attn.k_proj",
+                "model.language_model.layers.18.self_attn.v_proj",
+                "model.language_model.layers.18.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 2.829506993293207e-05,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -4.0968367829935515e-05,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 37,
+            "layers": [
+                "model.language_model.layers.18.mlp.gate_proj",
+                "model.language_model.layers.18.mlp.up_proj",
+                "model.language_model.layers.18.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -8.97957943379879e-06,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00019588181748986244,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 38,
+            "layers": [
+                "model.language_model.layers.19.self_attn.q_proj",
+                "model.language_model.layers.19.self_attn.k_proj",
+                "model.language_model.layers.19.self_attn.v_proj",
+                "model.language_model.layers.19.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.0001437588594853656,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 0.0001899583265185245,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 39,
+            "layers": [
+                "model.language_model.layers.19.mlp.gate_proj",
+                "model.language_model.layers.19.mlp.up_proj",
+                "model.language_model.layers.19.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0001980282366275954,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00021657398901880898,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 40,
+            "layers": [
+                "model.language_model.layers.20.self_attn.q_proj",
+                "model.language_model.layers.20.self_attn.k_proj",
+                "model.language_model.layers.20.self_attn.v_proj",
+                "model.language_model.layers.20.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00013915137387812693,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -7.083872333168983e-05,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 41,
+            "layers": [
+                "model.language_model.layers.20.mlp.gate_proj",
+                "model.language_model.layers.20.mlp.up_proj",
+                "model.language_model.layers.20.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -1.2000277638452017e-05,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -6.155669689178467e-05,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 42,
+            "layers": [
+                "model.language_model.layers.21.self_attn.q_proj",
+                "model.language_model.layers.21.self_attn.k_proj",
+                "model.language_model.layers.21.self_attn.v_proj",
+                "model.language_model.layers.21.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.00012218807823954903,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 0.00018510879017410997,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 43,
+            "layers": [
+                "model.language_model.layers.21.mlp.gate_proj",
+                "model.language_model.layers.21.mlp.up_proj",
+                "model.language_model.layers.21.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00025624348782005146,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0002975346054882033,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 44,
+            "layers": [
+                "model.language_model.layers.22.self_attn.q_proj",
+                "model.language_model.layers.22.self_attn.k_proj",
+                "model.language_model.layers.22.self_attn.v_proj",
+                "model.language_model.layers.22.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00014645280316472054,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.0002074604388326573,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 45,
+            "layers": [
+                "model.language_model.layers.22.mlp.gate_proj",
+                "model.language_model.layers.22.mlp.up_proj",
+                "model.language_model.layers.22.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -2.5052111596035598e-06,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": 8.436250500379328e-05,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 46,
+            "layers": [
+                "model.language_model.layers.23.self_attn.q_proj",
+                "model.language_model.layers.23.self_attn.k_proj",
+                "model.language_model.layers.23.self_attn.v_proj",
+                "model.language_model.layers.23.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -7.000300101936419e-05,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -5.9204548597341367e-05,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 47,
+            "layers": [
+                "model.language_model.layers.23.mlp.gate_proj",
+                "model.language_model.layers.23.mlp.up_proj",
+                "model.language_model.layers.23.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0003282364923506975,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0002834261395037174,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 48,
+            "layers": [
+                "model.language_model.layers.24.self_attn.q_proj",
+                "model.language_model.layers.24.self_attn.k_proj",
+                "model.language_model.layers.24.self_attn.v_proj",
+                "model.language_model.layers.24.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00010582348331811264,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 4.957020282744806e-05,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 49,
+            "layers": [
+                "model.language_model.layers.24.mlp.gate_proj",
+                "model.language_model.layers.24.mlp.up_proj",
+                "model.language_model.layers.24.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -4.923101514578421e-05,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -7.321541197599646e-05,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 50,
+            "layers": [
+                "model.language_model.layers.25.self_attn.q_proj",
+                "model.language_model.layers.25.self_attn.k_proj",
+                "model.language_model.layers.25.self_attn.v_proj",
+                "model.language_model.layers.25.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 1.7680786550039462e-05,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.00017909421585501906,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 51,
+            "layers": [
+                "model.language_model.layers.25.mlp.gate_proj",
+                "model.language_model.layers.25.mlp.up_proj",
+                "model.language_model.layers.25.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.0002028619870543258,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": 0.0002154889516532421,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 52,
+            "layers": [
+                "model.language_model.layers.26.self_attn.q_proj",
+                "model.language_model.layers.26.self_attn.k_proj",
+                "model.language_model.layers.26.self_attn.v_proj",
+                "model.language_model.layers.26.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -2.6849750429408514e-05,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.00013140486553311903,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 53,
+            "layers": [
+                "model.language_model.layers.26.mlp.gate_proj",
+                "model.language_model.layers.26.mlp.up_proj",
+                "model.language_model.layers.26.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -9.225751273334026e-05,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00036976165138186134,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 54,
+            "layers": [
+                "model.language_model.layers.27.self_attn.q_proj",
+                "model.language_model.layers.27.self_attn.k_proj",
+                "model.language_model.layers.27.self_attn.v_proj",
+                "model.language_model.layers.27.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.00020883963443338316,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 8.990662172436714e-05,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 55,
+            "layers": [
+                "model.language_model.layers.27.mlp.gate_proj",
+                "model.language_model.layers.27.mlp.up_proj",
+                "model.language_model.layers.27.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 2.523777075111311e-05,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00022528180852532387,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 56,
+            "layers": [
+                "model.language_model.layers.28.self_attn.q_proj",
+                "model.language_model.layers.28.self_attn.k_proj",
+                "model.language_model.layers.28.self_attn.v_proj",
+                "model.language_model.layers.28.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -4.28589060902651e-05,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 0.00010976642370222334,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 57,
+            "layers": [
+                "model.language_model.layers.28.mlp.gate_proj",
+                "model.language_model.layers.28.mlp.up_proj",
+                "model.language_model.layers.28.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00026168590411546067,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0003445466049015744,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 58,
+            "layers": [
+                "model.language_model.layers.29.self_attn.q_proj",
+                "model.language_model.layers.29.self_attn.k_proj",
+                "model.language_model.layers.29.self_attn.v_proj",
+                "model.language_model.layers.29.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 6.120707839726847e-05,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 0.00010455059818922918,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 59,
+            "layers": [
+                "model.language_model.layers.29.mlp.gate_proj",
+                "model.language_model.layers.29.mlp.up_proj",
+                "model.language_model.layers.29.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.000123786833137296,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0003314546775072813,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 60,
+            "layers": [
+                "model.language_model.layers.30.self_attn.q_proj",
+                "model.language_model.layers.30.self_attn.k_proj",
+                "model.language_model.layers.30.self_attn.v_proj",
+                "model.language_model.layers.30.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.00029394258745013557,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 0.00022767381742594406,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 61,
+            "layers": [
+                "model.language_model.layers.30.mlp.gate_proj",
+                "model.language_model.layers.30.mlp.up_proj",
+                "model.language_model.layers.30.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00012490730732681032,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0001571818720549345,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 62,
+            "layers": [
+                "model.language_model.layers.31.self_attn.q_proj",
+                "model.language_model.layers.31.self_attn.k_proj",
+                "model.language_model.layers.31.self_attn.v_proj",
+                "model.language_model.layers.31.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0004900929052382708,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.0005018283613026364,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 63,
+            "layers": [
+                "model.language_model.layers.31.mlp.gate_proj",
+                "model.language_model.layers.31.mlp.up_proj",
+                "model.language_model.layers.31.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.00022143623791634481,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": 0.00010823202319443226,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 64,
+            "layers": [
+                "model.language_model.layers.32.self_attn.q_proj",
+                "model.language_model.layers.32.self_attn.k_proj",
+                "model.language_model.layers.32.self_attn.v_proj",
+                "model.language_model.layers.32.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0003077854868024643,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.00022793915122748531,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 65,
+            "layers": [
+                "model.language_model.layers.32.mlp.gate_proj",
+                "model.language_model.layers.32.mlp.up_proj",
+                "model.language_model.layers.32.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -1.8477952107798234e-05,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00013991552405059893,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 66,
+            "layers": [
+                "model.language_model.layers.33.self_attn.q_proj",
+                "model.language_model.layers.33.self_attn.k_proj",
+                "model.language_model.layers.33.self_attn.v_proj",
+                "model.language_model.layers.33.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.0001139423344284074,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 0.00010652639903127592,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 67,
+            "layers": [
+                "model.language_model.layers.33.mlp.gate_proj",
+                "model.language_model.layers.33.mlp.up_proj",
+                "model.language_model.layers.33.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00015837834216655966,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00017344346269965172,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 68,
+            "layers": [
+                "model.language_model.layers.34.self_attn.q_proj",
+                "model.language_model.layers.34.self_attn.k_proj",
+                "model.language_model.layers.34.self_attn.v_proj",
+                "model.language_model.layers.34.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -2.0092679187677653e-05,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -4.768059588969309e-05,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 69,
+            "layers": [
+                "model.language_model.layers.34.mlp.gate_proj",
+                "model.language_model.layers.34.mlp.up_proj",
+                "model.language_model.layers.34.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00011453106999398388,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00014291619881989792,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 70,
+            "layers": [
+                "model.language_model.layers.35.self_attn.q_proj",
+                "model.language_model.layers.35.self_attn.k_proj",
+                "model.language_model.layers.35.self_attn.v_proj",
+                "model.language_model.layers.35.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.00015730569139121853,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 0.0002371510956436229,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 71,
+            "layers": [
+                "model.language_model.layers.35.mlp.gate_proj",
+                "model.language_model.layers.35.mlp.up_proj",
+                "model.language_model.layers.35.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00023716585710645277,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0004655693657696358,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 72,
+            "layers": [
+                "model.language_model.layers.36.self_attn.q_proj",
+                "model.language_model.layers.36.self_attn.k_proj",
+                "model.language_model.layers.36.self_attn.v_proj",
+                "model.language_model.layers.36.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00032587922178209383,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.0005042652599513697,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 73,
+            "layers": [
+                "model.language_model.layers.36.mlp.gate_proj",
+                "model.language_model.layers.36.mlp.up_proj",
+                "model.language_model.layers.36.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0006116618402302321,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0006361586041748746,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 74,
+            "layers": [
+                "model.language_model.layers.37.self_attn.q_proj",
+                "model.language_model.layers.37.self_attn.k_proj",
+                "model.language_model.layers.37.self_attn.v_proj",
+                "model.language_model.layers.37.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 5.171378143131733e-05,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -1.6777683049445935e-05,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 75,
+            "layers": [
+                "model.language_model.layers.37.mlp.gate_proj",
+                "model.language_model.layers.37.mlp.up_proj",
+                "model.language_model.layers.37.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0006116549018770623,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0006575850769877656,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 76,
+            "layers": [
+                "model.language_model.layers.38.self_attn.q_proj",
+                "model.language_model.layers.38.self_attn.k_proj",
+                "model.language_model.layers.38.self_attn.v_proj",
+                "model.language_model.layers.38.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00017759450711310465,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.0003292498644441588,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 77,
+            "layers": [
+                "model.language_model.layers.38.mlp.gate_proj",
+                "model.language_model.layers.38.mlp.up_proj",
+                "model.language_model.layers.38.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 9.779524989425581e-05,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": 0.00022586658596990272,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 78,
+            "layers": [
+                "model.language_model.layers.39.self_attn.q_proj",
+                "model.language_model.layers.39.self_attn.k_proj",
+                "model.language_model.layers.39.self_attn.v_proj",
+                "model.language_model.layers.39.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00019017886370420456,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.00014174073003234144,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 79,
+            "layers": [
+                "model.language_model.layers.39.mlp.gate_proj",
+                "model.language_model.layers.39.mlp.up_proj",
+                "model.language_model.layers.39.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.00011877082288264118,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00013138423673808575,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 80,
+            "layers": [
+                "model.language_model.layers.40.self_attn.q_proj",
+                "model.language_model.layers.40.self_attn.k_proj",
+                "model.language_model.layers.40.self_attn.v_proj",
+                "model.language_model.layers.40.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.00014259032905100666,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 0.0003622303716838138,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 81,
+            "layers": [
+                "model.language_model.layers.40.mlp.gate_proj",
+                "model.language_model.layers.40.mlp.up_proj",
+                "model.language_model.layers.40.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0004363835323601961,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0004892456810921542,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 82,
+            "layers": [
+                "model.language_model.layers.41.self_attn.q_proj",
+                "model.language_model.layers.41.self_attn.k_proj",
+                "model.language_model.layers.41.self_attn.v_proj",
+                "model.language_model.layers.41.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00037515051662923016,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.00036390214227141615,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 83,
+            "layers": [
+                "model.language_model.layers.41.mlp.gate_proj",
+                "model.language_model.layers.41.mlp.up_proj",
+                "model.language_model.layers.41.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -2.6784790679829085e-05,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0002820216584950852,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 84,
+            "layers": [
+                "model.language_model.layers.42.self_attn.q_proj",
+                "model.language_model.layers.42.self_attn.k_proj",
+                "model.language_model.layers.42.self_attn.v_proj",
+                "model.language_model.layers.42.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0002433180809021107,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.00022505610249937136,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 85,
+            "layers": [
+                "model.language_model.layers.42.mlp.gate_proj",
+                "model.language_model.layers.42.mlp.up_proj",
+                "model.language_model.layers.42.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00030364627018572565,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0002478116191923785,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 86,
+            "layers": [
+                "model.language_model.layers.43.self_attn.q_proj",
+                "model.language_model.layers.43.self_attn.k_proj",
+                "model.language_model.layers.43.self_attn.v_proj",
+                "model.language_model.layers.43.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.00013466081582008682,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 7.684668526047878e-06,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 87,
+            "layers": [
+                "model.language_model.layers.43.mlp.gate_proj",
+                "model.language_model.layers.43.mlp.up_proj",
+                "model.language_model.layers.43.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 2.2285990416981427e-05,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": 9.187171235658376e-06,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 88,
+            "layers": [
+                "model.language_model.layers.44.self_attn.q_proj",
+                "model.language_model.layers.44.self_attn.k_proj",
+                "model.language_model.layers.44.self_attn.v_proj",
+                "model.language_model.layers.44.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.0001239406876265836,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 0.00019352706149219912,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 89,
+            "layers": [
+                "model.language_model.layers.44.mlp.gate_proj",
+                "model.language_model.layers.44.mlp.up_proj",
+                "model.language_model.layers.44.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00016605677083136872,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00035673477686942734,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 90,
+            "layers": [
+                "model.language_model.layers.45.self_attn.q_proj",
+                "model.language_model.layers.45.self_attn.k_proj",
+                "model.language_model.layers.45.self_attn.v_proj",
+                "model.language_model.layers.45.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.00018179877661167465,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -9.876396507041418e-06,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 91,
+            "layers": [
+                "model.language_model.layers.45.mlp.gate_proj",
+                "model.language_model.layers.45.mlp.up_proj",
+                "model.language_model.layers.45.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0004525623749941643,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00030217254534364857,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 92,
+            "layers": [
+                "model.language_model.layers.46.self_attn.q_proj",
+                "model.language_model.layers.46.self_attn.k_proj",
+                "model.language_model.layers.46.self_attn.v_proj",
+                "model.language_model.layers.46.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00010313107632101137,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -1.639225520194687e-05,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 93,
+            "layers": [
+                "model.language_model.layers.46.mlp.gate_proj",
+                "model.language_model.layers.46.mlp.up_proj",
+                "model.language_model.layers.46.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00023827492259442806,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0001679945271462202,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 94,
+            "layers": [
+                "model.language_model.layers.47.self_attn.q_proj",
+                "model.language_model.layers.47.self_attn.k_proj",
+                "model.language_model.layers.47.self_attn.v_proj",
+                "model.language_model.layers.47.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.00047768964432179373,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 0.0004286430310457834,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 95,
+            "layers": [
+                "model.language_model.layers.47.mlp.gate_proj",
+                "model.language_model.layers.47.mlp.up_proj",
+                "model.language_model.layers.47.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0003748951479792817,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00041852202266456917,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 96,
+            "layers": [
+                "model.language_model.layers.48.self_attn.q_proj",
+                "model.language_model.layers.48.self_attn.k_proj",
+                "model.language_model.layers.48.self_attn.v_proj",
+                "model.language_model.layers.48.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00015185866504907608,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 2.9604835435736998e-05,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 97,
+            "layers": [
+                "model.language_model.layers.48.mlp.gate_proj",
+                "model.language_model.layers.48.mlp.up_proj",
+                "model.language_model.layers.48.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.000497997272759676,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0005106110125780161,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 98,
+            "layers": [
+                "model.language_model.layers.49.self_attn.q_proj",
+                "model.language_model.layers.49.self_attn.k_proj",
+                "model.language_model.layers.49.self_attn.v_proj",
+                "model.language_model.layers.49.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00013194279745221138,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.00026086703874173,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 99,
+            "layers": [
+                "model.language_model.layers.49.mlp.gate_proj",
+                "model.language_model.layers.49.mlp.up_proj",
+                "model.language_model.layers.49.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.00028887651860712293,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": 8.507091552018009e-05,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 100,
+            "layers": [
+                "model.language_model.layers.50.self_attn.q_proj",
+                "model.language_model.layers.50.self_attn.k_proj",
+                "model.language_model.layers.50.self_attn.v_proj",
+                "model.language_model.layers.50.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.00041282796300945446,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 0.00034878966398535494,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 101,
+            "layers": [
+                "model.language_model.layers.50.mlp.gate_proj",
+                "model.language_model.layers.50.mlp.up_proj",
+                "model.language_model.layers.50.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0003386356867849938,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0005072795320302248,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 102,
+            "layers": [
+                "model.language_model.layers.51.self_attn.q_proj",
+                "model.language_model.layers.51.self_attn.k_proj",
+                "model.language_model.layers.51.self_attn.v_proj",
+                "model.language_model.layers.51.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 8.664890192447428e-05,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 0.000201887823641278,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 103,
+            "layers": [
+                "model.language_model.layers.51.mlp.gate_proj",
+                "model.language_model.layers.51.mlp.up_proj",
+                "model.language_model.layers.51.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0009231174364686123,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0008698628749698456,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 104,
+            "layers": [
+                "model.language_model.layers.52.self_attn.q_proj",
+                "model.language_model.layers.52.self_attn.k_proj",
+                "model.language_model.layers.52.self_attn.v_proj",
+                "model.language_model.layers.52.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.0006682481616735458,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 0.00046845399774608776,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 105,
+            "layers": [
+                "model.language_model.layers.52.mlp.gate_proj",
+                "model.language_model.layers.52.mlp.up_proj",
+                "model.language_model.layers.52.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0005143266171216965,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00028531067073345184,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 106,
+            "layers": [
+                "model.language_model.layers.53.self_attn.q_proj",
+                "model.language_model.layers.53.self_attn.k_proj",
+                "model.language_model.layers.53.self_attn.v_proj",
+                "model.language_model.layers.53.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0011058779433369692,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.0008262845687568299,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 107,
+            "layers": [
+                "model.language_model.layers.53.mlp.gate_proj",
+                "model.language_model.layers.53.mlp.up_proj",
+                "model.language_model.layers.53.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0005857495125383139,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0007907640654593873,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 108,
+            "layers": [
+                "model.language_model.layers.54.self_attn.q_proj",
+                "model.language_model.layers.54.self_attn.k_proj",
+                "model.language_model.layers.54.self_attn.v_proj",
+                "model.language_model.layers.54.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0003247374668717551,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.00047106938436627943,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 109,
+            "layers": [
+                "model.language_model.layers.54.mlp.gate_proj",
+                "model.language_model.layers.54.mlp.up_proj",
+                "model.language_model.layers.54.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0006814375985413901,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0006852156948298216,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 110,
+            "layers": [
+                "model.language_model.layers.55.self_attn.q_proj",
+                "model.language_model.layers.55.self_attn.k_proj",
+                "model.language_model.layers.55.self_attn.v_proj",
+                "model.language_model.layers.55.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0008712090086192004,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.000794619973748939,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 111,
+            "layers": [
+                "model.language_model.layers.55.mlp.gate_proj",
+                "model.language_model.layers.55.mlp.up_proj",
+                "model.language_model.layers.55.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0004104899242520499,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0003673459868878126,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 112,
+            "layers": [
+                "model.language_model.layers.56.self_attn.q_proj",
+                "model.language_model.layers.56.self_attn.k_proj",
+                "model.language_model.layers.56.self_attn.v_proj",
+                "model.language_model.layers.56.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -8.137007243932959e-05,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.0005230564158409945,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 113,
+            "layers": [
+                "model.language_model.layers.56.mlp.gate_proj",
+                "model.language_model.layers.56.mlp.up_proj",
+                "model.language_model.layers.56.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0007926002144813538,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0004942396190017462,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 114,
+            "layers": [
+                "model.language_model.layers.57.self_attn.q_proj",
+                "model.language_model.layers.57.self_attn.k_proj",
+                "model.language_model.layers.57.self_attn.v_proj",
+                "model.language_model.layers.57.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -4.884987138212282e-05,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.00027763238176703453,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 115,
+            "layers": [
+                "model.language_model.layers.57.mlp.gate_proj",
+                "model.language_model.layers.57.mlp.up_proj",
+                "model.language_model.layers.57.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0007438508328050375,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0007115922868251856,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 116,
+            "layers": [
+                "model.language_model.layers.58.self_attn.q_proj",
+                "model.language_model.layers.58.self_attn.k_proj",
+                "model.language_model.layers.58.self_attn.v_proj",
+                "model.language_model.layers.58.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00016308482736349106,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.0003048637881875038,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 117,
+            "layers": [
+                "model.language_model.layers.58.mlp.gate_proj",
+                "model.language_model.layers.58.mlp.up_proj",
+                "model.language_model.layers.58.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0008469442371279168,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.000874680187553184,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 118,
+            "layers": [
+                "model.language_model.layers.59.self_attn.q_proj",
+                "model.language_model.layers.59.self_attn.k_proj",
+                "model.language_model.layers.59.self_attn.v_proj",
+                "model.language_model.layers.59.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 2.2697122767562083e-05,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.0004360306076705567,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 119,
+            "layers": [
+                "model.language_model.layers.59.mlp.gate_proj",
+                "model.language_model.layers.59.mlp.up_proj",
+                "model.language_model.layers.59.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0005720941349864062,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0006580702494830004,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 120,
+            "layers": [
+                "model.language_model.layers.60.self_attn.q_proj",
+                "model.language_model.layers.60.self_attn.k_proj",
+                "model.language_model.layers.60.self_attn.v_proj",
+                "model.language_model.layers.60.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.001185041572898643,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.0012297863606363646,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 121,
+            "layers": [
+                "model.language_model.layers.60.mlp.gate_proj",
+                "model.language_model.layers.60.mlp.up_proj",
+                "model.language_model.layers.60.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00043576946482062895,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.00037752632051707424,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 122,
+            "layers": [
+                "model.language_model.layers.61.self_attn.q_proj",
+                "model.language_model.layers.61.self_attn.k_proj",
+                "model.language_model.layers.61.self_attn.v_proj",
+                "model.language_model.layers.61.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.00023652003146706901,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.00035312902182341177,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 123,
+            "layers": [
+                "model.language_model.layers.61.mlp.gate_proj",
+                "model.language_model.layers.61.mlp.up_proj",
+                "model.language_model.layers.61.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.00031162113882601816,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.0005344925448298454,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 124,
+            "layers": [
+                "model.language_model.layers.62.self_attn.q_proj",
+                "model.language_model.layers.62.self_attn.k_proj",
+                "model.language_model.layers.62.self_attn.v_proj",
+                "model.language_model.layers.62.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0011726192198693863,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": -0.0011081209406256787,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 125,
+            "layers": [
+                "model.language_model.layers.62.mlp.gate_proj",
+                "model.language_model.layers.62.mlp.up_proj",
+                "model.language_model.layers.62.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0033292342443019263,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.003356328653171675,
+                    "dbits": 786432000
+                }
+            ]
+        },
+        {
+            "idx": 126,
+            "layers": [
+                "model.language_model.layers.63.self_attn.q_proj",
+                "model.language_model.layers.63.self_attn.k_proj",
+                "model.language_model.layers.63.self_attn.v_proj",
+                "model.language_model.layers.63.self_attn.o_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": 0.00011754971928892854,
+                    "dbits": 94371840
+                },
+                {
+                    "dkld": 0.00028725313022731624,
+                    "dbits": 188743680
+                }
+            ]
+        },
+        {
+            "idx": 127,
+            "layers": [
+                "model.language_model.layers.63.mlp.gate_proj",
+                "model.language_model.layers.63.mlp.up_proj",
+                "model.language_model.layers.63.mlp.down_proj"
+            ],
+            "candidates": [
+                {
+                    "dkld": -0.0020378781482577324,
+                    "dbits": 393216000
+                },
+                {
+                    "dkld": -0.002213719766587019,
+                    "dbits": 786432000
+                }
+            ]
+        }
+    ],
+    "base_kld": 0.19816926405765117,
+    "arch_string": "Qwen3VLForConditionalGeneration"
+}