{
  "average_bits": 3.4985438747254847,
  "embed_bits": 8,
  "budget": {
    "low_avg": 3.4,
    "high_avg": 3.5
  },
  "sum_kl": 2.977956267718296,
  "dp_sum_kl": 2.540285839100619,
  "joint_refine": {
    "calibration_n": 20,
    "calibration_ids": [
      "mnli/validation_matched/8418/relation",
      "banking77/test/1454/intent",
      "boolq/validation/1209/answer",
      "composition/v3-composition-dev/negation/1/relevant/a/decision",
      "amazon/test/605/stars",
      "dbpedia14/test/44634/category",
      "contrastive/age_eligibility/v3-control-dev-age_eligibility-0013/b/decision",
      "yelp/test/10368/rating",
      "agnews/test/4286/topic",
      "sst5/test/445/sentiment",
      "trec/test/455/answer_type",
      "imdb/test/3003/positive",
      "readme-urgency",
      "mnli/validation_matched/3117/relation",
      "dbpedia14/test/48873/category",
      "composition/v3-composition-dev/conditional/2/irrelevant/b/decision",
      "mnli/validation_matched/3000/relation",
      "boolq/validation/376/answer",
      "trec/test/280/none_absent/answer_type",
      "mnli/validation_matched/8418/permuted/relation"
    ],
    "joint_kl_before": 0.359920485658611,
    "joint_kl_after": 0.15394268493215005,
    "swaps": 12
  },
  "assignment": [
    {
      "path": "language_model.model.layers.0.linear_attn.in_proj_qkv",
      "layer": 0,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.036167649794940304
    },
    {
      "path": "language_model.model.layers.0.linear_attn.in_proj_z",
      "layer": 0,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.024077155345270138
    },
    {
      "path": "language_model.model.layers.0.linear_attn.in_proj_a",
      "layer": 0,
      "nparams": 32768,
      "bits": 3,
      "kind": "3",
      "kl": 0.004719737666264188
    },
    {
      "path": "language_model.model.layers.0.linear_attn.in_proj_b",
      "layer": 0,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.009882406261872978
    },
    {
      "path": "language_model.model.layers.0.linear_attn.out_proj",
      "layer": 0,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.19649393687983674
    },
    {
      "path": "language_model.model.layers.0.mlp.gate_proj",
      "layer": 0,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.014610158877154451
    },
    {
      "path": "language_model.model.layers.0.mlp.up_proj",
      "layer": 0,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.019142927850698384
    },
    {
      "path": "language_model.model.layers.0.mlp.down_proj",
      "layer": 0,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.018993703064068047
    },
    {
      "path": "language_model.model.layers.1.linear_attn.in_proj_qkv",
      "layer": 1,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.02159494921496259
    },
    {
      "path": "language_model.model.layers.1.linear_attn.in_proj_z",
      "layer": 1,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.009249335523625427
    },
    {
      "path": "language_model.model.layers.1.linear_attn.in_proj_a",
      "layer": 1,
      "nparams": 32768,
      "bits": 3,
      "kind": "3",
      "kl": 0.009942285073314044
    },
    {
      "path": "language_model.model.layers.1.linear_attn.in_proj_b",
      "layer": 1,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.011165700598795746
    },
    {
      "path": "language_model.model.layers.1.linear_attn.out_proj",
      "layer": 1,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.02533359810132539
    },
    {
      "path": "language_model.model.layers.1.mlp.gate_proj",
      "layer": 1,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.020001911429270484
    },
    {
      "path": "language_model.model.layers.1.mlp.up_proj",
      "layer": 1,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.018694505846325777
    },
    {
      "path": "language_model.model.layers.1.mlp.down_proj",
      "layer": 1,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.045032779148968485
    },
    {
      "path": "language_model.model.layers.2.linear_attn.in_proj_qkv",
      "layer": 2,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.028996766755013885
    },
    {
      "path": "language_model.model.layers.2.linear_attn.in_proj_z",
      "layer": 2,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.018902605744829708
    },
    {
      "path": "language_model.model.layers.2.linear_attn.in_proj_a",
      "layer": 2,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.006896526199611989
    },
    {
      "path": "language_model.model.layers.2.linear_attn.in_proj_b",
      "layer": 2,
      "nparams": 32768,
      "bits": 3,
      "kind": "3",
      "kl": 0.009745471732003758
    },
    {
      "path": "language_model.model.layers.2.linear_attn.out_proj",
      "layer": 2,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.037437193703247655
    },
    {
      "path": "language_model.model.layers.2.mlp.gate_proj",
      "layer": 2,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.020557114209522904
    },
    {
      "path": "language_model.model.layers.2.mlp.up_proj",
      "layer": 2,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.031465961257551245
    },
    {
      "path": "language_model.model.layers.2.mlp.down_proj",
      "layer": 2,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.032831266240288565
    },
    {
      "path": "language_model.model.layers.3.self_attn.q_proj",
      "layer": 3,
      "nparams": 8388608,
      "bits": 3,
      "kind": "3",
      "kl": 0.060484579321996676
    },
    {
      "path": "language_model.model.layers.3.self_attn.k_proj",
      "layer": 3,
      "nparams": 1048576,
      "bits": 4,
      "kind": "4",
      "kl": 0.01184541686985306
    },
    {
      "path": "language_model.model.layers.3.self_attn.v_proj",
      "layer": 3,
      "nparams": 1048576,
      "bits": 4,
      "kind": "4",
      "kl": 0.04672758597267979
    },
    {
      "path": "language_model.model.layers.3.self_attn.o_proj",
      "layer": 3,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.03858699061895039
    },
    {
      "path": "language_model.model.layers.3.mlp.gate_proj",
      "layer": 3,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.015935746183335358
    },
    {
      "path": "language_model.model.layers.3.mlp.up_proj",
      "layer": 3,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.04384401309329482
    },
    {
      "path": "language_model.model.layers.3.mlp.down_proj",
      "layer": 3,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.02399046514312047
    },
    {
      "path": "language_model.model.layers.4.linear_attn.in_proj_qkv",
      "layer": 4,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.03380978350370204
    },
    {
      "path": "language_model.model.layers.4.linear_attn.in_proj_z",
      "layer": 4,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.015917916860611572
    },
    {
      "path": "language_model.model.layers.4.linear_attn.in_proj_a",
      "layer": 4,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.0071051056101440815
    },
    {
      "path": "language_model.model.layers.4.linear_attn.in_proj_b",
      "layer": 4,
      "nparams": 32768,
      "bits": 3,
      "kind": "3",
      "kl": 0.011950433791683937
    },
    {
      "path": "language_model.model.layers.4.linear_attn.out_proj",
      "layer": 4,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.029419899497863965
    },
    {
      "path": "language_model.model.layers.4.mlp.gate_proj",
      "layer": 4,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.02037560206293295
    },
    {
      "path": "language_model.model.layers.4.mlp.up_proj",
      "layer": 4,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.026954146641484288
    },
    {
      "path": "language_model.model.layers.4.mlp.down_proj",
      "layer": 4,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.031966774444906774
    },
    {
      "path": "language_model.model.layers.5.linear_attn.in_proj_qkv",
      "layer": 5,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.024604880627422072
    },
    {
      "path": "language_model.model.layers.5.linear_attn.in_proj_z",
      "layer": 5,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.01946826237510793
    },
    {
      "path": "language_model.model.layers.5.linear_attn.in_proj_a",
      "layer": 5,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.006358107711251463
    },
    {
      "path": "language_model.model.layers.5.linear_attn.in_proj_b",
      "layer": 5,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.007007401410325599
    },
    {
      "path": "language_model.model.layers.5.linear_attn.out_proj",
      "layer": 5,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.04259233745882887
    },
    {
      "path": "language_model.model.layers.5.mlp.gate_proj",
      "layer": 5,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.021353867373001368
    },
    {
      "path": "language_model.model.layers.5.mlp.up_proj",
      "layer": 5,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.03279443582076645
    },
    {
      "path": "language_model.model.layers.5.mlp.down_proj",
      "layer": 5,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.030314831495057707
    },
    {
      "path": "language_model.model.layers.6.linear_attn.in_proj_qkv",
      "layer": 6,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.03271812705592554
    },
    {
      "path": "language_model.model.layers.6.linear_attn.in_proj_z",
      "layer": 6,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.01999809052483925
    },
    {
      "path": "language_model.model.layers.6.linear_attn.in_proj_a",
      "layer": 6,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.006959841064670462
    },
    {
      "path": "language_model.model.layers.6.linear_attn.in_proj_b",
      "layer": 6,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.008939020186841463
    },
    {
      "path": "language_model.model.layers.6.linear_attn.out_proj",
      "layer": 6,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.02744619635510964
    },
    {
      "path": "language_model.model.layers.6.mlp.gate_proj",
      "layer": 6,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.025605077184504846
    },
    {
      "path": "language_model.model.layers.6.mlp.up_proj",
      "layer": 6,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.04094105855935139
    },
    {
      "path": "language_model.model.layers.6.mlp.down_proj",
      "layer": 6,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.03757126084623266
    },
    {
      "path": "language_model.model.layers.7.self_attn.q_proj",
      "layer": 7,
      "nparams": 8388608,
      "bits": 3,
      "kind": "3",
      "kl": 0.12697585014100587
    },
    {
      "path": "language_model.model.layers.7.self_attn.k_proj",
      "layer": 7,
      "nparams": 1048576,
      "bits": 4,
      "kind": "4",
      "kl": 0.011872071865418204
    },
    {
      "path": "language_model.model.layers.7.self_attn.v_proj",
      "layer": 7,
      "nparams": 1048576,
      "bits": 4,
      "kind": "4",
      "kl": 0.03341289663710259
    },
    {
      "path": "language_model.model.layers.7.self_attn.o_proj",
      "layer": 7,
      "nparams": 4194304,
      "bits": 3,
      "kind": "3",
      "kl": 0.2620323669243249
    },
    {
      "path": "language_model.model.layers.7.mlp.gate_proj",
      "layer": 7,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.020175184201294494
    },
    {
      "path": "language_model.model.layers.7.mlp.up_proj",
      "layer": 7,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.028832777507502485
    },
    {
      "path": "language_model.model.layers.7.mlp.down_proj",
      "layer": 7,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.03271163825399574
    },
    {
      "path": "language_model.model.layers.8.linear_attn.in_proj_qkv",
      "layer": 8,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.017208185022725524
    },
    {
      "path": "language_model.model.layers.8.linear_attn.in_proj_z",
      "layer": 8,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.008539868907538503
    },
    {
      "path": "language_model.model.layers.8.linear_attn.in_proj_a",
      "layer": 8,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.005789555197582355
    },
    {
      "path": "language_model.model.layers.8.linear_attn.in_proj_b",
      "layer": 8,
      "nparams": 32768,
      "bits": 3,
      "kind": "3",
      "kl": 0.004701556493609789
    },
    {
      "path": "language_model.model.layers.8.linear_attn.out_proj",
      "layer": 8,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.017634184957528064
    },
    {
      "path": "language_model.model.layers.8.mlp.gate_proj",
      "layer": 8,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.0123514499375841
    },
    {
      "path": "language_model.model.layers.8.mlp.up_proj",
      "layer": 8,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.04801137650896942
    },
    {
      "path": "language_model.model.layers.8.mlp.down_proj",
      "layer": 8,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.024051073026259938
    },
    {
      "path": "language_model.model.layers.9.linear_attn.in_proj_qkv",
      "layer": 9,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.033309427679144944
    },
    {
      "path": "language_model.model.layers.9.linear_attn.in_proj_z",
      "layer": 9,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.008170563685431696
    },
    {
      "path": "language_model.model.layers.9.linear_attn.in_proj_a",
      "layer": 9,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.004981094336314765
    },
    {
      "path": "language_model.model.layers.9.linear_attn.in_proj_b",
      "layer": 9,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.004953332458841868
    },
    {
      "path": "language_model.model.layers.9.linear_attn.out_proj",
      "layer": 9,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.02677805449801729
    },
    {
      "path": "language_model.model.layers.9.mlp.gate_proj",
      "layer": 9,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.013528162657920756
    },
    {
      "path": "language_model.model.layers.9.mlp.up_proj",
      "layer": 9,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.022252418419872137
    },
    {
      "path": "language_model.model.layers.9.mlp.down_proj",
      "layer": 9,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.009998477832443698
    },
    {
      "path": "language_model.model.layers.10.linear_attn.in_proj_qkv",
      "layer": 10,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.021357037951699957
    },
    {
      "path": "language_model.model.layers.10.linear_attn.in_proj_z",
      "layer": 10,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.00935590656026332
    },
    {
      "path": "language_model.model.layers.10.linear_attn.in_proj_a",
      "layer": 10,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.003710759392787249
    },
    {
      "path": "language_model.model.layers.10.linear_attn.in_proj_b",
      "layer": 10,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.004896769494430089
    },
    {
      "path": "language_model.model.layers.10.linear_attn.out_proj",
      "layer": 10,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.014627255957999335
    },
    {
      "path": "language_model.model.layers.10.mlp.gate_proj",
      "layer": 10,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.01346382014639341
    },
    {
      "path": "language_model.model.layers.10.mlp.up_proj",
      "layer": 10,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.02852888862376233
    },
    {
      "path": "language_model.model.layers.10.mlp.down_proj",
      "layer": 10,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.01685266069767212
    },
    {
      "path": "language_model.model.layers.11.self_attn.q_proj",
      "layer": 11,
      "nparams": 8388608,
      "bits": 4,
      "kind": "4",
      "kl": 0.02689443748441111
    },
    {
      "path": "language_model.model.layers.11.self_attn.k_proj",
      "layer": 11,
      "nparams": 1048576,
      "bits": 4,
      "kind": "4",
      "kl": 0.015188959495439124
    },
    {
      "path": "language_model.model.layers.11.self_attn.v_proj",
      "layer": 11,
      "nparams": 1048576,
      "bits": 4,
      "kind": "4",
      "kl": 0.022758453695708634
    },
    {
      "path": "language_model.model.layers.11.self_attn.o_proj",
      "layer": 11,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.027215593169970794
    },
    {
      "path": "language_model.model.layers.11.mlp.gate_proj",
      "layer": 11,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.011182316594605536
    },
    {
      "path": "language_model.model.layers.11.mlp.up_proj",
      "layer": 11,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.01113242412163202
    },
    {
      "path": "language_model.model.layers.11.mlp.down_proj",
      "layer": 11,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.009885245854565492
    },
    {
      "path": "language_model.model.layers.12.linear_attn.in_proj_qkv",
      "layer": 12,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.018947691650334015
    },
    {
      "path": "language_model.model.layers.12.linear_attn.in_proj_z",
      "layer": 12,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.006582978430832682
    },
    {
      "path": "language_model.model.layers.12.linear_attn.in_proj_a",
      "layer": 12,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.004025139540875921
    },
    {
      "path": "language_model.model.layers.12.linear_attn.in_proj_b",
      "layer": 12,
      "nparams": 32768,
      "bits": 3,
      "kind": "3",
      "kl": 0.0050430632419457035
    },
    {
      "path": "language_model.model.layers.12.linear_attn.out_proj",
      "layer": 12,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.006496160800308889
    },
    {
      "path": "language_model.model.layers.12.mlp.gate_proj",
      "layer": 12,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.007851327559779145
    },
    {
      "path": "language_model.model.layers.12.mlp.up_proj",
      "layer": 12,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.011228132923593922
    },
    {
      "path": "language_model.model.layers.12.mlp.down_proj",
      "layer": 12,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.004278566865995799
    },
    {
      "path": "language_model.model.layers.13.linear_attn.in_proj_qkv",
      "layer": 13,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.01233742495220853
    },
    {
      "path": "language_model.model.layers.13.linear_attn.in_proj_z",
      "layer": 13,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.004191599132337177
    },
    {
      "path": "language_model.model.layers.13.linear_attn.in_proj_a",
      "layer": 13,
      "nparams": 32768,
      "bits": 3,
      "kind": "3",
      "kl": 0.0026647038361747354
    },
    {
      "path": "language_model.model.layers.13.linear_attn.in_proj_b",
      "layer": 13,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.002115649949411989
    },
    {
      "path": "language_model.model.layers.13.linear_attn.out_proj",
      "layer": 13,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.005793977031598178
    },
    {
      "path": "language_model.model.layers.13.mlp.gate_proj",
      "layer": 13,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.017964901610867313
    },
    {
      "path": "language_model.model.layers.13.mlp.up_proj",
      "layer": 13,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.008408326668711358
    },
    {
      "path": "language_model.model.layers.13.mlp.down_proj",
      "layer": 13,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.009545881598574094
    },
    {
      "path": "language_model.model.layers.14.linear_attn.in_proj_qkv",
      "layer": 14,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.009219910411077833
    },
    {
      "path": "language_model.model.layers.14.linear_attn.in_proj_z",
      "layer": 14,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.004161417746713759
    },
    {
      "path": "language_model.model.layers.14.linear_attn.in_proj_a",
      "layer": 14,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.003323217458649921
    },
    {
      "path": "language_model.model.layers.14.linear_attn.in_proj_b",
      "layer": 14,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.0021712184461907596
    },
    {
      "path": "language_model.model.layers.14.linear_attn.out_proj",
      "layer": 14,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.004262398622704975
    },
    {
      "path": "language_model.model.layers.14.mlp.gate_proj",
      "layer": 14,
      "nparams": 12582912,
      "bits": 2,
      "kind": "2",
      "kl": 0.028299536252745942
    },
    {
      "path": "language_model.model.layers.14.mlp.up_proj",
      "layer": 14,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.0029855243357640063
    },
    {
      "path": "language_model.model.layers.14.mlp.down_proj",
      "layer": 14,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.006347102378594772
    },
    {
      "path": "language_model.model.layers.15.self_attn.q_proj",
      "layer": 15,
      "nparams": 8388608,
      "bits": 4,
      "kind": "4",
      "kl": 0.003643140858535345
    },
    {
      "path": "language_model.model.layers.15.self_attn.k_proj",
      "layer": 15,
      "nparams": 1048576,
      "bits": 4,
      "kind": "4",
      "kl": 0.0052556378501606195
    },
    {
      "path": "language_model.model.layers.15.self_attn.v_proj",
      "layer": 15,
      "nparams": 1048576,
      "bits": 4,
      "kind": "4",
      "kl": 0.006322858785280738
    },
    {
      "path": "language_model.model.layers.15.self_attn.o_proj",
      "layer": 15,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.005030957821139618
    },
    {
      "path": "language_model.model.layers.15.mlp.gate_proj",
      "layer": 15,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.0031369600735519213
    },
    {
      "path": "language_model.model.layers.15.mlp.up_proj",
      "layer": 15,
      "nparams": 12582912,
      "bits": 2,
      "kind": "2",
      "kl": 0.011699879395203576
    },
    {
      "path": "language_model.model.layers.15.mlp.down_proj",
      "layer": 15,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.0065643853607818565
    },
    {
      "path": "language_model.model.layers.16.linear_attn.in_proj_qkv",
      "layer": 16,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.00652818737017804
    },
    {
      "path": "language_model.model.layers.16.linear_attn.in_proj_z",
      "layer": 16,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.002564514868387881
    },
    {
      "path": "language_model.model.layers.16.linear_attn.in_proj_a",
      "layer": 16,
      "nparams": 32768,
      "bits": 3,
      "kind": "3",
      "kl": 0.001746105079547036
    },
    {
      "path": "language_model.model.layers.16.linear_attn.in_proj_b",
      "layer": 16,
      "nparams": 32768,
      "bits": 2,
      "kind": "2",
      "kl": 0.0024505408935464497
    },
    {
      "path": "language_model.model.layers.16.linear_attn.out_proj",
      "layer": 16,
      "nparams": 4194304,
      "bits": 3,
      "kind": "3",
      "kl": 0.0035994804530835194
    },
    {
      "path": "language_model.model.layers.16.mlp.gate_proj",
      "layer": 16,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.005310726191144668
    },
    {
      "path": "language_model.model.layers.16.mlp.up_proj",
      "layer": 16,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.003419146426726522
    },
    {
      "path": "language_model.model.layers.16.mlp.down_proj",
      "layer": 16,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.004005140748420486
    },
    {
      "path": "language_model.model.layers.17.linear_attn.in_proj_qkv",
      "layer": 17,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.0037804420468492837
    },
    {
      "path": "language_model.model.layers.17.linear_attn.in_proj_z",
      "layer": 17,
      "nparams": 4194304,
      "bits": 2,
      "kind": "2",
      "kl": 0.0035376995942987717
    },
    {
      "path": "language_model.model.layers.17.linear_attn.in_proj_a",
      "layer": 17,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.001846708406487311
    },
    {
      "path": "language_model.model.layers.17.linear_attn.in_proj_b",
      "layer": 17,
      "nparams": 32768,
      "bits": 3,
      "kind": "3",
      "kl": 0.0016845755580104484
    },
    {
      "path": "language_model.model.layers.17.linear_attn.out_proj",
      "layer": 17,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.001894043444011438
    },
    {
      "path": "language_model.model.layers.17.mlp.gate_proj",
      "layer": 17,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.0032691978276667404
    },
    {
      "path": "language_model.model.layers.17.mlp.up_proj",
      "layer": 17,
      "nparams": 12582912,
      "bits": 2,
      "kind": "2",
      "kl": 0.007563979548310841
    },
    {
      "path": "language_model.model.layers.17.mlp.down_proj",
      "layer": 17,
      "nparams": 12582912,
      "bits": 2,
      "kind": "2",
      "kl": 0.01315814833027159
    },
    {
      "path": "language_model.model.layers.18.linear_attn.in_proj_qkv",
      "layer": 18,
      "nparams": 12582912,
      "bits": 2,
      "kind": "2",
      "kl": 0.0052731935273633256
    },
    {
      "path": "language_model.model.layers.18.linear_attn.in_proj_z",
      "layer": 18,
      "nparams": 4194304,
      "bits": 2,
      "kind": "2",
      "kl": 0.0031266101628979383
    },
    {
      "path": "language_model.model.layers.18.linear_attn.in_proj_a",
      "layer": 18,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.0019552802619046527
    },
    {
      "path": "language_model.model.layers.18.linear_attn.in_proj_b",
      "layer": 18,
      "nparams": 32768,
      "bits": 2,
      "kind": "2",
      "kl": 0.0018198732860189083
    },
    {
      "path": "language_model.model.layers.18.linear_attn.out_proj",
      "layer": 18,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.0028847887347183973
    },
    {
      "path": "language_model.model.layers.18.mlp.gate_proj",
      "layer": 18,
      "nparams": 12582912,
      "bits": 2,
      "kind": "2",
      "kl": 0.01898859203658337
    },
    {
      "path": "language_model.model.layers.18.mlp.up_proj",
      "layer": 18,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.0031293535751949733
    },
    {
      "path": "language_model.model.layers.18.mlp.down_proj",
      "layer": 18,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.0037580042827521183
    },
    {
      "path": "language_model.model.layers.19.self_attn.q_proj",
      "layer": 19,
      "nparams": 8388608,
      "bits": 4,
      "kind": "4",
      "kl": 0.0033964504416273814
    },
    {
      "path": "language_model.model.layers.19.self_attn.k_proj",
      "layer": 19,
      "nparams": 1048576,
      "bits": 4,
      "kind": "4",
      "kl": 0.0048760714135432336
    },
    {
      "path": "language_model.model.layers.19.self_attn.v_proj",
      "layer": 19,
      "nparams": 1048576,
      "bits": 4,
      "kind": "4",
      "kl": 0.003869429662332861
    },
    {
      "path": "language_model.model.layers.19.self_attn.o_proj",
      "layer": 19,
      "nparams": 4194304,
      "bits": 3,
      "kind": "3",
      "kl": 0.018552193105078302
    },
    {
      "path": "language_model.model.layers.19.mlp.gate_proj",
      "layer": 19,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.0020007518508758588
    },
    {
      "path": "language_model.model.layers.19.mlp.up_proj",
      "layer": 19,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.0020792066361257477
    },
    {
      "path": "language_model.model.layers.19.mlp.down_proj",
      "layer": 19,
      "nparams": 12582912,
      "bits": 2,
      "kind": "ternary",
      "kl": 0.006833145492931654
    },
    {
      "path": "language_model.model.layers.20.linear_attn.in_proj_qkv",
      "layer": 20,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.0025472445161104753
    },
    {
      "path": "language_model.model.layers.20.linear_attn.in_proj_z",
      "layer": 20,
      "nparams": 4194304,
      "bits": 2,
      "kind": "2",
      "kl": 0.002592213691815188
    },
    {
      "path": "language_model.model.layers.20.linear_attn.in_proj_a",
      "layer": 20,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.0009051710490314626
    },
    {
      "path": "language_model.model.layers.20.linear_attn.in_proj_b",
      "layer": 20,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.0011463968076723563
    },
    {
      "path": "language_model.model.layers.20.linear_attn.out_proj",
      "layer": 20,
      "nparams": 4194304,
      "bits": 2,
      "kind": "2",
      "kl": 0.005356271293440156
    },
    {
      "path": "language_model.model.layers.20.mlp.gate_proj",
      "layer": 20,
      "nparams": 12582912,
      "bits": 2,
      "kind": "2",
      "kl": 0.0021228656968947774
    },
    {
      "path": "language_model.model.layers.20.mlp.up_proj",
      "layer": 20,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.0024053604971531754
    },
    {
      "path": "language_model.model.layers.20.mlp.down_proj",
      "layer": 20,
      "nparams": 12582912,
      "bits": 2,
      "kind": "2",
      "kl": 0.004477630598479986
    },
    {
      "path": "language_model.model.layers.21.linear_attn.in_proj_qkv",
      "layer": 21,
      "nparams": 12582912,
      "bits": 2,
      "kind": "2",
      "kl": 0.004200707271791614
    },
    {
      "path": "language_model.model.layers.21.linear_attn.in_proj_z",
      "layer": 21,
      "nparams": 4194304,
      "bits": 2,
      "kind": "2",
      "kl": 0.002413397581031617
    },
    {
      "path": "language_model.model.layers.21.linear_attn.in_proj_a",
      "layer": 21,
      "nparams": 32768,
      "bits": 3,
      "kind": "3",
      "kl": 0.0005669125558536048
    },
    {
      "path": "language_model.model.layers.21.linear_attn.in_proj_b",
      "layer": 21,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.0006892036110613707
    },
    {
      "path": "language_model.model.layers.21.linear_attn.out_proj",
      "layer": 21,
      "nparams": 4194304,
      "bits": 3,
      "kind": "3",
      "kl": 0.002135502337464641
    },
    {
      "path": "language_model.model.layers.21.mlp.gate_proj",
      "layer": 21,
      "nparams": 12582912,
      "bits": 2,
      "kind": "2",
      "kl": 0.003934425296862159
    },
    {
      "path": "language_model.model.layers.21.mlp.up_proj",
      "layer": 21,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.0022890209206124167
    },
    {
      "path": "language_model.model.layers.21.mlp.down_proj",
      "layer": 21,
      "nparams": 12582912,
      "bits": 2,
      "kind": "2",
      "kl": 0.009127860100761054
    },
    {
      "path": "language_model.model.layers.22.linear_attn.in_proj_qkv",
      "layer": 22,
      "nparams": 12582912,
      "bits": 2,
      "kind": "2",
      "kl": 0.0053570328330360085
    },
    {
      "path": "language_model.model.layers.22.linear_attn.in_proj_z",
      "layer": 22,
      "nparams": 4194304,
      "bits": 3,
      "kind": "3",
      "kl": 0.0030685547854956505
    },
    {
      "path": "language_model.model.layers.22.linear_attn.in_proj_a",
      "layer": 22,
      "nparams": 32768,
      "bits": 4,
      "kind": "4",
      "kl": 0.0006597311378955131
    },
    {
      "path": "language_model.model.layers.22.linear_attn.in_proj_b",
      "layer": 22,
      "nparams": 32768,
      "bits": 3,
      "kind": "3",
      "kl": 0.001038058904486026
    },
    {
      "path": "language_model.model.layers.22.linear_attn.out_proj",
      "layer": 22,
      "nparams": 4194304,
      "bits": 3,
      "kind": "3",
      "kl": 0.0029542600866622193
    },
    {
      "path": "language_model.model.layers.22.mlp.gate_proj",
      "layer": 22,
      "nparams": 12582912,
      "bits": 2,
      "kind": "2",
      "kl": 0.005691486545448434
    },
    {
      "path": "language_model.model.layers.22.mlp.up_proj",
      "layer": 22,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.0028654545434373922
    },
    {
      "path": "language_model.model.layers.22.mlp.down_proj",
      "layer": 22,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.004033058114196539
    },
    {
      "path": "language_model.model.layers.23.self_attn.q_proj",
      "layer": 23,
      "nparams": 8388608,
      "bits": 4,
      "kind": "4",
      "kl": 0.0034863865396836083
    },
    {
      "path": "language_model.model.layers.23.self_attn.k_proj",
      "layer": 23,
      "nparams": 1048576,
      "bits": 4,
      "kind": "4",
      "kl": 0.003449246781242146
    },
    {
      "path": "language_model.model.layers.23.self_attn.v_proj",
      "layer": 23,
      "nparams": 1048576,
      "bits": 4,
      "kind": "4",
      "kl": 0.004781225921661813
    },
    {
      "path": "language_model.model.layers.23.self_attn.o_proj",
      "layer": 23,
      "nparams": 4194304,
      "bits": 4,
      "kind": "4",
      "kl": 0.004980091902814513
    },
    {
      "path": "language_model.model.layers.23.mlp.gate_proj",
      "layer": 23,
      "nparams": 12582912,
      "bits": 3,
      "kind": "3",
      "kl": 0.005546539586926521
    },
    {
      "path": "language_model.model.layers.23.mlp.up_proj",
      "layer": 23,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.0036412261524133302
    },
    {
      "path": "language_model.model.layers.23.mlp.down_proj",
      "layer": 23,
      "nparams": 12582912,
      "bits": 4,
      "kind": "4",
      "kl": 0.009847478632996235
    }
  ]
}