{
  "source": "/workspace/svd_safety/assets/models/Mistral-7B-Instruct-v0.2",
  "upstream": "hiahei/Swift-SVD (compression) + AIoT-MLSys-Lab/SVD-LLM utils/LoRA.py (recovery)",
  "stage": "finetuned",
  "ratio_removed": 0.2,
  "internal_ratio_kept": 0.8,
  "from_pt": "/workspace/svd_safety/results/swift_ft/cells/mis7_swift_remove20/finetune_v/merge.pt",
  "mean_rel_err": 0.40079765720292926,
  "max_rel_err": 0.5248842835426331,
  "param_fraction": 0.8004338191105769,
  "ranks": {
    "0.q_proj": 1843,
    "0.k_proj": 776,
    "0.v_proj": 706,
    "0.o_proj": 1683,
    "0.gate_proj": 2688,
    "0.up_proj": 2688,
    "0.down_proj": 2723,
    "1.q_proj": 1851,
    "1.k_proj": 789,
    "1.v_proj": 743,
    "1.o_proj": 1733,
    "1.gate_proj": 2688,
    "1.up_proj": 2688,
    "1.down_proj": 3161,
    "2.q_proj": 1508,
    "2.k_proj": 598,
    "2.v_proj": 587,
    "2.o_proj": 1451,
    "2.gate_proj": 2688,
    "2.up_proj": 2688,
    "2.down_proj": 2163,
    "3.q_proj": 1504,
    "3.k_proj": 595,
    "3.v_proj": 581,
    "3.o_proj": 1437,
    "3.gate_proj": 2688,
    "3.up_proj": 2688,
    "3.down_proj": 2166,
    "4.q_proj": 1501,
    "4.k_proj": 594,
    "4.v_proj": 586,
    "4.o_proj": 1445,
    "4.gate_proj": 2688,
    "4.up_proj": 2688,
    "4.down_proj": 2193,
    "5.q_proj": 1501,
    "5.k_proj": 595,
    "5.v_proj": 586,
    "5.o_proj": 1456,
    "5.gate_proj": 2688,
    "5.up_proj": 2688,
    "5.down_proj": 2216,
    "6.q_proj": 1503,
    "6.k_proj": 596,
    "6.v_proj": 588,
    "6.o_proj": 1468,
    "6.gate_proj": 2688,
    "6.up_proj": 2688,
    "6.down_proj": 2233,
    "7.q_proj": 1501,
    "7.k_proj": 597,
    "7.v_proj": 589,
    "7.o_proj": 1480,
    "7.gate_proj": 2688,
    "7.up_proj": 2688,
    "7.down_proj": 2250,
    "8.q_proj": 1497,
    "8.k_proj": 593,
    "8.v_proj": 589,
    "8.o_proj": 1479,
    "8.gate_proj": 2688,
    "8.up_proj": 2688,
    "8.down_proj": 2246,
    "9.q_proj": 1494,
    "9.k_proj": 592,
    "9.v_proj": 590,
    "9.o_proj": 1479,
    "9.gate_proj": 2688,
    "9.up_proj": 2688,
    "9.down_proj": 2255,
    "10.q_proj": 1496,
    "10.k_proj": 593,
    "10.v_proj": 590,
    "10.o_proj": 1491,
    "10.gate_proj": 2688,
    "10.up_proj": 2688,
    "10.down_proj": 2256,
    "11.q_proj": 1494,
    "11.k_proj": 592,
    "11.v_proj": 592,
    "11.o_proj": 1495,
    "11.gate_proj": 2688,
    "11.up_proj": 2688,
    "11.down_proj": 2260,
    "12.q_proj": 1503,
    "12.k_proj": 598,
    "12.v_proj": 597,
    "12.o_proj": 1506,
    "12.gate_proj": 2688,
    "12.up_proj": 2688,
    "12.down_proj": 2285,
    "13.q_proj": 1498,
    "13.k_proj": 596,
    "13.v_proj": 595,
    "13.o_proj": 1503,
    "13.gate_proj": 2688,
    "13.up_proj": 2688,
    "13.down_proj": 2285,
    "14.q_proj": 1510,
    "14.k_proj": 601,
    "14.v_proj": 604,
    "14.o_proj": 1519,
    "14.gate_proj": 2688,
    "14.up_proj": 2688,
    "14.down_proj": 2323,
    "15.q_proj": 1527,
    "15.k_proj": 612,
    "15.v_proj": 615,
    "15.o_proj": 1542,
    "15.gate_proj": 2688,
    "15.up_proj": 2688,
    "15.down_proj": 2369,
    "16.q_proj": 1534,
    "16.k_proj": 615,
    "16.v_proj": 618,
    "16.o_proj": 1549,
    "16.gate_proj": 2688,
    "16.up_proj": 2688,
    "16.down_proj": 2399,
    "17.q_proj": 1565,
    "17.k_proj": 631,
    "17.v_proj": 635,
    "17.o_proj": 1584,
    "17.gate_proj": 2688,
    "17.up_proj": 2688,
    "17.down_proj": 2469,
    "18.q_proj": 1565,
    "18.k_proj": 630,
    "18.v_proj": 636,
    "18.o_proj": 1588,
    "18.gate_proj": 2688,
    "18.up_proj": 2688,
    "18.down_proj": 2487,
    "19.q_proj": 1544,
    "19.k_proj": 620,
    "19.v_proj": 626,
    "19.o_proj": 1572,
    "19.gate_proj": 2688,
    "19.up_proj": 2688,
    "19.down_proj": 2458,
    "20.q_proj": 1507,
    "20.k_proj": 599,
    "20.v_proj": 606,
    "20.o_proj": 1527,
    "20.gate_proj": 2688,
    "20.up_proj": 2688,
    "20.down_proj": 2381,
    "21.q_proj": 1497,
    "21.k_proj": 593,
    "21.v_proj": 601,
    "21.o_proj": 1515,
    "21.gate_proj": 2688,
    "21.up_proj": 2688,
    "21.down_proj": 2363,
    "22.q_proj": 1482,
    "22.k_proj": 583,
    "22.v_proj": 594,
    "22.o_proj": 1495,
    "22.gate_proj": 2688,
    "22.up_proj": 2688,
    "22.down_proj": 2332,
    "23.q_proj": 1483,
    "23.k_proj": 584,
    "23.v_proj": 593,
    "23.o_proj": 1505,
    "23.gate_proj": 2688,
    "23.up_proj": 2688,
    "23.down_proj": 2336,
    "24.q_proj": 1482,
    "24.k_proj": 584,
    "24.v_proj": 594,
    "24.o_proj": 1511,
    "24.gate_proj": 2688,
    "24.up_proj": 2688,
    "24.down_proj": 2338,
    "25.q_proj": 1486,
    "25.k_proj": 586,
    "25.v_proj": 597,
    "25.o_proj": 1521,
    "25.gate_proj": 2688,
    "25.up_proj": 2688,
    "25.down_proj": 2354,
    "26.q_proj": 1493,
    "26.k_proj": 589,
    "26.v_proj": 602,
    "26.o_proj": 1532,
    "26.gate_proj": 2688,
    "26.up_proj": 2688,
    "26.down_proj": 2373,
    "27.q_proj": 1498,
    "27.k_proj": 592,
    "27.v_proj": 606,
    "27.o_proj": 1532,
    "27.gate_proj": 2688,
    "27.up_proj": 2688,
    "27.down_proj": 2389,
    "28.q_proj": 1518,
    "28.k_proj": 603,
    "28.v_proj": 618,
    "28.o_proj": 1561,
    "28.gate_proj": 2688,
    "28.up_proj": 2688,
    "28.down_proj": 2451,
    "29.q_proj": 1530,
    "29.k_proj": 609,
    "29.v_proj": 626,
    "29.o_proj": 1591,
    "29.gate_proj": 2688,
    "29.up_proj": 2688,
    "29.down_proj": 2480,
    "30.q_proj": 1553,
    "30.k_proj": 621,
    "30.v_proj": 642,
    "30.o_proj": 1620,
    "30.gate_proj": 2688,
    "30.up_proj": 2688,
    "30.down_proj": 2549,
    "31.q_proj": 1684,
    "31.k_proj": 692,
    "31.v_proj": 716,
    "31.o_proj": 1782,
    "31.gate_proj": 2688,
    "31.up_proj": 2688,
    "31.down_proj": 2905
  },
  "rel_err": {
    "0.q_proj": 0.08500257134437561,
    "0.k_proj": 0.0716991052031517,
    "0.v_proj": 0.22834858298301697,
    "0.o_proj": 0.3019510805606842,
    "0.gate_proj": 0.26146504282951355,
    "0.up_proj": 0.2859381437301636,
    "0.down_proj": 0.3900061547756195,
    "1.q_proj": 0.13657206296920776,
    "1.k_proj": 0.07040991634130478,
    "1.v_proj": 0.34303975105285645,
    "1.o_proj": 0.4063681960105896,
    "1.gate_proj": 0.3133699595928192,
    "1.up_proj": 0.33793723583221436,
    "1.down_proj": 0.3351709246635437,
    "2.q_proj": 0.35540199279785156,
    "2.k_proj": 0.3083972930908203,
    "2.v_proj": 0.4814577102661133,
    "2.o_proj": 0.47539830207824707,
    "2.gate_proj": 0.3777889311313629,
    "2.up_proj": 0.39566245675086975,
    "2.down_proj": 0.5091328620910645,
    "3.q_proj": 0.3229297995567322,
    "3.k_proj": 0.25059062242507935,
    "3.v_proj": 0.5148245096206665,
    "3.o_proj": 0.5196603536605835,
    "3.gate_proj": 0.38781943917274475,
    "3.up_proj": 0.4100722670555115,
    "3.down_proj": 0.49922603368759155,
    "4.q_proj": 0.3310073912143707,
    "4.k_proj": 0.2755686044692993,
    "4.v_proj": 0.4732750952243805,
    "4.o_proj": 0.4666188955307007,
    "4.gate_proj": 0.3899378180503845,
    "4.up_proj": 0.4183727204799652,
    "4.down_proj": 0.5054365992546082,
    "5.q_proj": 0.3576917052268982,
    "5.k_proj": 0.3083855211734772,
    "5.v_proj": 0.5068833231925964,
    "5.o_proj": 0.496389240026474,
    "5.gate_proj": 0.3855798542499542,
    "5.up_proj": 0.4201187789440155,
    "5.down_proj": 0.4902642071247101,
    "6.q_proj": 0.346874475479126,
    "6.k_proj": 0.2829170227050781,
    "6.v_proj": 0.4889771640300751,
    "6.o_proj": 0.48441195487976074,
    "6.gate_proj": 0.39520981907844543,
    "6.up_proj": 0.4232458472251892,
    "6.down_proj": 0.4810233414173126,
    "7.q_proj": 0.33142852783203125,
    "7.k_proj": 0.2655273973941803,
    "7.v_proj": 0.4808761775493622,
    "7.o_proj": 0.4556846618652344,
    "7.gate_proj": 0.387683242559433,
    "7.up_proj": 0.4178556799888611,
    "7.down_proj": 0.4722205698490143,
    "8.q_proj": 0.33769723773002625,
    "8.k_proj": 0.2662927508354187,
    "8.v_proj": 0.4425075054168701,
    "8.o_proj": 0.44497787952423096,
    "8.gate_proj": 0.3896368443965912,
    "8.up_proj": 0.41572481393814087,
    "8.down_proj": 0.4673706591129303,
    "9.q_proj": 0.3340452313423157,
    "9.k_proj": 0.2718675434589386,
    "9.v_proj": 0.4268815815448761,
    "9.o_proj": 0.44764506816864014,
    "9.gate_proj": 0.38748565316200256,
    "9.up_proj": 0.4152057468891144,
    "9.down_proj": 0.4528351426124573,
    "10.q_proj": 0.3335397243499756,
    "10.k_proj": 0.2776794731616974,
    "10.v_proj": 0.4279860556125641,
    "10.o_proj": 0.44602537155151367,
    "10.gate_proj": 0.38787901401519775,
    "10.up_proj": 0.4141860604286194,
    "10.down_proj": 0.46012479066848755,
    "11.q_proj": 0.3297421634197235,
    "11.k_proj": 0.26538947224617004,
    "11.v_proj": 0.3986572325229645,
    "11.o_proj": 0.4476287066936493,
    "11.gate_proj": 0.38680461049079895,
    "11.up_proj": 0.41310203075408936,
    "11.down_proj": 0.45812714099884033,
    "12.q_proj": 0.319181889295578,
    "12.k_proj": 0.24617546796798706,
    "12.v_proj": 0.4383234977722168,
    "12.o_proj": 0.42985114455223083,
    "12.gate_proj": 0.3842836320400238,
    "12.up_proj": 0.4086284637451172,
    "12.down_proj": 0.45287853479385376,
    "13.q_proj": 0.342117041349411,
    "13.k_proj": 0.28727155923843384,
    "13.v_proj": 0.4403918981552124,
    "13.o_proj": 0.43300414085388184,
    "13.gate_proj": 0.3891292214393616,
    "13.up_proj": 0.4081304669380188,
    "13.down_proj": 0.4504336416721344,
    "14.q_proj": 0.31687843799591064,
    "14.k_proj": 0.26946091651916504,
    "14.v_proj": 0.44405218958854675,
    "14.o_proj": 0.43850401043891907,
    "14.gate_proj": 0.38567572832107544,
    "14.up_proj": 0.4120667278766632,
    "14.down_proj": 0.4714415967464447,
    "15.q_proj": 0.31565016508102417,
    "15.k_proj": 0.2453945130109787,
    "15.v_proj": 0.4641188681125641,
    "15.o_proj": 0.4399622678756714,
    "15.gate_proj": 0.38257908821105957,
    "15.up_proj": 0.408530592918396,
    "15.down_proj": 0.4638931155204773,
    "16.q_proj": 0.32283511757850647,
    "16.k_proj": 0.2652204930782318,
    "16.v_proj": 0.464958131313324,
    "16.o_proj": 0.42660167813301086,
    "16.gate_proj": 0.3851650655269623,
    "16.up_proj": 0.41526761651039124,
    "16.down_proj": 0.46140268445014954,
    "17.q_proj": 0.3427833616733551,
    "17.k_proj": 0.2890065610408783,
    "17.v_proj": 0.4380885064601898,
    "17.o_proj": 0.44451603293418884,
    "17.gate_proj": 0.3959171772003174,
    "17.up_proj": 0.4251500368118286,
    "17.down_proj": 0.4505082368850708,
    "18.q_proj": 0.3227398097515106,
    "18.k_proj": 0.2645529508590698,
    "18.v_proj": 0.486581027507782,
    "18.o_proj": 0.4234537184238434,
    "18.gate_proj": 0.39415696263313293,
    "18.up_proj": 0.42284369468688965,
    "18.down_proj": 0.4652669429779053,
    "19.q_proj": 0.36690786480903625,
    "19.k_proj": 0.27805012464523315,
    "19.v_proj": 0.5099502205848694,
    "19.o_proj": 0.4442979693412781,
    "19.gate_proj": 0.39896416664123535,
    "19.up_proj": 0.42378658056259155,
    "19.down_proj": 0.48121050000190735,
    "20.q_proj": 0.38949716091156006,
    "20.k_proj": 0.32147786021232605,
    "20.v_proj": 0.4748760461807251,
    "20.o_proj": 0.46483227610588074,
    "20.gate_proj": 0.4035407602787018,
    "20.up_proj": 0.423808217048645,
    "20.down_proj": 0.49269551038742065,
    "21.q_proj": 0.41730770468711853,
    "21.k_proj": 0.3455001711845398,
    "21.v_proj": 0.45047393441200256,
    "21.o_proj": 0.5031458139419556,
    "21.gate_proj": 0.4041469395160675,
    "21.up_proj": 0.4253368079662323,
    "21.down_proj": 0.4889674484729767,
    "22.q_proj": 0.4534510374069214,
    "22.k_proj": 0.37348392605781555,
    "22.v_proj": 0.4685239791870117,
    "22.o_proj": 0.5248842835426331,
    "22.gate_proj": 0.4087425768375397,
    "22.up_proj": 0.42668619751930237,
    "22.down_proj": 0.4892253875732422,
    "23.q_proj": 0.4440714418888092,
    "23.k_proj": 0.3682272434234619,
    "23.v_proj": 0.49346232414245605,
    "23.o_proj": 0.502592921257019,
    "23.gate_proj": 0.4109232425689697,
    "23.up_proj": 0.42807736992836,
    "23.down_proj": 0.48296502232551575,
    "24.q_proj": 0.4487209916114807,
    "24.k_proj": 0.38935786485671997,
    "24.v_proj": 0.5036438703536987,
    "24.o_proj": 0.5222789645195007,
    "24.gate_proj": 0.4119880795478821,
    "24.up_proj": 0.42954087257385254,
    "24.down_proj": 0.47408875823020935,
    "25.q_proj": 0.4551296532154083,
    "25.k_proj": 0.37871479988098145,
    "25.v_proj": 0.4806521534919739,
    "25.o_proj": 0.5046061873435974,
    "25.gate_proj": 0.4138759672641754,
    "25.up_proj": 0.4307102560997009,
    "25.down_proj": 0.463013231754303,
    "26.q_proj": 0.44990450143814087,
    "26.k_proj": 0.3649565577507019,
    "26.v_proj": 0.46789830923080444,
    "26.o_proj": 0.4847952425479889,
    "26.gate_proj": 0.414262056350708,
    "26.up_proj": 0.43036970496177673,
    "26.down_proj": 0.4495799243450165,
    "27.q_proj": 0.4583161473274231,
    "27.k_proj": 0.39734572172164917,
    "27.v_proj": 0.4797323942184448,
    "27.o_proj": 0.47943976521492004,
    "27.gate_proj": 0.41736742854118347,
    "27.up_proj": 0.4318309724330902,
    "27.down_proj": 0.44924622774124146,
    "28.q_proj": 0.41977617144584656,
    "28.k_proj": 0.3314531147480011,
    "28.v_proj": 0.4329354465007782,
    "28.o_proj": 0.4519560635089874,
    "28.gate_proj": 0.41815441846847534,
    "28.up_proj": 0.4296439588069916,
    "28.down_proj": 0.4507562816143036,
    "29.q_proj": 0.4013705253601074,
    "29.k_proj": 0.3302054703235626,
    "29.v_proj": 0.45994922518730164,
    "29.o_proj": 0.4338970482349396,
    "29.gate_proj": 0.4139324426651001,
    "29.up_proj": 0.4252754747867584,
    "29.down_proj": 0.4538421928882599,
    "30.q_proj": 0.3819069266319275,
    "30.k_proj": 0.300334095954895,
    "30.v_proj": 0.3898763358592987,
    "30.o_proj": 0.4189046621322632,
    "30.gate_proj": 0.4136664569377899,
    "30.up_proj": 0.4180448055267334,
    "30.down_proj": 0.43752264976501465,
    "31.q_proj": 0.3336416184902191,
    "31.k_proj": 0.2430993765592575,
    "31.v_proj": 0.32917752861976624,
    "31.o_proj": 0.3434261381626129,
    "31.gate_proj": 0.4216300845146179,
    "31.up_proj": 0.40061134099960327,
    "31.down_proj": 0.3710506558418274
  },
  "step": "swift_svd_dynamic_finetuned",
  "compression": {
    "upstream": "hiahei/Swift-SVD",
    "step": "swift_svd_dynamic",
    "alloc": "dynamic",
    "alpha": 0.6,
    "nsamples": 256,
    "seqlen": 2048,
    "dataset": "WikiText2",
    "seed": 42,
    "gqa_budget": "shape",
    "param_fraction": 0.8004338191105769,
    "mean_rel_err": 0.3994709092990628,
    "max_rel_err": 0.5239740610122681
  },
  "factorisation": {
    "max_rel_err": 0.0013197725638747215,
    "factor_params": 5586485248,
    "dense_params": 6979321856
  },
  "finetune": {
    "protocol": "svdllm_stage2_u_then_v",
    "script": "gitclones/SVD-LLM/utils/LoRA.py",
    "data": "alpaca-cleaned",
    "epochs_per_half": 2,
    "learning_rate": 0.0001,
    "batch_size": 64,
    "micro_batch_size": 4,
    "lora_r": 8,
    "lora_alpha": 16,
    "lora_dropout": 0.05,
    "cutoff_len": 256,
    "val_set_size": 2000,
    "max_train_samples": "all",
    "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)"
  }
}