| { |
| "source": "/workspace/svd_safety/assets/models/Mistral-7B-Instruct-v0.2", |
| "upstream": "hiahei/Swift-SVD (compression) + AIoT-MLSys-Lab/SVD-LLM utils/LoRA.py (recovery)", |
| "stage": "finetuned", |
| "ratio_removed": 0.2, |
| "internal_ratio_kept": 0.8, |
| "from_pt": "/workspace/svd_safety/results/swift_ft/cells/mis7_swift_remove20/finetune_v/merge.pt", |
| "mean_rel_err": 0.40079765720292926, |
| "max_rel_err": 0.5248842835426331, |
| "param_fraction": 0.8004338191105769, |
| "ranks": { |
| "0.q_proj": 1843, |
| "0.k_proj": 776, |
| "0.v_proj": 706, |
| "0.o_proj": 1683, |
| "0.gate_proj": 2688, |
| "0.up_proj": 2688, |
| "0.down_proj": 2723, |
| "1.q_proj": 1851, |
| "1.k_proj": 789, |
| "1.v_proj": 743, |
| "1.o_proj": 1733, |
| "1.gate_proj": 2688, |
| "1.up_proj": 2688, |
| "1.down_proj": 3161, |
| "2.q_proj": 1508, |
| "2.k_proj": 598, |
| "2.v_proj": 587, |
| "2.o_proj": 1451, |
| "2.gate_proj": 2688, |
| "2.up_proj": 2688, |
| "2.down_proj": 2163, |
| "3.q_proj": 1504, |
| "3.k_proj": 595, |
| "3.v_proj": 581, |
| "3.o_proj": 1437, |
| "3.gate_proj": 2688, |
| "3.up_proj": 2688, |
| "3.down_proj": 2166, |
| "4.q_proj": 1501, |
| "4.k_proj": 594, |
| "4.v_proj": 586, |
| "4.o_proj": 1445, |
| "4.gate_proj": 2688, |
| "4.up_proj": 2688, |
| "4.down_proj": 2193, |
| "5.q_proj": 1501, |
| "5.k_proj": 595, |
| "5.v_proj": 586, |
| "5.o_proj": 1456, |
| "5.gate_proj": 2688, |
| "5.up_proj": 2688, |
| "5.down_proj": 2216, |
| "6.q_proj": 1503, |
| "6.k_proj": 596, |
| "6.v_proj": 588, |
| "6.o_proj": 1468, |
| "6.gate_proj": 2688, |
| "6.up_proj": 2688, |
| "6.down_proj": 2233, |
| "7.q_proj": 1501, |
| "7.k_proj": 597, |
| "7.v_proj": 589, |
| "7.o_proj": 1480, |
| "7.gate_proj": 2688, |
| "7.up_proj": 2688, |
| "7.down_proj": 2250, |
| "8.q_proj": 1497, |
| "8.k_proj": 593, |
| "8.v_proj": 589, |
| "8.o_proj": 1479, |
| "8.gate_proj": 2688, |
| "8.up_proj": 2688, |
| "8.down_proj": 2246, |
| "9.q_proj": 1494, |
| "9.k_proj": 592, |
| "9.v_proj": 590, |
| "9.o_proj": 1479, |
| "9.gate_proj": 2688, |
| "9.up_proj": 2688, |
| "9.down_proj": 2255, |
| "10.q_proj": 1496, |
| "10.k_proj": 593, |
| "10.v_proj": 590, |
| "10.o_proj": 1491, |
| "10.gate_proj": 2688, |
| "10.up_proj": 2688, |
| "10.down_proj": 2256, |
| "11.q_proj": 1494, |
| "11.k_proj": 592, |
| "11.v_proj": 592, |
| "11.o_proj": 1495, |
| "11.gate_proj": 2688, |
| "11.up_proj": 2688, |
| "11.down_proj": 2260, |
| "12.q_proj": 1503, |
| "12.k_proj": 598, |
| "12.v_proj": 597, |
| "12.o_proj": 1506, |
| "12.gate_proj": 2688, |
| "12.up_proj": 2688, |
| "12.down_proj": 2285, |
| "13.q_proj": 1498, |
| "13.k_proj": 596, |
| "13.v_proj": 595, |
| "13.o_proj": 1503, |
| "13.gate_proj": 2688, |
| "13.up_proj": 2688, |
| "13.down_proj": 2285, |
| "14.q_proj": 1510, |
| "14.k_proj": 601, |
| "14.v_proj": 604, |
| "14.o_proj": 1519, |
| "14.gate_proj": 2688, |
| "14.up_proj": 2688, |
| "14.down_proj": 2323, |
| "15.q_proj": 1527, |
| "15.k_proj": 612, |
| "15.v_proj": 615, |
| "15.o_proj": 1542, |
| "15.gate_proj": 2688, |
| "15.up_proj": 2688, |
| "15.down_proj": 2369, |
| "16.q_proj": 1534, |
| "16.k_proj": 615, |
| "16.v_proj": 618, |
| "16.o_proj": 1549, |
| "16.gate_proj": 2688, |
| "16.up_proj": 2688, |
| "16.down_proj": 2399, |
| "17.q_proj": 1565, |
| "17.k_proj": 631, |
| "17.v_proj": 635, |
| "17.o_proj": 1584, |
| "17.gate_proj": 2688, |
| "17.up_proj": 2688, |
| "17.down_proj": 2469, |
| "18.q_proj": 1565, |
| "18.k_proj": 630, |
| "18.v_proj": 636, |
| "18.o_proj": 1588, |
| "18.gate_proj": 2688, |
| "18.up_proj": 2688, |
| "18.down_proj": 2487, |
| "19.q_proj": 1544, |
| "19.k_proj": 620, |
| "19.v_proj": 626, |
| "19.o_proj": 1572, |
| "19.gate_proj": 2688, |
| "19.up_proj": 2688, |
| "19.down_proj": 2458, |
| "20.q_proj": 1507, |
| "20.k_proj": 599, |
| "20.v_proj": 606, |
| "20.o_proj": 1527, |
| "20.gate_proj": 2688, |
| "20.up_proj": 2688, |
| "20.down_proj": 2381, |
| "21.q_proj": 1497, |
| "21.k_proj": 593, |
| "21.v_proj": 601, |
| "21.o_proj": 1515, |
| "21.gate_proj": 2688, |
| "21.up_proj": 2688, |
| "21.down_proj": 2363, |
| "22.q_proj": 1482, |
| "22.k_proj": 583, |
| "22.v_proj": 594, |
| "22.o_proj": 1495, |
| "22.gate_proj": 2688, |
| "22.up_proj": 2688, |
| "22.down_proj": 2332, |
| "23.q_proj": 1483, |
| "23.k_proj": 584, |
| "23.v_proj": 593, |
| "23.o_proj": 1505, |
| "23.gate_proj": 2688, |
| "23.up_proj": 2688, |
| "23.down_proj": 2336, |
| "24.q_proj": 1482, |
| "24.k_proj": 584, |
| "24.v_proj": 594, |
| "24.o_proj": 1511, |
| "24.gate_proj": 2688, |
| "24.up_proj": 2688, |
| "24.down_proj": 2338, |
| "25.q_proj": 1486, |
| "25.k_proj": 586, |
| "25.v_proj": 597, |
| "25.o_proj": 1521, |
| "25.gate_proj": 2688, |
| "25.up_proj": 2688, |
| "25.down_proj": 2354, |
| "26.q_proj": 1493, |
| "26.k_proj": 589, |
| "26.v_proj": 602, |
| "26.o_proj": 1532, |
| "26.gate_proj": 2688, |
| "26.up_proj": 2688, |
| "26.down_proj": 2373, |
| "27.q_proj": 1498, |
| "27.k_proj": 592, |
| "27.v_proj": 606, |
| "27.o_proj": 1532, |
| "27.gate_proj": 2688, |
| "27.up_proj": 2688, |
| "27.down_proj": 2389, |
| "28.q_proj": 1518, |
| "28.k_proj": 603, |
| "28.v_proj": 618, |
| "28.o_proj": 1561, |
| "28.gate_proj": 2688, |
| "28.up_proj": 2688, |
| "28.down_proj": 2451, |
| "29.q_proj": 1530, |
| "29.k_proj": 609, |
| "29.v_proj": 626, |
| "29.o_proj": 1591, |
| "29.gate_proj": 2688, |
| "29.up_proj": 2688, |
| "29.down_proj": 2480, |
| "30.q_proj": 1553, |
| "30.k_proj": 621, |
| "30.v_proj": 642, |
| "30.o_proj": 1620, |
| "30.gate_proj": 2688, |
| "30.up_proj": 2688, |
| "30.down_proj": 2549, |
| "31.q_proj": 1684, |
| "31.k_proj": 692, |
| "31.v_proj": 716, |
| "31.o_proj": 1782, |
| "31.gate_proj": 2688, |
| "31.up_proj": 2688, |
| "31.down_proj": 2905 |
| }, |
| "rel_err": { |
| "0.q_proj": 0.08500257134437561, |
| "0.k_proj": 0.0716991052031517, |
| "0.v_proj": 0.22834858298301697, |
| "0.o_proj": 0.3019510805606842, |
| "0.gate_proj": 0.26146504282951355, |
| "0.up_proj": 0.2859381437301636, |
| "0.down_proj": 0.3900061547756195, |
| "1.q_proj": 0.13657206296920776, |
| "1.k_proj": 0.07040991634130478, |
| "1.v_proj": 0.34303975105285645, |
| "1.o_proj": 0.4063681960105896, |
| "1.gate_proj": 0.3133699595928192, |
| "1.up_proj": 0.33793723583221436, |
| "1.down_proj": 0.3351709246635437, |
| "2.q_proj": 0.35540199279785156, |
| "2.k_proj": 0.3083972930908203, |
| "2.v_proj": 0.4814577102661133, |
| "2.o_proj": 0.47539830207824707, |
| "2.gate_proj": 0.3777889311313629, |
| "2.up_proj": 0.39566245675086975, |
| "2.down_proj": 0.5091328620910645, |
| "3.q_proj": 0.3229297995567322, |
| "3.k_proj": 0.25059062242507935, |
| "3.v_proj": 0.5148245096206665, |
| "3.o_proj": 0.5196603536605835, |
| "3.gate_proj": 0.38781943917274475, |
| "3.up_proj": 0.4100722670555115, |
| "3.down_proj": 0.49922603368759155, |
| "4.q_proj": 0.3310073912143707, |
| "4.k_proj": 0.2755686044692993, |
| "4.v_proj": 0.4732750952243805, |
| "4.o_proj": 0.4666188955307007, |
| "4.gate_proj": 0.3899378180503845, |
| "4.up_proj": 0.4183727204799652, |
| "4.down_proj": 0.5054365992546082, |
| "5.q_proj": 0.3576917052268982, |
| "5.k_proj": 0.3083855211734772, |
| "5.v_proj": 0.5068833231925964, |
| "5.o_proj": 0.496389240026474, |
| "5.gate_proj": 0.3855798542499542, |
| "5.up_proj": 0.4201187789440155, |
| "5.down_proj": 0.4902642071247101, |
| "6.q_proj": 0.346874475479126, |
| "6.k_proj": 0.2829170227050781, |
| "6.v_proj": 0.4889771640300751, |
| "6.o_proj": 0.48441195487976074, |
| "6.gate_proj": 0.39520981907844543, |
| "6.up_proj": 0.4232458472251892, |
| "6.down_proj": 0.4810233414173126, |
| "7.q_proj": 0.33142852783203125, |
| "7.k_proj": 0.2655273973941803, |
| "7.v_proj": 0.4808761775493622, |
| "7.o_proj": 0.4556846618652344, |
| "7.gate_proj": 0.387683242559433, |
| "7.up_proj": 0.4178556799888611, |
| "7.down_proj": 0.4722205698490143, |
| "8.q_proj": 0.33769723773002625, |
| "8.k_proj": 0.2662927508354187, |
| "8.v_proj": 0.4425075054168701, |
| "8.o_proj": 0.44497787952423096, |
| "8.gate_proj": 0.3896368443965912, |
| "8.up_proj": 0.41572481393814087, |
| "8.down_proj": 0.4673706591129303, |
| "9.q_proj": 0.3340452313423157, |
| "9.k_proj": 0.2718675434589386, |
| "9.v_proj": 0.4268815815448761, |
| "9.o_proj": 0.44764506816864014, |
| "9.gate_proj": 0.38748565316200256, |
| "9.up_proj": 0.4152057468891144, |
| "9.down_proj": 0.4528351426124573, |
| "10.q_proj": 0.3335397243499756, |
| "10.k_proj": 0.2776794731616974, |
| "10.v_proj": 0.4279860556125641, |
| "10.o_proj": 0.44602537155151367, |
| "10.gate_proj": 0.38787901401519775, |
| "10.up_proj": 0.4141860604286194, |
| "10.down_proj": 0.46012479066848755, |
| "11.q_proj": 0.3297421634197235, |
| "11.k_proj": 0.26538947224617004, |
| "11.v_proj": 0.3986572325229645, |
| "11.o_proj": 0.4476287066936493, |
| "11.gate_proj": 0.38680461049079895, |
| "11.up_proj": 0.41310203075408936, |
| "11.down_proj": 0.45812714099884033, |
| "12.q_proj": 0.319181889295578, |
| "12.k_proj": 0.24617546796798706, |
| "12.v_proj": 0.4383234977722168, |
| "12.o_proj": 0.42985114455223083, |
| "12.gate_proj": 0.3842836320400238, |
| "12.up_proj": 0.4086284637451172, |
| "12.down_proj": 0.45287853479385376, |
| "13.q_proj": 0.342117041349411, |
| "13.k_proj": 0.28727155923843384, |
| "13.v_proj": 0.4403918981552124, |
| "13.o_proj": 0.43300414085388184, |
| "13.gate_proj": 0.3891292214393616, |
| "13.up_proj": 0.4081304669380188, |
| "13.down_proj": 0.4504336416721344, |
| "14.q_proj": 0.31687843799591064, |
| "14.k_proj": 0.26946091651916504, |
| "14.v_proj": 0.44405218958854675, |
| "14.o_proj": 0.43850401043891907, |
| "14.gate_proj": 0.38567572832107544, |
| "14.up_proj": 0.4120667278766632, |
| "14.down_proj": 0.4714415967464447, |
| "15.q_proj": 0.31565016508102417, |
| "15.k_proj": 0.2453945130109787, |
| "15.v_proj": 0.4641188681125641, |
| "15.o_proj": 0.4399622678756714, |
| "15.gate_proj": 0.38257908821105957, |
| "15.up_proj": 0.408530592918396, |
| "15.down_proj": 0.4638931155204773, |
| "16.q_proj": 0.32283511757850647, |
| "16.k_proj": 0.2652204930782318, |
| "16.v_proj": 0.464958131313324, |
| "16.o_proj": 0.42660167813301086, |
| "16.gate_proj": 0.3851650655269623, |
| "16.up_proj": 0.41526761651039124, |
| "16.down_proj": 0.46140268445014954, |
| "17.q_proj": 0.3427833616733551, |
| "17.k_proj": 0.2890065610408783, |
| "17.v_proj": 0.4380885064601898, |
| "17.o_proj": 0.44451603293418884, |
| "17.gate_proj": 0.3959171772003174, |
| "17.up_proj": 0.4251500368118286, |
| "17.down_proj": 0.4505082368850708, |
| "18.q_proj": 0.3227398097515106, |
| "18.k_proj": 0.2645529508590698, |
| "18.v_proj": 0.486581027507782, |
| "18.o_proj": 0.4234537184238434, |
| "18.gate_proj": 0.39415696263313293, |
| "18.up_proj": 0.42284369468688965, |
| "18.down_proj": 0.4652669429779053, |
| "19.q_proj": 0.36690786480903625, |
| "19.k_proj": 0.27805012464523315, |
| "19.v_proj": 0.5099502205848694, |
| "19.o_proj": 0.4442979693412781, |
| "19.gate_proj": 0.39896416664123535, |
| "19.up_proj": 0.42378658056259155, |
| "19.down_proj": 0.48121050000190735, |
| "20.q_proj": 0.38949716091156006, |
| "20.k_proj": 0.32147786021232605, |
| "20.v_proj": 0.4748760461807251, |
| "20.o_proj": 0.46483227610588074, |
| "20.gate_proj": 0.4035407602787018, |
| "20.up_proj": 0.423808217048645, |
| "20.down_proj": 0.49269551038742065, |
| "21.q_proj": 0.41730770468711853, |
| "21.k_proj": 0.3455001711845398, |
| "21.v_proj": 0.45047393441200256, |
| "21.o_proj": 0.5031458139419556, |
| "21.gate_proj": 0.4041469395160675, |
| "21.up_proj": 0.4253368079662323, |
| "21.down_proj": 0.4889674484729767, |
| "22.q_proj": 0.4534510374069214, |
| "22.k_proj": 0.37348392605781555, |
| "22.v_proj": 0.4685239791870117, |
| "22.o_proj": 0.5248842835426331, |
| "22.gate_proj": 0.4087425768375397, |
| "22.up_proj": 0.42668619751930237, |
| "22.down_proj": 0.4892253875732422, |
| "23.q_proj": 0.4440714418888092, |
| "23.k_proj": 0.3682272434234619, |
| "23.v_proj": 0.49346232414245605, |
| "23.o_proj": 0.502592921257019, |
| "23.gate_proj": 0.4109232425689697, |
| "23.up_proj": 0.42807736992836, |
| "23.down_proj": 0.48296502232551575, |
| "24.q_proj": 0.4487209916114807, |
| "24.k_proj": 0.38935786485671997, |
| "24.v_proj": 0.5036438703536987, |
| "24.o_proj": 0.5222789645195007, |
| "24.gate_proj": 0.4119880795478821, |
| "24.up_proj": 0.42954087257385254, |
| "24.down_proj": 0.47408875823020935, |
| "25.q_proj": 0.4551296532154083, |
| "25.k_proj": 0.37871479988098145, |
| "25.v_proj": 0.4806521534919739, |
| "25.o_proj": 0.5046061873435974, |
| "25.gate_proj": 0.4138759672641754, |
| "25.up_proj": 0.4307102560997009, |
| "25.down_proj": 0.463013231754303, |
| "26.q_proj": 0.44990450143814087, |
| "26.k_proj": 0.3649565577507019, |
| "26.v_proj": 0.46789830923080444, |
| "26.o_proj": 0.4847952425479889, |
| "26.gate_proj": 0.414262056350708, |
| "26.up_proj": 0.43036970496177673, |
| "26.down_proj": 0.4495799243450165, |
| "27.q_proj": 0.4583161473274231, |
| "27.k_proj": 0.39734572172164917, |
| "27.v_proj": 0.4797323942184448, |
| "27.o_proj": 0.47943976521492004, |
| "27.gate_proj": 0.41736742854118347, |
| "27.up_proj": 0.4318309724330902, |
| "27.down_proj": 0.44924622774124146, |
| "28.q_proj": 0.41977617144584656, |
| "28.k_proj": 0.3314531147480011, |
| "28.v_proj": 0.4329354465007782, |
| "28.o_proj": 0.4519560635089874, |
| "28.gate_proj": 0.41815441846847534, |
| "28.up_proj": 0.4296439588069916, |
| "28.down_proj": 0.4507562816143036, |
| "29.q_proj": 0.4013705253601074, |
| "29.k_proj": 0.3302054703235626, |
| "29.v_proj": 0.45994922518730164, |
| "29.o_proj": 0.4338970482349396, |
| "29.gate_proj": 0.4139324426651001, |
| "29.up_proj": 0.4252754747867584, |
| "29.down_proj": 0.4538421928882599, |
| "30.q_proj": 0.3819069266319275, |
| "30.k_proj": 0.300334095954895, |
| "30.v_proj": 0.3898763358592987, |
| "30.o_proj": 0.4189046621322632, |
| "30.gate_proj": 0.4136664569377899, |
| "30.up_proj": 0.4180448055267334, |
| "30.down_proj": 0.43752264976501465, |
| "31.q_proj": 0.3336416184902191, |
| "31.k_proj": 0.2430993765592575, |
| "31.v_proj": 0.32917752861976624, |
| "31.o_proj": 0.3434261381626129, |
| "31.gate_proj": 0.4216300845146179, |
| "31.up_proj": 0.40061134099960327, |
| "31.down_proj": 0.3710506558418274 |
| }, |
| "step": "swift_svd_dynamic_finetuned", |
| "compression": { |
| "upstream": "hiahei/Swift-SVD", |
| "step": "swift_svd_dynamic", |
| "alloc": "dynamic", |
| "alpha": 0.6, |
| "nsamples": 256, |
| "seqlen": 2048, |
| "dataset": "WikiText2", |
| "seed": 42, |
| "gqa_budget": "shape", |
| "param_fraction": 0.8004338191105769, |
| "mean_rel_err": 0.3994709092990628, |
| "max_rel_err": 0.5239740610122681 |
| }, |
| "factorisation": { |
| "max_rel_err": 0.0013197725638747215, |
| "factor_params": 5586485248, |
| "dense_params": 6979321856 |
| }, |
| "finetune": { |
| "protocol": "svdllm_stage2_u_then_v", |
| "script": "gitclones/SVD-LLM/utils/LoRA.py", |
| "data": "alpaca-cleaned", |
| "epochs_per_half": 2, |
| "learning_rate": 0.0001, |
| "batch_size": 64, |
| "micro_batch_size": 4, |
| "lora_r": 8, |
| "lora_alpha": 16, |
| "lora_dropout": 0.05, |
| "cutoff_len": 256, |
| "val_set_size": 2000, |
| "max_train_samples": "all", |
| "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)" |
| } |
| } |
