| { |
| "source": "/workspace/svd_safety/assets/models/Llama-2-7b-chat-hf", |
| "upstream": "hiahei/Swift-SVD (compression) + AIoT-MLSys-Lab/SVD-LLM utils/LoRA.py (recovery)", |
| "stage": "finetuned", |
| "ratio_removed": 0.5, |
| "internal_ratio_kept": 0.5, |
| "from_pt": "/workspace/svd_safety/results/swift_ft/cells/l2_swift_remove50/finetune_v/merge.pt", |
| "mean_rel_err": 0.5770199772502694, |
| "max_rel_err": 0.6914882659912109, |
| "param_fraction": 0.4998633824481865, |
| "ranks": { |
| "0.q_proj": 1212, |
| "0.k_proj": 1202, |
| "0.v_proj": 1147, |
| "0.o_proj": 1160, |
| "0.gate_proj": 1526, |
| "0.up_proj": 1527, |
| "0.down_proj": 1707, |
| "1.q_proj": 1121, |
| "1.k_proj": 1118, |
| "1.v_proj": 1078, |
| "1.o_proj": 1075, |
| "1.gate_proj": 1509, |
| "1.up_proj": 1509, |
| "1.down_proj": 1775, |
| "2.q_proj": 1022, |
| "2.k_proj": 1022, |
| "2.v_proj": 1006, |
| "2.o_proj": 983, |
| "2.gate_proj": 1487, |
| "2.up_proj": 1487, |
| "2.down_proj": 1404, |
| "3.q_proj": 1029, |
| "3.k_proj": 1029, |
| "3.v_proj": 1022, |
| "3.o_proj": 998, |
| "3.gate_proj": 1490, |
| "3.up_proj": 1490, |
| "3.down_proj": 1433, |
| "4.q_proj": 1052, |
| "4.k_proj": 1053, |
| "4.v_proj": 1044, |
| "4.o_proj": 1026, |
| "4.gate_proj": 1496, |
| "4.up_proj": 1496, |
| "4.down_proj": 1492, |
| "5.q_proj": 1031, |
| "5.k_proj": 1032, |
| "5.v_proj": 1024, |
| "5.o_proj": 1011, |
| "5.gate_proj": 1492, |
| "5.up_proj": 1492, |
| "5.down_proj": 1461, |
| "6.q_proj": 1047, |
| "6.k_proj": 1048, |
| "6.v_proj": 1045, |
| "6.o_proj": 1033, |
| "6.gate_proj": 1497, |
| "6.up_proj": 1496, |
| "6.down_proj": 1500, |
| "7.q_proj": 1025, |
| "7.k_proj": 1025, |
| "7.v_proj": 1023, |
| "7.o_proj": 1013, |
| "7.gate_proj": 1492, |
| "7.up_proj": 1491, |
| "7.down_proj": 1465, |
| "8.q_proj": 1033, |
| "8.k_proj": 1034, |
| "8.v_proj": 1033, |
| "8.o_proj": 1026, |
| "8.gate_proj": 1494, |
| "8.up_proj": 1494, |
| "8.down_proj": 1486, |
| "9.q_proj": 1017, |
| "9.k_proj": 1018, |
| "9.v_proj": 1017, |
| "9.o_proj": 1014, |
| "9.gate_proj": 1491, |
| "9.up_proj": 1490, |
| "9.down_proj": 1459, |
| "10.q_proj": 1009, |
| "10.k_proj": 1010, |
| "10.v_proj": 1009, |
| "10.o_proj": 1008, |
| "10.gate_proj": 1489, |
| "10.up_proj": 1489, |
| "10.down_proj": 1445, |
| "11.q_proj": 1008, |
| "11.k_proj": 1009, |
| "11.v_proj": 1011, |
| "11.o_proj": 1007, |
| "11.gate_proj": 1489, |
| "11.up_proj": 1488, |
| "11.down_proj": 1445, |
| "12.q_proj": 1004, |
| "12.k_proj": 1004, |
| "12.v_proj": 1006, |
| "12.o_proj": 1004, |
| "12.gate_proj": 1487, |
| "12.up_proj": 1487, |
| "12.down_proj": 1441, |
| "13.q_proj": 1007, |
| "13.k_proj": 1008, |
| "13.v_proj": 1010, |
| "13.o_proj": 1009, |
| "13.gate_proj": 1489, |
| "13.up_proj": 1489, |
| "13.down_proj": 1450, |
| "14.q_proj": 1004, |
| "14.k_proj": 1005, |
| "14.v_proj": 1006, |
| "14.o_proj": 1007, |
| "14.gate_proj": 1487, |
| "14.up_proj": 1487, |
| "14.down_proj": 1446, |
| "15.q_proj": 1005, |
| "15.k_proj": 1006, |
| "15.v_proj": 1007, |
| "15.o_proj": 1010, |
| "15.gate_proj": 1488, |
| "15.up_proj": 1487, |
| "15.down_proj": 1452, |
| "16.q_proj": 1005, |
| "16.k_proj": 1006, |
| "16.v_proj": 1008, |
| "16.o_proj": 1014, |
| "16.gate_proj": 1489, |
| "16.up_proj": 1489, |
| "16.down_proj": 1459, |
| "17.q_proj": 1002, |
| "17.k_proj": 1003, |
| "17.v_proj": 1006, |
| "17.o_proj": 1007, |
| "17.gate_proj": 1488, |
| "17.up_proj": 1487, |
| "17.down_proj": 1456, |
| "18.q_proj": 1000, |
| "18.k_proj": 1000, |
| "18.v_proj": 1005, |
| "18.o_proj": 1005, |
| "18.gate_proj": 1487, |
| "18.up_proj": 1487, |
| "18.down_proj": 1456, |
| "19.q_proj": 994, |
| "19.k_proj": 995, |
| "19.v_proj": 1000, |
| "19.o_proj": 1000, |
| "19.gate_proj": 1486, |
| "19.up_proj": 1486, |
| "19.down_proj": 1447, |
| "20.q_proj": 994, |
| "20.k_proj": 994, |
| "20.v_proj": 999, |
| "20.o_proj": 1003, |
| "20.gate_proj": 1486, |
| "20.up_proj": 1486, |
| "20.down_proj": 1450, |
| "21.q_proj": 991, |
| "21.k_proj": 991, |
| "21.v_proj": 998, |
| "21.o_proj": 997, |
| "21.gate_proj": 1485, |
| "21.up_proj": 1485, |
| "21.down_proj": 1446, |
| "22.q_proj": 991, |
| "22.k_proj": 991, |
| "22.v_proj": 998, |
| "22.o_proj": 1002, |
| "22.gate_proj": 1485, |
| "22.up_proj": 1485, |
| "22.down_proj": 1447, |
| "23.q_proj": 989, |
| "23.k_proj": 989, |
| "23.v_proj": 999, |
| "23.o_proj": 999, |
| "23.gate_proj": 1485, |
| "23.up_proj": 1485, |
| "23.down_proj": 1446, |
| "24.q_proj": 988, |
| "24.k_proj": 989, |
| "24.v_proj": 997, |
| "24.o_proj": 1001, |
| "24.gate_proj": 1485, |
| "24.up_proj": 1485, |
| "24.down_proj": 1445, |
| "25.q_proj": 989, |
| "25.k_proj": 988, |
| "25.v_proj": 1000, |
| "25.o_proj": 1006, |
| "25.gate_proj": 1485, |
| "25.up_proj": 1485, |
| "25.down_proj": 1449, |
| "26.q_proj": 989, |
| "26.k_proj": 989, |
| "26.v_proj": 1000, |
| "26.o_proj": 1007, |
| "26.gate_proj": 1485, |
| "26.up_proj": 1486, |
| "26.down_proj": 1451, |
| "27.q_proj": 990, |
| "27.k_proj": 990, |
| "27.v_proj": 1001, |
| "27.o_proj": 1006, |
| "27.gate_proj": 1486, |
| "27.up_proj": 1486, |
| "27.down_proj": 1457, |
| "28.q_proj": 991, |
| "28.k_proj": 991, |
| "28.v_proj": 1003, |
| "28.o_proj": 1013, |
| "28.gate_proj": 1486, |
| "28.up_proj": 1487, |
| "28.down_proj": 1462, |
| "29.q_proj": 992, |
| "29.k_proj": 992, |
| "29.v_proj": 1003, |
| "29.o_proj": 1017, |
| "29.gate_proj": 1487, |
| "29.up_proj": 1487, |
| "29.down_proj": 1469, |
| "30.q_proj": 1003, |
| "30.k_proj": 1002, |
| "30.v_proj": 1016, |
| "30.o_proj": 1026, |
| "30.gate_proj": 1491, |
| "30.up_proj": 1492, |
| "30.down_proj": 1525, |
| "31.q_proj": 1234, |
| "31.k_proj": 1235, |
| "31.v_proj": 1247, |
| "31.o_proj": 1281, |
| "31.gate_proj": 1545, |
| "31.up_proj": 1547, |
| "31.down_proj": 2018 |
| }, |
| "rel_err": { |
| "0.q_proj": 0.1519796997308731, |
| "0.k_proj": 0.1192033439874649, |
| "0.v_proj": 0.38923344016075134, |
| "0.o_proj": 0.3858165740966797, |
| "0.gate_proj": 0.500773549079895, |
| "0.up_proj": 0.5168083906173706, |
| "0.down_proj": 0.5532068014144897, |
| "1.q_proj": 0.16660481691360474, |
| "1.k_proj": 0.1496836543083191, |
| "1.v_proj": 0.5328048467636108, |
| "1.o_proj": 0.4690547585487366, |
| "1.gate_proj": 0.48617124557495117, |
| "1.up_proj": 0.5469284653663635, |
| "1.down_proj": 0.5976937413215637, |
| "2.q_proj": 0.35211285948753357, |
| "2.k_proj": 0.3386683166027069, |
| "2.v_proj": 0.5837520956993103, |
| "2.o_proj": 0.6466013193130493, |
| "2.gate_proj": 0.556005597114563, |
| "2.up_proj": 0.5962874293327332, |
| "2.down_proj": 0.6686410903930664, |
| "3.q_proj": 0.45740288496017456, |
| "3.k_proj": 0.45834606885910034, |
| "3.v_proj": 0.5986074209213257, |
| "3.o_proj": 0.6326659917831421, |
| "3.gate_proj": 0.613806426525116, |
| "3.up_proj": 0.6380705833435059, |
| "3.down_proj": 0.6272472739219666, |
| "4.q_proj": 0.4375986158847809, |
| "4.k_proj": 0.4429137408733368, |
| "4.v_proj": 0.5889207720756531, |
| "4.o_proj": 0.6280702352523804, |
| "4.gate_proj": 0.6121411919593811, |
| "4.up_proj": 0.6420316100120544, |
| "4.down_proj": 0.6181107759475708, |
| "5.q_proj": 0.4545028805732727, |
| "5.k_proj": 0.45424583554267883, |
| "5.v_proj": 0.600082278251648, |
| "5.o_proj": 0.636687159538269, |
| "5.gate_proj": 0.6082773804664612, |
| "5.up_proj": 0.6440009474754333, |
| "5.down_proj": 0.6173216700553894, |
| "6.q_proj": 0.4331511855125427, |
| "6.k_proj": 0.44137337803840637, |
| "6.v_proj": 0.5765433311462402, |
| "6.o_proj": 0.599454939365387, |
| "6.gate_proj": 0.6015103459358215, |
| "6.up_proj": 0.6436546444892883, |
| "6.down_proj": 0.6076619625091553, |
| "7.q_proj": 0.4461621642112732, |
| "7.k_proj": 0.4536467492580414, |
| "7.v_proj": 0.5764104127883911, |
| "7.o_proj": 0.6230320930480957, |
| "7.gate_proj": 0.599027693271637, |
| "7.up_proj": 0.6438522934913635, |
| "7.down_proj": 0.6086878180503845, |
| "8.q_proj": 0.45493605732917786, |
| "8.k_proj": 0.4706614911556244, |
| "8.v_proj": 0.5578523874282837, |
| "8.o_proj": 0.6163948178291321, |
| "8.gate_proj": 0.5954751968383789, |
| "8.up_proj": 0.6374110579490662, |
| "8.down_proj": 0.5979208946228027, |
| "9.q_proj": 0.4763244092464447, |
| "9.k_proj": 0.48309847712516785, |
| "9.v_proj": 0.5613107085227966, |
| "9.o_proj": 0.6197980642318726, |
| "9.gate_proj": 0.5919429659843445, |
| "9.up_proj": 0.632355809211731, |
| "9.down_proj": 0.6026204228401184, |
| "10.q_proj": 0.44654256105422974, |
| "10.k_proj": 0.459806352853775, |
| "10.v_proj": 0.5551357865333557, |
| "10.o_proj": 0.5909467339515686, |
| "10.gate_proj": 0.5873189568519592, |
| "10.up_proj": 0.6278164982795715, |
| "10.down_proj": 0.6019299626350403, |
| "11.q_proj": 0.4331582188606262, |
| "11.k_proj": 0.44750073552131653, |
| "11.v_proj": 0.539775550365448, |
| "11.o_proj": 0.5791270136833191, |
| "11.gate_proj": 0.5880950689315796, |
| "11.up_proj": 0.62898188829422, |
| "11.down_proj": 0.6085322499275208, |
| "12.q_proj": 0.47685113549232483, |
| "12.k_proj": 0.47473812103271484, |
| "12.v_proj": 0.5630918741226196, |
| "12.o_proj": 0.62078458070755, |
| "12.gate_proj": 0.5933976769447327, |
| "12.up_proj": 0.6293315887451172, |
| "12.down_proj": 0.6097376346588135, |
| "13.q_proj": 0.4816206395626068, |
| "13.k_proj": 0.5003007650375366, |
| "13.v_proj": 0.5929780602455139, |
| "13.o_proj": 0.60856032371521, |
| "13.gate_proj": 0.5962281823158264, |
| "13.up_proj": 0.6323710680007935, |
| "13.down_proj": 0.6194044947624207, |
| "14.q_proj": 0.45993104577064514, |
| "14.k_proj": 0.4645041525363922, |
| "14.v_proj": 0.5858851671218872, |
| "14.o_proj": 0.6054980754852295, |
| "14.gate_proj": 0.5965513586997986, |
| "14.up_proj": 0.6316479444503784, |
| "14.down_proj": 0.6229026913642883, |
| "15.q_proj": 0.4835875630378723, |
| "15.k_proj": 0.48482608795166016, |
| "15.v_proj": 0.59104323387146, |
| "15.o_proj": 0.6179000735282898, |
| "15.gate_proj": 0.6003255248069763, |
| "15.up_proj": 0.6353552341461182, |
| "15.down_proj": 0.63531893491745, |
| "16.q_proj": 0.4797825813293457, |
| "16.k_proj": 0.48336681723594666, |
| "16.v_proj": 0.620274007320404, |
| "16.o_proj": 0.6427569389343262, |
| "16.gate_proj": 0.5996921062469482, |
| "16.up_proj": 0.632495105266571, |
| "16.down_proj": 0.6475937366485596, |
| "17.q_proj": 0.5249422788619995, |
| "17.k_proj": 0.5217642784118652, |
| "17.v_proj": 0.635169506072998, |
| "17.o_proj": 0.6561335921287537, |
| "17.gate_proj": 0.6028484106063843, |
| "17.up_proj": 0.6339511871337891, |
| "17.down_proj": 0.6465868353843689, |
| "18.q_proj": 0.5513262748718262, |
| "18.k_proj": 0.5563322901725769, |
| "18.v_proj": 0.6416341066360474, |
| "18.o_proj": 0.6792891025543213, |
| "18.gate_proj": 0.6122872233390808, |
| "18.up_proj": 0.6402170658111572, |
| "18.down_proj": 0.6462416052818298, |
| "19.q_proj": 0.5357663035392761, |
| "19.k_proj": 0.530946671962738, |
| "19.v_proj": 0.6263485550880432, |
| "19.o_proj": 0.6681627631187439, |
| "19.gate_proj": 0.6159399151802063, |
| "19.up_proj": 0.6408361196517944, |
| "19.down_proj": 0.6481027603149414, |
| "20.q_proj": 0.5267692804336548, |
| "20.k_proj": 0.5284019708633423, |
| "20.v_proj": 0.6414449214935303, |
| "20.o_proj": 0.6711487174034119, |
| "20.gate_proj": 0.6140052080154419, |
| "20.up_proj": 0.6376996636390686, |
| "20.down_proj": 0.6590219140052795, |
| "21.q_proj": 0.5438850522041321, |
| "21.k_proj": 0.5463284850120544, |
| "21.v_proj": 0.6339064836502075, |
| "21.o_proj": 0.6861968040466309, |
| "21.gate_proj": 0.61533522605896, |
| "21.up_proj": 0.6377876400947571, |
| "21.down_proj": 0.65590900182724, |
| "22.q_proj": 0.5565619468688965, |
| "22.k_proj": 0.5612902045249939, |
| "22.v_proj": 0.6452638506889343, |
| "22.o_proj": 0.6826830506324768, |
| "22.gate_proj": 0.6154788732528687, |
| "22.up_proj": 0.6370535492897034, |
| "22.down_proj": 0.6531917452812195, |
| "23.q_proj": 0.5894484519958496, |
| "23.k_proj": 0.5889238715171814, |
| "23.v_proj": 0.6437049508094788, |
| "23.o_proj": 0.6743727326393127, |
| "23.gate_proj": 0.6192212700843811, |
| "23.up_proj": 0.6390761137008667, |
| "23.down_proj": 0.6452059745788574, |
| "24.q_proj": 0.5363700985908508, |
| "24.k_proj": 0.5404852628707886, |
| "24.v_proj": 0.6358501315116882, |
| "24.o_proj": 0.6789605617523193, |
| "24.gate_proj": 0.6206092238426208, |
| "24.up_proj": 0.6403064727783203, |
| "24.down_proj": 0.6378594636917114, |
| "25.q_proj": 0.578160285949707, |
| "25.k_proj": 0.5866066217422485, |
| "25.v_proj": 0.6443796753883362, |
| "25.o_proj": 0.6698061227798462, |
| "25.gate_proj": 0.6242399215698242, |
| "25.up_proj": 0.6427105069160461, |
| "25.down_proj": 0.6289896965026855, |
| "26.q_proj": 0.5467449426651001, |
| "26.k_proj": 0.5561713576316833, |
| "26.v_proj": 0.6325230002403259, |
| "26.o_proj": 0.6914882659912109, |
| "26.gate_proj": 0.6260568499565125, |
| "26.up_proj": 0.6445198059082031, |
| "26.down_proj": 0.6264070868492126, |
| "27.q_proj": 0.5962280631065369, |
| "27.k_proj": 0.6041812300682068, |
| "27.v_proj": 0.6541686058044434, |
| "27.o_proj": 0.6721190810203552, |
| "27.gate_proj": 0.6289679408073425, |
| "27.up_proj": 0.6461732983589172, |
| "27.down_proj": 0.622887909412384, |
| "28.q_proj": 0.577075719833374, |
| "28.k_proj": 0.5881885290145874, |
| "28.v_proj": 0.649710476398468, |
| "28.o_proj": 0.6659877896308899, |
| "28.gate_proj": 0.6287640333175659, |
| "28.up_proj": 0.6446463465690613, |
| "28.down_proj": 0.6314065456390381, |
| "29.q_proj": 0.5415160059928894, |
| "29.k_proj": 0.5502261519432068, |
| "29.v_proj": 0.6414359211921692, |
| "29.o_proj": 0.6688233613967896, |
| "29.gate_proj": 0.6273680329322815, |
| "29.up_proj": 0.6402621865272522, |
| "29.down_proj": 0.6377818584442139, |
| "30.q_proj": 0.5716490745544434, |
| "30.k_proj": 0.5806272625923157, |
| "30.v_proj": 0.650600016117096, |
| "30.o_proj": 0.6727860569953918, |
| "30.gate_proj": 0.6200348138809204, |
| "30.up_proj": 0.6269094944000244, |
| "30.down_proj": 0.635309100151062, |
| "31.q_proj": 0.4421253800392151, |
| "31.k_proj": 0.4497847557067871, |
| "31.v_proj": 0.5590528845787048, |
| "31.o_proj": 0.5637112855911255, |
| "31.gate_proj": 0.5982303023338318, |
| "31.up_proj": 0.600795328617096, |
| "31.down_proj": 0.5228932499885559 |
| }, |
| "step": "swift_svd_dynamic_finetuned", |
| "compression": { |
| "upstream": "hiahei/Swift-SVD", |
| "step": "swift_svd_dynamic", |
| "alloc": "dynamic", |
| "alpha": 0.6, |
| "nsamples": 256, |
| "seqlen": 2048, |
| "dataset": "WikiText2", |
| "seed": 42, |
| "gqa_budget": null, |
| "param_fraction": 0.4998633824481865, |
| "mean_rel_err": 0.5765954015909561, |
| "max_rel_err": 0.691561222076416 |
| }, |
| "factorisation": { |
| "max_rel_err": 0.0014744423096999526, |
| "factor_params": 3237117952, |
| "dense_params": 6476005376 |
| }, |
| "finetune": { |
| "protocol": "svdllm_stage2_u_then_v", |
| "script": "gitclones/SVD-LLM/utils/LoRA.py", |
| "data": "alpaca-cleaned", |
| "epochs_per_half": 2, |
| "learning_rate": 0.0001, |
| "batch_size": 64, |
| "micro_batch_size": 4, |
| "lora_r": 8, |
| "lora_alpha": 16, |
| "lora_dropout": 0.05, |
| "cutoff_len": 256, |
| "val_set_size": 2000, |
| "max_train_samples": "all", |
| "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)" |
| } |
| } |
