| { |
| "source": "/workspace/svd_safety/assets/models/Mistral-7B-Instruct-v0.2", |
| "upstream": "hiahei/Swift-SVD (compression) + AIoT-MLSys-Lab/SVD-LLM utils/LoRA.py (recovery)", |
| "stage": "finetuned", |
| "ratio_removed": 0.4, |
| "internal_ratio_kept": 0.6, |
| "from_pt": "/workspace/svd_safety/results/swift_ft/cells/mis7_swift_remove40/finetune_v/merge.pt", |
| "mean_rel_err": 0.524213519213455, |
| "max_rel_err": 0.6444101929664612, |
| "param_fraction": 0.6003230168269231, |
| "ranks": { |
| "0.q_proj": 1383, |
| "0.k_proj": 581, |
| "0.v_proj": 529, |
| "0.o_proj": 1263, |
| "0.gate_proj": 2016, |
| "0.up_proj": 2016, |
| "0.down_proj": 2043, |
| "1.q_proj": 1388, |
| "1.k_proj": 591, |
| "1.v_proj": 557, |
| "1.o_proj": 1300, |
| "1.gate_proj": 2016, |
| "1.up_proj": 2016, |
| "1.down_proj": 2371, |
| "2.q_proj": 1131, |
| "2.k_proj": 448, |
| "2.v_proj": 439, |
| "2.o_proj": 1088, |
| "2.gate_proj": 2016, |
| "2.up_proj": 2016, |
| "2.down_proj": 1622, |
| "3.q_proj": 1128, |
| "3.k_proj": 446, |
| "3.v_proj": 435, |
| "3.o_proj": 1077, |
| "3.gate_proj": 2016, |
| "3.up_proj": 2016, |
| "3.down_proj": 1624, |
| "4.q_proj": 1126, |
| "4.k_proj": 445, |
| "4.v_proj": 439, |
| "4.o_proj": 1084, |
| "4.gate_proj": 2016, |
| "4.up_proj": 2016, |
| "4.down_proj": 1645, |
| "5.q_proj": 1126, |
| "5.k_proj": 445, |
| "5.v_proj": 439, |
| "5.o_proj": 1092, |
| "5.gate_proj": 2016, |
| "5.up_proj": 2016, |
| "5.down_proj": 1662, |
| "6.q_proj": 1127, |
| "6.k_proj": 446, |
| "6.v_proj": 440, |
| "6.o_proj": 1101, |
| "6.gate_proj": 2016, |
| "6.up_proj": 2016, |
| "6.down_proj": 1675, |
| "7.q_proj": 1126, |
| "7.k_proj": 446, |
| "7.v_proj": 441, |
| "7.o_proj": 1110, |
| "7.gate_proj": 2016, |
| "7.up_proj": 2016, |
| "7.down_proj": 1688, |
| "8.q_proj": 1122, |
| "8.k_proj": 444, |
| "8.v_proj": 441, |
| "8.o_proj": 1109, |
| "8.gate_proj": 2016, |
| "8.up_proj": 2016, |
| "8.down_proj": 1685, |
| "9.q_proj": 1121, |
| "9.k_proj": 444, |
| "9.v_proj": 442, |
| "9.o_proj": 1109, |
| "9.gate_proj": 2016, |
| "9.up_proj": 2016, |
| "9.down_proj": 1692, |
| "10.q_proj": 1122, |
| "10.k_proj": 444, |
| "10.v_proj": 442, |
| "10.o_proj": 1118, |
| "10.gate_proj": 2016, |
| "10.up_proj": 2016, |
| "10.down_proj": 1692, |
| "11.q_proj": 1120, |
| "11.k_proj": 444, |
| "11.v_proj": 443, |
| "11.o_proj": 1121, |
| "11.gate_proj": 2016, |
| "11.up_proj": 2016, |
| "11.down_proj": 1695, |
| "12.q_proj": 1127, |
| "12.k_proj": 447, |
| "12.v_proj": 447, |
| "12.o_proj": 1129, |
| "12.gate_proj": 2016, |
| "12.up_proj": 2016, |
| "12.down_proj": 1714, |
| "13.q_proj": 1123, |
| "13.k_proj": 446, |
| "13.v_proj": 445, |
| "13.o_proj": 1127, |
| "13.gate_proj": 2016, |
| "13.up_proj": 2016, |
| "13.down_proj": 1714, |
| "14.q_proj": 1132, |
| "14.k_proj": 451, |
| "14.v_proj": 453, |
| "14.o_proj": 1139, |
| "14.gate_proj": 2016, |
| "14.up_proj": 2016, |
| "14.down_proj": 1742, |
| "15.q_proj": 1145, |
| "15.k_proj": 458, |
| "15.v_proj": 461, |
| "15.o_proj": 1157, |
| "15.gate_proj": 2016, |
| "15.up_proj": 2016, |
| "15.down_proj": 1777, |
| "16.q_proj": 1151, |
| "16.k_proj": 461, |
| "16.v_proj": 463, |
| "16.o_proj": 1162, |
| "16.gate_proj": 2016, |
| "16.up_proj": 2016, |
| "16.down_proj": 1800, |
| "17.q_proj": 1174, |
| "17.k_proj": 473, |
| "17.v_proj": 476, |
| "17.o_proj": 1188, |
| "17.gate_proj": 2016, |
| "17.up_proj": 2016, |
| "17.down_proj": 1852, |
| "18.q_proj": 1174, |
| "18.k_proj": 472, |
| "18.v_proj": 476, |
| "18.o_proj": 1191, |
| "18.gate_proj": 2016, |
| "18.up_proj": 2016, |
| "18.down_proj": 1866, |
| "19.q_proj": 1158, |
| "19.k_proj": 465, |
| "19.v_proj": 469, |
| "19.o_proj": 1179, |
| "19.gate_proj": 2016, |
| "19.up_proj": 2016, |
| "19.down_proj": 1844, |
| "20.q_proj": 1130, |
| "20.k_proj": 449, |
| "20.v_proj": 454, |
| "20.o_proj": 1146, |
| "20.gate_proj": 2016, |
| "20.up_proj": 2016, |
| "20.down_proj": 1786, |
| "21.q_proj": 1123, |
| "21.k_proj": 444, |
| "21.v_proj": 451, |
| "21.o_proj": 1137, |
| "21.gate_proj": 2016, |
| "21.up_proj": 2016, |
| "21.down_proj": 1772, |
| "22.q_proj": 1112, |
| "22.k_proj": 437, |
| "22.v_proj": 445, |
| "22.o_proj": 1121, |
| "22.gate_proj": 2016, |
| "22.up_proj": 2016, |
| "22.down_proj": 1749, |
| "23.q_proj": 1112, |
| "23.k_proj": 438, |
| "23.v_proj": 444, |
| "23.o_proj": 1129, |
| "23.gate_proj": 2016, |
| "23.up_proj": 2016, |
| "23.down_proj": 1752, |
| "24.q_proj": 1112, |
| "24.k_proj": 437, |
| "24.v_proj": 445, |
| "24.o_proj": 1133, |
| "24.gate_proj": 2016, |
| "24.up_proj": 2016, |
| "24.down_proj": 1754, |
| "25.q_proj": 1114, |
| "25.k_proj": 439, |
| "25.v_proj": 449, |
| "25.o_proj": 1141, |
| "25.gate_proj": 2016, |
| "25.up_proj": 2016, |
| "25.down_proj": 1766, |
| "26.q_proj": 1119, |
| "26.k_proj": 441, |
| "26.v_proj": 452, |
| "26.o_proj": 1149, |
| "26.gate_proj": 2016, |
| "26.up_proj": 2016, |
| "26.down_proj": 1780, |
| "27.q_proj": 1123, |
| "27.k_proj": 444, |
| "27.v_proj": 454, |
| "27.o_proj": 1149, |
| "27.gate_proj": 2016, |
| "27.up_proj": 2016, |
| "27.down_proj": 1792, |
| "28.q_proj": 1139, |
| "28.k_proj": 452, |
| "28.v_proj": 463, |
| "28.o_proj": 1171, |
| "28.gate_proj": 2016, |
| "28.up_proj": 2016, |
| "28.down_proj": 1838, |
| "29.q_proj": 1148, |
| "29.k_proj": 457, |
| "29.v_proj": 469, |
| "29.o_proj": 1193, |
| "29.gate_proj": 2016, |
| "29.up_proj": 2016, |
| "29.down_proj": 1861, |
| "30.q_proj": 1165, |
| "30.k_proj": 466, |
| "30.v_proj": 481, |
| "30.o_proj": 1215, |
| "30.gate_proj": 2016, |
| "30.up_proj": 2016, |
| "30.down_proj": 1912, |
| "31.q_proj": 1263, |
| "31.k_proj": 519, |
| "31.v_proj": 536, |
| "31.o_proj": 1336, |
| "31.gate_proj": 2016, |
| "31.up_proj": 2016, |
| "31.down_proj": 2179 |
| }, |
| "rel_err": { |
| "0.q_proj": 0.09311120212078094, |
| "0.k_proj": 0.08253687620162964, |
| "0.v_proj": 0.3433501720428467, |
| "0.o_proj": 0.3911000192165375, |
| "0.gate_proj": 0.3707573413848877, |
| "0.up_proj": 0.41067802906036377, |
| "0.down_proj": 0.5313096046447754, |
| "1.q_proj": 0.16670213639736176, |
| "1.k_proj": 0.10798212885856628, |
| "1.v_proj": 0.5118529796600342, |
| "1.o_proj": 0.5213813781738281, |
| "1.gate_proj": 0.4368397891521454, |
| "1.up_proj": 0.4705049991607666, |
| "1.down_proj": 0.4971780776977539, |
| "2.q_proj": 0.46672654151916504, |
| "2.k_proj": 0.46059876680374146, |
| "2.v_proj": 0.6164757013320923, |
| "2.o_proj": 0.5800544619560242, |
| "2.gate_proj": 0.512566328048706, |
| "2.up_proj": 0.5365164875984192, |
| "2.down_proj": 0.6227588057518005, |
| "3.q_proj": 0.4239010214805603, |
| "3.k_proj": 0.3750090003013611, |
| "3.v_proj": 0.6444101929664612, |
| "3.o_proj": 0.6232957243919373, |
| "3.gate_proj": 0.5219470858573914, |
| "3.up_proj": 0.551502525806427, |
| "3.down_proj": 0.6144695281982422, |
| "4.q_proj": 0.4298405945301056, |
| "4.k_proj": 0.3984021842479706, |
| "4.v_proj": 0.6045357584953308, |
| "4.o_proj": 0.5685608386993408, |
| "4.gate_proj": 0.523536205291748, |
| "4.up_proj": 0.5611042976379395, |
| "4.down_proj": 0.6211814284324646, |
| "5.q_proj": 0.46155083179473877, |
| "5.k_proj": 0.4410848319530487, |
| "5.v_proj": 0.6360407471656799, |
| "5.o_proj": 0.6054348349571228, |
| "5.gate_proj": 0.5178220868110657, |
| "5.up_proj": 0.5632009506225586, |
| "5.down_proj": 0.6081773638725281, |
| "6.q_proj": 0.4475008249282837, |
| "6.k_proj": 0.4137071967124939, |
| "6.v_proj": 0.6230500340461731, |
| "6.o_proj": 0.5891891121864319, |
| "6.gate_proj": 0.5309041738510132, |
| "6.up_proj": 0.5662814378738403, |
| "6.down_proj": 0.5977262258529663, |
| "7.q_proj": 0.427289217710495, |
| "7.k_proj": 0.38710105419158936, |
| "7.v_proj": 0.6161573529243469, |
| "7.o_proj": 0.5648438930511475, |
| "7.gate_proj": 0.5202533602714539, |
| "7.up_proj": 0.5593512654304504, |
| "7.down_proj": 0.592329740524292, |
| "8.q_proj": 0.43415379524230957, |
| "8.k_proj": 0.3839348256587982, |
| "8.v_proj": 0.576714813709259, |
| "8.o_proj": 0.5553375482559204, |
| "8.gate_proj": 0.523369312286377, |
| "8.up_proj": 0.5563741326332092, |
| "8.down_proj": 0.5858190059661865, |
| "9.q_proj": 0.42894482612609863, |
| "9.k_proj": 0.3965420722961426, |
| "9.v_proj": 0.5610228180885315, |
| "9.o_proj": 0.5573288202285767, |
| "9.gate_proj": 0.5195508599281311, |
| "9.up_proj": 0.5543780326843262, |
| "9.down_proj": 0.5704837441444397, |
| "10.q_proj": 0.42824241518974304, |
| "10.k_proj": 0.401052862405777, |
| "10.v_proj": 0.5595612525939941, |
| "10.o_proj": 0.5597419738769531, |
| "10.gate_proj": 0.5190359354019165, |
| "10.up_proj": 0.5517367720603943, |
| "10.down_proj": 0.5768575668334961, |
| "11.q_proj": 0.4241175949573517, |
| "11.k_proj": 0.3747548758983612, |
| "11.v_proj": 0.5366345643997192, |
| "11.o_proj": 0.5581367611885071, |
| "11.gate_proj": 0.5184133052825928, |
| "11.up_proj": 0.5504465103149414, |
| "11.down_proj": 0.5751660466194153, |
| "12.q_proj": 0.41172537207603455, |
| "12.k_proj": 0.3747641444206238, |
| "12.v_proj": 0.5728425979614258, |
| "12.o_proj": 0.5404180288314819, |
| "12.gate_proj": 0.5148531198501587, |
| "12.up_proj": 0.5455449223518372, |
| "12.down_proj": 0.5717161893844604, |
| "13.q_proj": 0.4387548565864563, |
| "13.k_proj": 0.40697500109672546, |
| "13.v_proj": 0.5708914399147034, |
| "13.o_proj": 0.5474628210067749, |
| "13.gate_proj": 0.5218628644943237, |
| "13.up_proj": 0.5454148054122925, |
| "13.down_proj": 0.5676140189170837, |
| "14.q_proj": 0.4094313979148865, |
| "14.k_proj": 0.38163262605667114, |
| "14.v_proj": 0.5798335075378418, |
| "14.o_proj": 0.5494734048843384, |
| "14.gate_proj": 0.5187147259712219, |
| "14.up_proj": 0.5501715540885925, |
| "14.down_proj": 0.593805193901062, |
| "15.q_proj": 0.40906858444213867, |
| "15.k_proj": 0.3624928295612335, |
| "15.v_proj": 0.6080702543258667, |
| "15.o_proj": 0.5543991327285767, |
| "15.gate_proj": 0.5166524648666382, |
| "15.up_proj": 0.5478196144104004, |
| "15.down_proj": 0.5904906988143921, |
| "16.q_proj": 0.4163205027580261, |
| "16.k_proj": 0.3787270486354828, |
| "16.v_proj": 0.6027750372886658, |
| "16.o_proj": 0.5409226417541504, |
| "16.gate_proj": 0.520693838596344, |
| "16.up_proj": 0.556447446346283, |
| "16.down_proj": 0.5904657244682312, |
| "17.q_proj": 0.4439752697944641, |
| "17.k_proj": 0.41725632548332214, |
| "17.v_proj": 0.5800758004188538, |
| "17.o_proj": 0.5581274628639221, |
| "17.gate_proj": 0.5324439406394958, |
| "17.up_proj": 0.5666811466217041, |
| "17.down_proj": 0.5825478434562683, |
| "18.q_proj": 0.4187489449977875, |
| "18.k_proj": 0.3872785270214081, |
| "18.v_proj": 0.6302375793457031, |
| "18.o_proj": 0.5343031883239746, |
| "18.gate_proj": 0.5309059619903564, |
| "18.up_proj": 0.564786434173584, |
| "18.down_proj": 0.5998350381851196, |
| "19.q_proj": 0.4693866968154907, |
| "19.k_proj": 0.40707793831825256, |
| "19.v_proj": 0.6421195268630981, |
| "19.o_proj": 0.5577782392501831, |
| "19.gate_proj": 0.536626398563385, |
| "19.up_proj": 0.5659677386283875, |
| "19.down_proj": 0.6147719621658325, |
| "20.q_proj": 0.4903647303581238, |
| "20.k_proj": 0.44427719712257385, |
| "20.v_proj": 0.6202300190925598, |
| "20.o_proj": 0.5741354823112488, |
| "20.gate_proj": 0.5429403185844421, |
| "20.up_proj": 0.5663204789161682, |
| "20.down_proj": 0.6225840449333191, |
| "21.q_proj": 0.5229761004447937, |
| "21.k_proj": 0.47562214732170105, |
| "21.v_proj": 0.5926244854927063, |
| "21.o_proj": 0.611431360244751, |
| "21.gate_proj": 0.5432093143463135, |
| "21.up_proj": 0.5681016445159912, |
| "21.down_proj": 0.6183050274848938, |
| "22.q_proj": 0.5626530051231384, |
| "22.k_proj": 0.5008488297462463, |
| "22.v_proj": 0.5983458757400513, |
| "22.o_proj": 0.6302191019058228, |
| "22.gate_proj": 0.5491480231285095, |
| "22.up_proj": 0.5697852969169617, |
| "22.down_proj": 0.6164385676383972, |
| "23.q_proj": 0.551480233669281, |
| "23.k_proj": 0.5012710690498352, |
| "23.v_proj": 0.6238268613815308, |
| "23.o_proj": 0.6080763339996338, |
| "23.gate_proj": 0.5521300435066223, |
| "23.up_proj": 0.5716087222099304, |
| "23.down_proj": 0.6099771857261658, |
| "24.q_proj": 0.5572198629379272, |
| "24.k_proj": 0.5183809399604797, |
| "24.v_proj": 0.6330755352973938, |
| "24.o_proj": 0.6300069689750671, |
| "24.gate_proj": 0.5530964136123657, |
| "24.up_proj": 0.5734592080116272, |
| "24.down_proj": 0.600332498550415, |
| "25.q_proj": 0.5632444620132446, |
| "25.k_proj": 0.5026987791061401, |
| "25.v_proj": 0.6118474006652832, |
| "25.o_proj": 0.6110818982124329, |
| "25.gate_proj": 0.5554454922676086, |
| "25.up_proj": 0.57483971118927, |
| "25.down_proj": 0.588827908039093, |
| "26.q_proj": 0.558449923992157, |
| "26.k_proj": 0.4919636845588684, |
| "26.v_proj": 0.5971678495407104, |
| "26.o_proj": 0.5927439332008362, |
| "26.gate_proj": 0.5562448501586914, |
| "26.up_proj": 0.5749430060386658, |
| "26.down_proj": 0.5749134421348572, |
| "27.q_proj": 0.5691151022911072, |
| "27.k_proj": 0.5340715050697327, |
| "27.v_proj": 0.6143185496330261, |
| "27.o_proj": 0.5845725536346436, |
| "27.gate_proj": 0.5597459673881531, |
| "27.up_proj": 0.5763874650001526, |
| "27.down_proj": 0.5742335319519043, |
| "28.q_proj": 0.5281823873519897, |
| "28.k_proj": 0.4590943157672882, |
| "28.v_proj": 0.5685400366783142, |
| "28.o_proj": 0.5630350112915039, |
| "28.gate_proj": 0.5594995617866516, |
| "28.up_proj": 0.5738884806632996, |
| "28.down_proj": 0.5813770294189453, |
| "29.q_proj": 0.5108082890510559, |
| "29.k_proj": 0.46057870984077454, |
| "29.v_proj": 0.6014853715896606, |
| "29.o_proj": 0.5444827079772949, |
| "29.gate_proj": 0.5542632937431335, |
| "29.up_proj": 0.5680719017982483, |
| "29.down_proj": 0.5874636769294739, |
| "30.q_proj": 0.4883614480495453, |
| "30.k_proj": 0.4195691645145416, |
| "30.v_proj": 0.535009503364563, |
| "30.o_proj": 0.5329458713531494, |
| "30.gate_proj": 0.5540768504142761, |
| "30.up_proj": 0.5595806241035461, |
| "30.down_proj": 0.5732670426368713, |
| "31.q_proj": 0.4402003884315491, |
| "31.k_proj": 0.3733157217502594, |
| "31.v_proj": 0.48748257756233215, |
| "31.o_proj": 0.45593205094337463, |
| "31.gate_proj": 0.5609548091888428, |
| "31.up_proj": 0.5358024835586548, |
| "31.down_proj": 0.5254908800125122 |
| }, |
| "step": "swift_svd_dynamic_finetuned", |
| "compression": { |
| "upstream": "hiahei/Swift-SVD", |
| "step": "swift_svd_dynamic", |
| "alloc": "dynamic", |
| "alpha": 0.6, |
| "nsamples": 256, |
| "seqlen": 2048, |
| "dataset": "WikiText2", |
| "seed": 42, |
| "gqa_budget": "shape", |
| "param_fraction": 0.6003230168269231, |
| "mean_rel_err": 0.523012746524598, |
| "max_rel_err": 0.6436969041824341 |
| }, |
| "factorisation": { |
| "max_rel_err": 0.001442187582142651, |
| "factor_params": 4189847552, |
| "dense_params": 6979321856 |
| }, |
| "finetune": { |
| "protocol": "svdllm_stage2_u_then_v", |
| "script": "gitclones/SVD-LLM/utils/LoRA.py", |
| "data": "alpaca-cleaned", |
| "epochs_per_half": 2, |
| "learning_rate": 0.0001, |
| "batch_size": 64, |
| "micro_batch_size": 4, |
| "lora_r": 8, |
| "lora_alpha": 16, |
| "lora_dropout": 0.05, |
| "cutoff_len": 256, |
| "val_set_size": 2000, |
| "max_train_samples": "all", |
| "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)" |
| } |
| } |
