| { |
| "source": "/workspace/svd_safety/assets/models/Llama-2-7b-chat-hf", |
| "upstream": "hiahei/Swift-SVD (compression) + AIoT-MLSys-Lab/SVD-LLM utils/LoRA.py (recovery)", |
| "stage": "finetuned", |
| "ratio_removed": 0.3, |
| "internal_ratio_kept": 0.7, |
| "from_pt": "/workspace/svd_safety/results/swift_ft/cells/l2_swift_remove30/finetune_v/merge.pt", |
| "mean_rel_err": 0.45528166218926863, |
| "max_rel_err": 0.5735374689102173, |
| "param_fraction": 0.6997563653659327, |
| "ranks": { |
| "0.q_proj": 1697, |
| "0.k_proj": 1682, |
| "0.v_proj": 1605, |
| "0.o_proj": 1624, |
| "0.gate_proj": 2137, |
| "0.up_proj": 2137, |
| "0.down_proj": 2390, |
| "1.q_proj": 1568, |
| "1.k_proj": 1564, |
| "1.v_proj": 1509, |
| "1.o_proj": 1504, |
| "1.gate_proj": 2113, |
| "1.up_proj": 2112, |
| "1.down_proj": 2485, |
| "2.q_proj": 1431, |
| "2.k_proj": 1430, |
| "2.v_proj": 1407, |
| "2.o_proj": 1375, |
| "2.gate_proj": 2082, |
| "2.up_proj": 2082, |
| "2.down_proj": 1966, |
| "3.q_proj": 1440, |
| "3.k_proj": 1440, |
| "3.v_proj": 1430, |
| "3.o_proj": 1397, |
| "3.gate_proj": 2086, |
| "3.up_proj": 2086, |
| "3.down_proj": 2007, |
| "4.q_proj": 1472, |
| "4.k_proj": 1473, |
| "4.v_proj": 1461, |
| "4.o_proj": 1436, |
| "4.gate_proj": 2095, |
| "4.up_proj": 2094, |
| "4.down_proj": 2089, |
| "5.q_proj": 1442, |
| "5.k_proj": 1444, |
| "5.v_proj": 1433, |
| "5.o_proj": 1415, |
| "5.gate_proj": 2089, |
| "5.up_proj": 2089, |
| "5.down_proj": 2045, |
| "6.q_proj": 1466, |
| "6.k_proj": 1466, |
| "6.v_proj": 1462, |
| "6.o_proj": 1446, |
| "6.gate_proj": 2096, |
| "6.up_proj": 2094, |
| "6.down_proj": 2100, |
| "7.q_proj": 1434, |
| "7.k_proj": 1434, |
| "7.v_proj": 1431, |
| "7.o_proj": 1417, |
| "7.gate_proj": 2089, |
| "7.up_proj": 2088, |
| "7.down_proj": 2050, |
| "8.q_proj": 1446, |
| "8.k_proj": 1447, |
| "8.v_proj": 1445, |
| "8.o_proj": 1436, |
| "8.gate_proj": 2092, |
| "8.up_proj": 2091, |
| "8.down_proj": 2080, |
| "9.q_proj": 1423, |
| "9.k_proj": 1424, |
| "9.v_proj": 1423, |
| "9.o_proj": 1418, |
| "9.gate_proj": 2087, |
| "9.up_proj": 2086, |
| "9.down_proj": 2042, |
| "10.q_proj": 1411, |
| "10.k_proj": 1413, |
| "10.v_proj": 1412, |
| "10.o_proj": 1410, |
| "10.gate_proj": 2084, |
| "10.up_proj": 2084, |
| "10.down_proj": 2024, |
| "11.q_proj": 1411, |
| "11.k_proj": 1412, |
| "11.v_proj": 1415, |
| "11.o_proj": 1409, |
| "11.gate_proj": 2084, |
| "11.up_proj": 2084, |
| "11.down_proj": 2024, |
| "12.q_proj": 1404, |
| "12.k_proj": 1405, |
| "12.v_proj": 1408, |
| "12.o_proj": 1405, |
| "12.gate_proj": 2082, |
| "12.up_proj": 2082, |
| "12.down_proj": 2017, |
| "13.q_proj": 1408, |
| "13.k_proj": 1410, |
| "13.v_proj": 1413, |
| "13.o_proj": 1413, |
| "13.gate_proj": 2084, |
| "13.up_proj": 2084, |
| "13.down_proj": 2031, |
| "14.q_proj": 1404, |
| "14.k_proj": 1407, |
| "14.v_proj": 1407, |
| "14.o_proj": 1409, |
| "14.gate_proj": 2082, |
| "14.up_proj": 2082, |
| "14.down_proj": 2025, |
| "15.q_proj": 1405, |
| "15.k_proj": 1407, |
| "15.v_proj": 1409, |
| "15.o_proj": 1414, |
| "15.gate_proj": 2083, |
| "15.up_proj": 2083, |
| "15.down_proj": 2034, |
| "16.q_proj": 1405, |
| "16.k_proj": 1408, |
| "16.v_proj": 1410, |
| "16.o_proj": 1418, |
| "16.gate_proj": 2084, |
| "16.up_proj": 2084, |
| "16.down_proj": 2042, |
| "17.q_proj": 1402, |
| "17.k_proj": 1403, |
| "17.v_proj": 1408, |
| "17.o_proj": 1410, |
| "17.gate_proj": 2083, |
| "17.up_proj": 2083, |
| "17.down_proj": 2039, |
| "18.q_proj": 1400, |
| "18.k_proj": 1400, |
| "18.v_proj": 1407, |
| "18.o_proj": 1407, |
| "18.gate_proj": 2082, |
| "18.up_proj": 2082, |
| "18.down_proj": 2038, |
| "19.q_proj": 1392, |
| "19.k_proj": 1393, |
| "19.v_proj": 1400, |
| "19.o_proj": 1399, |
| "19.gate_proj": 2081, |
| "19.up_proj": 2081, |
| "19.down_proj": 2026, |
| "20.q_proj": 1391, |
| "20.k_proj": 1392, |
| "20.v_proj": 1398, |
| "20.o_proj": 1404, |
| "20.gate_proj": 2081, |
| "20.up_proj": 2081, |
| "20.down_proj": 2030, |
| "21.q_proj": 1387, |
| "21.k_proj": 1388, |
| "21.v_proj": 1397, |
| "21.o_proj": 1395, |
| "21.gate_proj": 2080, |
| "21.up_proj": 2080, |
| "21.down_proj": 2025, |
| "22.q_proj": 1388, |
| "22.k_proj": 1387, |
| "22.v_proj": 1397, |
| "22.o_proj": 1403, |
| "22.gate_proj": 2080, |
| "22.up_proj": 2080, |
| "22.down_proj": 2026, |
| "23.q_proj": 1385, |
| "23.k_proj": 1384, |
| "23.v_proj": 1398, |
| "23.o_proj": 1398, |
| "23.gate_proj": 2079, |
| "23.up_proj": 2079, |
| "23.down_proj": 2024, |
| "24.q_proj": 1383, |
| "24.k_proj": 1384, |
| "24.v_proj": 1396, |
| "24.o_proj": 1401, |
| "24.gate_proj": 2079, |
| "24.up_proj": 2079, |
| "24.down_proj": 2024, |
| "25.q_proj": 1384, |
| "25.k_proj": 1383, |
| "25.v_proj": 1399, |
| "25.o_proj": 1408, |
| "25.gate_proj": 2079, |
| "25.up_proj": 2080, |
| "25.down_proj": 2029, |
| "26.q_proj": 1385, |
| "26.k_proj": 1385, |
| "26.v_proj": 1399, |
| "26.o_proj": 1409, |
| "26.gate_proj": 2080, |
| "26.up_proj": 2080, |
| "26.down_proj": 2032, |
| "27.q_proj": 1386, |
| "27.k_proj": 1386, |
| "27.v_proj": 1401, |
| "27.o_proj": 1408, |
| "27.gate_proj": 2081, |
| "27.up_proj": 2081, |
| "27.down_proj": 2040, |
| "28.q_proj": 1387, |
| "28.k_proj": 1387, |
| "28.v_proj": 1404, |
| "28.o_proj": 1417, |
| "28.gate_proj": 2081, |
| "28.up_proj": 2082, |
| "28.down_proj": 2047, |
| "29.q_proj": 1388, |
| "29.k_proj": 1388, |
| "29.v_proj": 1404, |
| "29.o_proj": 1423, |
| "29.gate_proj": 2082, |
| "29.up_proj": 2083, |
| "29.down_proj": 2056, |
| "30.q_proj": 1403, |
| "30.k_proj": 1402, |
| "30.v_proj": 1422, |
| "30.o_proj": 1435, |
| "30.gate_proj": 2087, |
| "30.up_proj": 2088, |
| "30.down_proj": 2135, |
| "31.q_proj": 1728, |
| "31.k_proj": 1728, |
| "31.v_proj": 1746, |
| "31.o_proj": 1793, |
| "31.gate_proj": 2164, |
| "31.up_proj": 2167, |
| "31.down_proj": 2826 |
| }, |
| "rel_err": { |
| "0.q_proj": 0.13674421608448029, |
| "0.k_proj": 0.10850787907838821, |
| "0.v_proj": 0.2810197174549103, |
| "0.o_proj": 0.2883068323135376, |
| "0.gate_proj": 0.3820878565311432, |
| "0.up_proj": 0.3937278091907501, |
| "0.down_proj": 0.40569689869880676, |
| "1.q_proj": 0.11325579136610031, |
| "1.k_proj": 0.10081755369901657, |
| "1.v_proj": 0.4143904745578766, |
| "1.o_proj": 0.3389497399330139, |
| "1.gate_proj": 0.3668021559715271, |
| "1.up_proj": 0.4152345657348633, |
| "1.down_proj": 0.453666090965271, |
| "2.q_proj": 0.2440570592880249, |
| "2.k_proj": 0.2325412780046463, |
| "2.v_proj": 0.46705976128578186, |
| "2.o_proj": 0.5312212705612183, |
| "2.gate_proj": 0.4374466836452484, |
| "2.up_proj": 0.4691368341445923, |
| "2.down_proj": 0.5438665747642517, |
| "3.q_proj": 0.33946898579597473, |
| "3.k_proj": 0.33782610297203064, |
| "3.v_proj": 0.4778972566127777, |
| "3.o_proj": 0.5165881514549255, |
| "3.gate_proj": 0.48844465613365173, |
| "3.up_proj": 0.5085727572441101, |
| "3.down_proj": 0.5010958909988403, |
| "4.q_proj": 0.32332727313041687, |
| "4.k_proj": 0.32587742805480957, |
| "4.v_proj": 0.4671332538127899, |
| "4.o_proj": 0.5094566345214844, |
| "4.gate_proj": 0.48788368701934814, |
| "4.up_proj": 0.5130534768104553, |
| "4.down_proj": 0.48729878664016724, |
| "5.q_proj": 0.3404356837272644, |
| "5.k_proj": 0.3388126492500305, |
| "5.v_proj": 0.4766134023666382, |
| "5.o_proj": 0.5210713148117065, |
| "5.gate_proj": 0.4849763810634613, |
| "5.up_proj": 0.5152232646942139, |
| "5.down_proj": 0.48836052417755127, |
| "6.q_proj": 0.32434237003326416, |
| "6.k_proj": 0.33024048805236816, |
| "6.v_proj": 0.45296233892440796, |
| "6.o_proj": 0.4798653721809387, |
| "6.gate_proj": 0.47807908058166504, |
| "6.up_proj": 0.5142185091972351, |
| "6.down_proj": 0.4754852056503296, |
| "7.q_proj": 0.33590492606163025, |
| "7.k_proj": 0.34187355637550354, |
| "7.v_proj": 0.454049676656723, |
| "7.o_proj": 0.505239725112915, |
| "7.gate_proj": 0.4762929379940033, |
| "7.up_proj": 0.5152503848075867, |
| "7.down_proj": 0.4799094796180725, |
| "8.q_proj": 0.3426366448402405, |
| "8.k_proj": 0.3538832366466522, |
| "8.v_proj": 0.43721604347229004, |
| "8.o_proj": 0.4970349967479706, |
| "8.gate_proj": 0.47296127676963806, |
| "8.up_proj": 0.5100219249725342, |
| "8.down_proj": 0.46890032291412354, |
| "9.q_proj": 0.3625483214855194, |
| "9.k_proj": 0.3672429919242859, |
| "9.v_proj": 0.44050657749176025, |
| "9.o_proj": 0.5047832727432251, |
| "9.gate_proj": 0.47035032510757446, |
| "9.up_proj": 0.505872368812561, |
| "9.down_proj": 0.47603756189346313, |
| "10.q_proj": 0.3382200598716736, |
| "10.k_proj": 0.34823906421661377, |
| "10.v_proj": 0.4375532865524292, |
| "10.o_proj": 0.46764662861824036, |
| "10.gate_proj": 0.46595898270606995, |
| "10.up_proj": 0.5017201900482178, |
| "10.down_proj": 0.47638988494873047, |
| "11.q_proj": 0.32728126645088196, |
| "11.k_proj": 0.3394625782966614, |
| "11.v_proj": 0.4230077862739563, |
| "11.o_proj": 0.4579029679298401, |
| "11.gate_proj": 0.4656955301761627, |
| "11.up_proj": 0.5019961595535278, |
| "11.down_proj": 0.4826474189758301, |
| "12.q_proj": 0.36442241072654724, |
| "12.k_proj": 0.3636908531188965, |
| "12.v_proj": 0.4427216649055481, |
| "12.o_proj": 0.49870848655700684, |
| "12.gate_proj": 0.4701372981071472, |
| "12.up_proj": 0.5024581551551819, |
| "12.down_proj": 0.4839555323123932, |
| "13.q_proj": 0.3692917227745056, |
| "13.k_proj": 0.38481763005256653, |
| "13.v_proj": 0.46928614377975464, |
| "13.o_proj": 0.48683831095695496, |
| "13.gate_proj": 0.47251319885253906, |
| "13.up_proj": 0.5052496790885925, |
| "13.down_proj": 0.4925529658794403, |
| "14.q_proj": 0.35274773836135864, |
| "14.k_proj": 0.3569324016571045, |
| "14.v_proj": 0.46338796615600586, |
| "14.o_proj": 0.4851902723312378, |
| "14.gate_proj": 0.47242483496665955, |
| "14.up_proj": 0.5040963292121887, |
| "14.down_proj": 0.49548181891441345, |
| "15.q_proj": 0.37204164266586304, |
| "15.k_proj": 0.3740440607070923, |
| "15.v_proj": 0.4692009389400482, |
| "15.o_proj": 0.49231812357902527, |
| "15.gate_proj": 0.4763273298740387, |
| "15.up_proj": 0.5079472064971924, |
| "15.down_proj": 0.506531834602356, |
| "16.q_proj": 0.3697231411933899, |
| "16.k_proj": 0.37245628237724304, |
| "16.v_proj": 0.4960777759552002, |
| "16.o_proj": 0.5187640190124512, |
| "16.gate_proj": 0.4750789403915405, |
| "16.up_proj": 0.5048896670341492, |
| "16.down_proj": 0.516768753528595, |
| "17.q_proj": 0.40830114483833313, |
| "17.k_proj": 0.4063715934753418, |
| "17.v_proj": 0.5102226734161377, |
| "17.o_proj": 0.5319307446479797, |
| "17.gate_proj": 0.4774886965751648, |
| "17.up_proj": 0.505339503288269, |
| "17.down_proj": 0.5146914124488831, |
| "18.q_proj": 0.4319038391113281, |
| "18.k_proj": 0.43669185042381287, |
| "18.v_proj": 0.5166221261024475, |
| "18.o_proj": 0.5600407719612122, |
| "18.gate_proj": 0.48564109206199646, |
| "18.up_proj": 0.5114160180091858, |
| "18.down_proj": 0.5150787830352783, |
| "19.q_proj": 0.4182446599006653, |
| "19.k_proj": 0.4145455062389374, |
| "19.v_proj": 0.5035737156867981, |
| "19.o_proj": 0.5499191284179688, |
| "19.gate_proj": 0.4894777834415436, |
| "19.up_proj": 0.5121901035308838, |
| "19.down_proj": 0.5176525712013245, |
| "20.q_proj": 0.40922680497169495, |
| "20.k_proj": 0.41069889068603516, |
| "20.v_proj": 0.5174776911735535, |
| "20.o_proj": 0.5493062138557434, |
| "20.gate_proj": 0.48762205243110657, |
| "20.up_proj": 0.5089380741119385, |
| "20.down_proj": 0.5267716646194458, |
| "21.q_proj": 0.4238526523113251, |
| "21.k_proj": 0.4254571795463562, |
| "21.v_proj": 0.511239767074585, |
| "21.o_proj": 0.566460132598877, |
| "21.gate_proj": 0.48842716217041016, |
| "21.up_proj": 0.5083724856376648, |
| "21.down_proj": 0.5235127806663513, |
| "22.q_proj": 0.43453019857406616, |
| "22.k_proj": 0.438902884721756, |
| "22.v_proj": 0.5219279527664185, |
| "22.o_proj": 0.5610068440437317, |
| "22.gate_proj": 0.48826220631599426, |
| "22.up_proj": 0.5073332190513611, |
| "22.down_proj": 0.520425021648407, |
| "23.q_proj": 0.4653097093105316, |
| "23.k_proj": 0.46524491906166077, |
| "23.v_proj": 0.5198501944541931, |
| "23.o_proj": 0.5531998872756958, |
| "23.gate_proj": 0.49149203300476074, |
| "23.up_proj": 0.509039580821991, |
| "23.down_proj": 0.5131181478500366, |
| "24.q_proj": 0.4144188165664673, |
| "24.k_proj": 0.4183977246284485, |
| "24.v_proj": 0.5133265852928162, |
| "24.o_proj": 0.557678759098053, |
| "24.gate_proj": 0.4918333888053894, |
| "24.up_proj": 0.5094541907310486, |
| "24.down_proj": 0.5062175393104553, |
| "25.q_proj": 0.4548630118370056, |
| "25.k_proj": 0.46258294582366943, |
| "25.v_proj": 0.5203204154968262, |
| "25.o_proj": 0.548804521560669, |
| "25.gate_proj": 0.4945409893989563, |
| "25.up_proj": 0.5112118124961853, |
| "25.down_proj": 0.498342901468277, |
| "26.q_proj": 0.42560890316963196, |
| "26.k_proj": 0.4341040253639221, |
| "26.v_proj": 0.5099557042121887, |
| "26.o_proj": 0.5735374689102173, |
| "26.gate_proj": 0.49582305550575256, |
| "26.up_proj": 0.5132369995117188, |
| "26.down_proj": 0.4963560402393341, |
| "27.q_proj": 0.47353821992874146, |
| "27.k_proj": 0.4811285436153412, |
| "27.v_proj": 0.5287732481956482, |
| "27.o_proj": 0.5502170324325562, |
| "27.gate_proj": 0.49853190779685974, |
| "27.up_proj": 0.5145693421363831, |
| "27.down_proj": 0.49317970871925354, |
| "28.q_proj": 0.4549473822116852, |
| "28.k_proj": 0.46495798230171204, |
| "28.v_proj": 0.5245335102081299, |
| "28.o_proj": 0.5447710156440735, |
| "28.gate_proj": 0.4986145496368408, |
| "28.up_proj": 0.513562023639679, |
| "28.down_proj": 0.5004502534866333, |
| "29.q_proj": 0.42228034138679504, |
| "29.k_proj": 0.43027618527412415, |
| "29.v_proj": 0.5161402225494385, |
| "29.o_proj": 0.5447760224342346, |
| "29.gate_proj": 0.49749407172203064, |
| "29.up_proj": 0.5095394253730774, |
| "29.down_proj": 0.5059615969657898, |
| "30.q_proj": 0.45088398456573486, |
| "30.k_proj": 0.4586084485054016, |
| "30.v_proj": 0.5238063335418701, |
| "30.o_proj": 0.5494594573974609, |
| "30.gate_proj": 0.4911543130874634, |
| "30.up_proj": 0.49852705001831055, |
| "30.down_proj": 0.49798041582107544, |
| "31.q_proj": 0.3175491690635681, |
| "31.k_proj": 0.3234711289405823, |
| "31.v_proj": 0.41444289684295654, |
| "31.o_proj": 0.41846781969070435, |
| "31.gate_proj": 0.4670276939868927, |
| "31.up_proj": 0.47030943632125854, |
| "31.down_proj": 0.347734272480011 |
| }, |
| "step": "swift_svd_dynamic_finetuned", |
| "compression": { |
| "upstream": "hiahei/Swift-SVD", |
| "step": "swift_svd_dynamic", |
| "alloc": "dynamic", |
| "alpha": 0.6, |
| "nsamples": 256, |
| "seqlen": 2048, |
| "dataset": "WikiText2", |
| "seed": 42, |
| "gqa_budget": null, |
| "param_fraction": 0.6997563653659327, |
| "mean_rel_err": 0.45483418411043075, |
| "max_rel_err": 0.5735054612159729 |
| }, |
| "factorisation": { |
| "max_rel_err": 0.0013921487843617797, |
| "factor_params": 4531625984, |
| "dense_params": 6476005376 |
| }, |
| "finetune": { |
| "protocol": "svdllm_stage2_u_then_v", |
| "script": "gitclones/SVD-LLM/utils/LoRA.py", |
| "data": "alpaca-cleaned", |
| "epochs_per_half": 2, |
| "learning_rate": 0.0001, |
| "batch_size": 64, |
| "micro_batch_size": 4, |
| "lora_r": 8, |
| "lora_alpha": 16, |
| "lora_dropout": 0.05, |
| "cutoff_len": 256, |
| "val_set_size": 2000, |
| "max_train_samples": "all", |
| "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)" |
| } |
| } |
