| { |
| "source": "/workspace/svd_safety/assets/models/Llama-2-7b-chat-hf", |
| "upstream": "hiahei/Swift-SVD (compression) + AIoT-MLSys-Lab/SVD-LLM utils/LoRA.py (recovery)", |
| "stage": "finetuned", |
| "ratio_removed": 0.2, |
| "internal_ratio_kept": 0.8, |
| "from_pt": "/workspace/svd_safety/results/swift_ft/cells/l2_swift_remove20/finetune_v/merge.pt", |
| "mean_rel_err": 0.39835471925990923, |
| "max_rel_err": 0.5163237452507019, |
| "param_fraction": 0.7998957658678757, |
| "ranks": { |
| "0.q_proj": 1939, |
| "0.k_proj": 1922, |
| "0.v_proj": 1834, |
| "0.o_proj": 1856, |
| "0.gate_proj": 2443, |
| "0.up_proj": 2444, |
| "0.down_proj": 2732, |
| "1.q_proj": 1792, |
| "1.k_proj": 1788, |
| "1.v_proj": 1724, |
| "1.o_proj": 1719, |
| "1.gate_proj": 2415, |
| "1.up_proj": 2414, |
| "1.down_proj": 2841, |
| "2.q_proj": 1635, |
| "2.k_proj": 1635, |
| "2.v_proj": 1609, |
| "2.o_proj": 1572, |
| "2.gate_proj": 2380, |
| "2.up_proj": 2380, |
| "2.down_proj": 2247, |
| "3.q_proj": 1646, |
| "3.k_proj": 1646, |
| "3.v_proj": 1634, |
| "3.o_proj": 1597, |
| "3.gate_proj": 2385, |
| "3.up_proj": 2385, |
| "3.down_proj": 2294, |
| "4.q_proj": 1682, |
| "4.k_proj": 1684, |
| "4.v_proj": 1670, |
| "4.o_proj": 1641, |
| "4.gate_proj": 2395, |
| "4.up_proj": 2394, |
| "4.down_proj": 2388, |
| "5.q_proj": 1648, |
| "5.k_proj": 1650, |
| "5.v_proj": 1638, |
| "5.o_proj": 1617, |
| "5.gate_proj": 2388, |
| "5.up_proj": 2387, |
| "5.down_proj": 2338, |
| "6.q_proj": 1675, |
| "6.k_proj": 1675, |
| "6.v_proj": 1671, |
| "6.o_proj": 1653, |
| "6.gate_proj": 2395, |
| "6.up_proj": 2394, |
| "6.down_proj": 2401, |
| "7.q_proj": 1639, |
| "7.k_proj": 1639, |
| "7.v_proj": 1636, |
| "7.o_proj": 1619, |
| "7.gate_proj": 2388, |
| "7.up_proj": 2386, |
| "7.down_proj": 2344, |
| "8.q_proj": 1653, |
| "8.k_proj": 1654, |
| "8.v_proj": 1652, |
| "8.o_proj": 1641, |
| "8.gate_proj": 2391, |
| "8.up_proj": 2390, |
| "8.down_proj": 2377, |
| "9.q_proj": 1626, |
| "9.k_proj": 1628, |
| "9.v_proj": 1627, |
| "9.o_proj": 1621, |
| "9.gate_proj": 2386, |
| "9.up_proj": 2385, |
| "9.down_proj": 2335, |
| "10.q_proj": 1613, |
| "10.k_proj": 1615, |
| "10.v_proj": 1614, |
| "10.o_proj": 1613, |
| "10.gate_proj": 2383, |
| "10.up_proj": 2382, |
| "10.down_proj": 2314, |
| "11.q_proj": 1612, |
| "11.k_proj": 1613, |
| "11.v_proj": 1617, |
| "11.o_proj": 1610, |
| "11.gate_proj": 2383, |
| "11.up_proj": 2382, |
| "11.down_proj": 2314, |
| "12.q_proj": 1605, |
| "12.k_proj": 1607, |
| "12.v_proj": 1610, |
| "12.o_proj": 1606, |
| "12.gate_proj": 2380, |
| "12.up_proj": 2380, |
| "12.down_proj": 2306, |
| "13.q_proj": 1611, |
| "13.k_proj": 1612, |
| "13.v_proj": 1615, |
| "13.o_proj": 1616, |
| "13.gate_proj": 2382, |
| "13.up_proj": 2382, |
| "13.down_proj": 2321, |
| "14.q_proj": 1605, |
| "14.k_proj": 1607, |
| "14.v_proj": 1609, |
| "14.o_proj": 1610, |
| "14.gate_proj": 2380, |
| "14.up_proj": 2380, |
| "14.down_proj": 2315, |
| "15.q_proj": 1607, |
| "15.k_proj": 1608, |
| "15.v_proj": 1612, |
| "15.o_proj": 1616, |
| "15.gate_proj": 2382, |
| "15.up_proj": 2382, |
| "15.down_proj": 2324, |
| "16.q_proj": 1607, |
| "16.k_proj": 1610, |
| "16.v_proj": 1612, |
| "16.o_proj": 1621, |
| "16.gate_proj": 2382, |
| "16.up_proj": 2382, |
| "16.down_proj": 2334, |
| "17.q_proj": 1603, |
| "17.k_proj": 1604, |
| "17.v_proj": 1609, |
| "17.o_proj": 1612, |
| "17.gate_proj": 2381, |
| "17.up_proj": 2381, |
| "17.down_proj": 2330, |
| "18.q_proj": 1600, |
| "18.k_proj": 1601, |
| "18.v_proj": 1608, |
| "18.o_proj": 1608, |
| "18.gate_proj": 2380, |
| "18.up_proj": 2380, |
| "18.down_proj": 2329, |
| "19.q_proj": 1591, |
| "19.k_proj": 1592, |
| "19.v_proj": 1600, |
| "19.o_proj": 1600, |
| "19.gate_proj": 2378, |
| "19.up_proj": 2378, |
| "19.down_proj": 2316, |
| "20.q_proj": 1590, |
| "20.k_proj": 1591, |
| "20.v_proj": 1598, |
| "20.o_proj": 1605, |
| "20.gate_proj": 2378, |
| "20.up_proj": 2378, |
| "20.down_proj": 2320, |
| "21.q_proj": 1586, |
| "21.k_proj": 1586, |
| "21.v_proj": 1597, |
| "21.o_proj": 1595, |
| "21.gate_proj": 2378, |
| "21.up_proj": 2378, |
| "21.down_proj": 2315, |
| "22.q_proj": 1586, |
| "22.k_proj": 1586, |
| "22.v_proj": 1597, |
| "22.o_proj": 1603, |
| "22.gate_proj": 2377, |
| "22.up_proj": 2378, |
| "22.down_proj": 2317, |
| "23.q_proj": 1583, |
| "23.k_proj": 1582, |
| "23.v_proj": 1598, |
| "23.o_proj": 1598, |
| "23.gate_proj": 2377, |
| "23.up_proj": 2377, |
| "23.down_proj": 2314, |
| "24.q_proj": 1581, |
| "24.k_proj": 1582, |
| "24.v_proj": 1595, |
| "24.o_proj": 1602, |
| "24.gate_proj": 2377, |
| "24.up_proj": 2377, |
| "24.down_proj": 2314, |
| "25.q_proj": 1582, |
| "25.k_proj": 1581, |
| "25.v_proj": 1599, |
| "25.o_proj": 1610, |
| "25.gate_proj": 2377, |
| "25.up_proj": 2377, |
| "25.down_proj": 2320, |
| "26.q_proj": 1583, |
| "26.k_proj": 1583, |
| "26.v_proj": 1599, |
| "26.o_proj": 1611, |
| "26.gate_proj": 2378, |
| "26.up_proj": 2378, |
| "26.down_proj": 2323, |
| "27.q_proj": 1585, |
| "27.k_proj": 1584, |
| "27.v_proj": 1602, |
| "27.o_proj": 1609, |
| "27.gate_proj": 2378, |
| "27.up_proj": 2379, |
| "27.down_proj": 2331, |
| "28.q_proj": 1586, |
| "28.k_proj": 1586, |
| "28.v_proj": 1605, |
| "28.o_proj": 1619, |
| "28.gate_proj": 2379, |
| "28.up_proj": 2380, |
| "28.down_proj": 2340, |
| "29.q_proj": 1587, |
| "29.k_proj": 1587, |
| "29.v_proj": 1605, |
| "29.o_proj": 1627, |
| "29.gate_proj": 2380, |
| "29.up_proj": 2382, |
| "29.down_proj": 2350, |
| "30.q_proj": 1604, |
| "30.k_proj": 1603, |
| "30.v_proj": 1625, |
| "30.o_proj": 1640, |
| "30.gate_proj": 2386, |
| "30.up_proj": 2387, |
| "30.down_proj": 2441, |
| "31.q_proj": 1974, |
| "31.k_proj": 1975, |
| "31.v_proj": 1995, |
| "31.o_proj": 2049, |
| "31.gate_proj": 2474, |
| "31.up_proj": 2477, |
| "31.down_proj": 3231 |
| }, |
| "rel_err": { |
| "0.q_proj": 0.13252992928028107, |
| "0.k_proj": 0.10504010319709778, |
| "0.v_proj": 0.23756639659404755, |
| "0.o_proj": 0.2499486804008484, |
| "0.gate_proj": 0.32737889885902405, |
| "0.up_proj": 0.336822509765625, |
| "0.down_proj": 0.33574700355529785, |
| "1.q_proj": 0.09638579189777374, |
| "1.k_proj": 0.0869532972574234, |
| "1.v_proj": 0.3620476722717285, |
| "1.o_proj": 0.2894635498523712, |
| "1.gate_proj": 0.31223469972610474, |
| "1.up_proj": 0.3540322184562683, |
| "1.down_proj": 0.3827917277812958, |
| "2.q_proj": 0.20203469693660736, |
| "2.k_proj": 0.19166691601276398, |
| "2.v_proj": 0.4135085940361023, |
| "2.o_proj": 0.4775710105895996, |
| "2.gate_proj": 0.38029783964157104, |
| "2.up_proj": 0.40832197666168213, |
| "2.down_proj": 0.4817647933959961, |
| "3.q_proj": 0.28911951184272766, |
| "3.k_proj": 0.2867107093334198, |
| "3.v_proj": 0.42236825823783875, |
| "3.o_proj": 0.4629066586494446, |
| "3.gate_proj": 0.4270183742046356, |
| "3.up_proj": 0.4446209967136383, |
| "3.down_proj": 0.43980148434638977, |
| "4.q_proj": 0.2749663293361664, |
| "4.k_proj": 0.27662697434425354, |
| "4.v_proj": 0.410868376493454, |
| "4.o_proj": 0.45410028100013733, |
| "4.gate_proj": 0.4263415038585663, |
| "4.up_proj": 0.449140727519989, |
| "4.down_proj": 0.4236817955970764, |
| "5.q_proj": 0.2918066382408142, |
| "5.k_proj": 0.2901315689086914, |
| "5.v_proj": 0.4196918308734894, |
| "5.o_proj": 0.46734270453453064, |
| "5.gate_proj": 0.42409467697143555, |
| "5.up_proj": 0.4514884650707245, |
| "5.down_proj": 0.42588138580322266, |
| "6.q_proj": 0.27811768651008606, |
| "6.k_proj": 0.2831922173500061, |
| "6.v_proj": 0.39679214358329773, |
| "6.o_proj": 0.42545533180236816, |
| "6.gate_proj": 0.41754117608070374, |
| "6.up_proj": 0.44990018010139465, |
| "6.down_proj": 0.4116297662258148, |
| "7.q_proj": 0.2889671325683594, |
| "7.k_proj": 0.2941109836101532, |
| "7.v_proj": 0.3983648121356964, |
| "7.o_proj": 0.45035600662231445, |
| "7.gate_proj": 0.4161233603954315, |
| "7.up_proj": 0.4515121281147003, |
| "7.down_proj": 0.41752707958221436, |
| "8.q_proj": 0.2945907711982727, |
| "8.k_proj": 0.30408018827438354, |
| "8.v_proj": 0.38267946243286133, |
| "8.o_proj": 0.44145527482032776, |
| "8.gate_proj": 0.41320326924324036, |
| "8.up_proj": 0.4467734396457672, |
| "8.down_proj": 0.406986266374588, |
| "9.q_proj": 0.3136454224586487, |
| "9.k_proj": 0.31721559166908264, |
| "9.v_proj": 0.3859423100948334, |
| "9.o_proj": 0.4510596990585327, |
| "9.gate_proj": 0.4109175205230713, |
| "9.up_proj": 0.4431936740875244, |
| "9.down_proj": 0.4149797558784485, |
| "10.q_proj": 0.29216355085372925, |
| "10.k_proj": 0.3005751371383667, |
| "10.v_proj": 0.3847027122974396, |
| "10.o_proj": 0.4111638367176056, |
| "10.gate_proj": 0.4067463278770447, |
| "10.up_proj": 0.43945708870887756, |
| "10.down_proj": 0.4159848392009735, |
| "11.q_proj": 0.2826056480407715, |
| "11.k_proj": 0.2936619520187378, |
| "11.v_proj": 0.3708671033382416, |
| "11.o_proj": 0.40357112884521484, |
| "11.gate_proj": 0.40603089332580566, |
| "11.up_proj": 0.4393159747123718, |
| "11.down_proj": 0.42167046666145325, |
| "12.q_proj": 0.31589677929878235, |
| "12.k_proj": 0.3152616322040558, |
| "12.v_proj": 0.38857823610305786, |
| "12.o_proj": 0.4415610432624817, |
| "12.gate_proj": 0.4102623462677002, |
| "12.up_proj": 0.43980276584625244, |
| "12.down_proj": 0.4229101836681366, |
| "13.q_proj": 0.320039302110672, |
| "13.k_proj": 0.33416274189949036, |
| "13.v_proj": 0.41291582584381104, |
| "13.o_proj": 0.4305266737937927, |
| "13.gate_proj": 0.4123358428478241, |
| "13.up_proj": 0.4423108994960785, |
| "13.down_proj": 0.43072378635406494, |
| "14.q_proj": 0.30624768137931824, |
| "14.k_proj": 0.31049633026123047, |
| "14.v_proj": 0.4073112905025482, |
| "14.o_proj": 0.4292844831943512, |
| "14.gate_proj": 0.41197630763053894, |
| "14.up_proj": 0.44101130962371826, |
| "14.down_proj": 0.4332798719406128, |
| "15.q_proj": 0.32311615347862244, |
| "15.k_proj": 0.325520783662796, |
| "15.v_proj": 0.41313716769218445, |
| "15.o_proj": 0.43440884351730347, |
| "15.gate_proj": 0.415483683347702, |
| "15.up_proj": 0.44479092955589294, |
| "15.down_proj": 0.44361311197280884, |
| "16.q_proj": 0.32134386897087097, |
| "16.k_proj": 0.32373708486557007, |
| "16.v_proj": 0.4383922517299652, |
| "16.o_proj": 0.460711807012558, |
| "16.gate_proj": 0.41429081559181213, |
| "16.up_proj": 0.4417015612125397, |
| "16.down_proj": 0.4521862864494324, |
| "17.q_proj": 0.35595712065696716, |
| "17.k_proj": 0.3545474112033844, |
| "17.v_proj": 0.4520525336265564, |
| "17.o_proj": 0.4732700288295746, |
| "17.gate_proj": 0.41633883118629456, |
| "17.up_proj": 0.4417755901813507, |
| "17.down_proj": 0.4500233232975006, |
| "18.q_proj": 0.37810665369033813, |
| "18.k_proj": 0.3822422921657562, |
| "18.v_proj": 0.45829951763153076, |
| "18.o_proj": 0.5030460357666016, |
| "18.gate_proj": 0.4236604869365692, |
| "18.up_proj": 0.4474699795246124, |
| "18.down_proj": 0.450743168592453, |
| "19.q_proj": 0.3656962513923645, |
| "19.k_proj": 0.3624645471572876, |
| "19.v_proj": 0.4467334449291229, |
| "19.o_proj": 0.4933334290981293, |
| "19.gate_proj": 0.42747756838798523, |
| "19.up_proj": 0.44855284690856934, |
| "19.down_proj": 0.4535249173641205, |
| "20.q_proj": 0.3567904829978943, |
| "20.k_proj": 0.35822567343711853, |
| "20.v_proj": 0.45949605107307434, |
| "20.o_proj": 0.4911755919456482, |
| "20.gate_proj": 0.42556217312812805, |
| "20.up_proj": 0.44537943601608276, |
| "20.down_proj": 0.4616205394268036, |
| "21.q_proj": 0.3698049783706665, |
| "21.k_proj": 0.371379554271698, |
| "21.v_proj": 0.45389649271965027, |
| "21.o_proj": 0.5087231397628784, |
| "21.gate_proj": 0.42624449729919434, |
| "21.up_proj": 0.44437286257743835, |
| "21.down_proj": 0.45840954780578613, |
| "22.q_proj": 0.37954530119895935, |
| "22.k_proj": 0.38333773612976074, |
| "22.v_proj": 0.463877409696579, |
| "22.o_proj": 0.5026487112045288, |
| "22.gate_proj": 0.4261413812637329, |
| "22.up_proj": 0.443158358335495, |
| "22.down_proj": 0.4550263583660126, |
| "23.q_proj": 0.4084702432155609, |
| "23.k_proj": 0.40841731429100037, |
| "23.v_proj": 0.4616864025592804, |
| "23.o_proj": 0.4950319528579712, |
| "23.gate_proj": 0.4287131428718567, |
| "23.up_proj": 0.444684237241745, |
| "23.down_proj": 0.44846802949905396, |
| "24.q_proj": 0.36004146933555603, |
| "24.k_proj": 0.363835871219635, |
| "24.v_proj": 0.45619717240333557, |
| "24.o_proj": 0.4993390142917633, |
| "24.gate_proj": 0.4288971722126007, |
| "24.up_proj": 0.4449731707572937, |
| "24.down_proj": 0.44221973419189453, |
| "25.q_proj": 0.39867693185806274, |
| "25.k_proj": 0.40591371059417725, |
| "25.v_proj": 0.4620579481124878, |
| "25.o_proj": 0.491049200296402, |
| "25.gate_proj": 0.4311861991882324, |
| "25.up_proj": 0.44645044207572937, |
| "25.down_proj": 0.43493691086769104, |
| "26.q_proj": 0.37144312262535095, |
| "26.k_proj": 0.37911906838417053, |
| "26.v_proj": 0.45244038105010986, |
| "26.o_proj": 0.5163237452507019, |
| "26.gate_proj": 0.4322204887866974, |
| "26.up_proj": 0.44827595353126526, |
| "26.down_proj": 0.4333139657974243, |
| "27.q_proj": 0.4169563055038452, |
| "27.k_proj": 0.4245080351829529, |
| "27.v_proj": 0.46958377957344055, |
| "27.o_proj": 0.49201327562332153, |
| "27.gate_proj": 0.4350823760032654, |
| "27.up_proj": 0.44976213574409485, |
| "27.down_proj": 0.43056654930114746, |
| "28.q_proj": 0.399061918258667, |
| "28.k_proj": 0.4083995521068573, |
| "28.v_proj": 0.46559157967567444, |
| "28.o_proj": 0.4871852695941925, |
| "28.gate_proj": 0.4352478086948395, |
| "28.up_proj": 0.4488693177700043, |
| "28.down_proj": 0.4368627965450287, |
| "29.q_proj": 0.36882483959198, |
| "29.k_proj": 0.37630507349967957, |
| "29.v_proj": 0.4574759304523468, |
| "29.o_proj": 0.48539915680885315, |
| "29.gate_proj": 0.43431898951530457, |
| "29.up_proj": 0.4452069103717804, |
| "29.down_proj": 0.4417956471443176, |
| "30.q_proj": 0.3955976068973541, |
| "30.k_proj": 0.4027300477027893, |
| "30.v_proj": 0.4643517732620239, |
| "30.o_proj": 0.4904117286205292, |
| "30.gate_proj": 0.42837339639663696, |
| "30.up_proj": 0.4352115988731384, |
| "30.down_proj": 0.430807888507843, |
| "31.q_proj": 0.2638401985168457, |
| "31.k_proj": 0.26833730936050415, |
| "31.v_proj": 0.34875962138175964, |
| "31.o_proj": 0.35200828313827515, |
| "31.gate_proj": 0.40333712100982666, |
| "31.up_proj": 0.4065763056278229, |
| "31.down_proj": 0.26060518622398376 |
| }, |
| "step": "swift_svd_dynamic_finetuned", |
| "compression": { |
| "upstream": "hiahei/Swift-SVD", |
| "step": "swift_svd_dynamic", |
| "alloc": "dynamic", |
| "alpha": 0.6, |
| "nsamples": 256, |
| "seqlen": 2048, |
| "dataset": "WikiText2", |
| "seed": 42, |
| "gqa_budget": null, |
| "param_fraction": 0.7993212685066183, |
| "mean_rel_err": 0.39799320328581544, |
| "max_rel_err": 0.516345202922821 |
| }, |
| "factorisation": { |
| "max_rel_err": 0.0013380160089582205, |
| "factor_params": 5180129280, |
| "dense_params": 6476005376 |
| }, |
| "finetune": { |
| "protocol": "svdllm_stage2_u_then_v", |
| "script": "gitclones/SVD-LLM/utils/LoRA.py", |
| "data": "alpaca-cleaned", |
| "epochs_per_half": 2, |
| "learning_rate": 0.0001, |
| "batch_size": 64, |
| "micro_batch_size": 4, |
| "lora_r": 8, |
| "lora_alpha": 16, |
| "lora_dropout": 0.05, |
| "cutoff_len": 256, |
| "val_set_size": 2000, |
| "max_train_samples": "all", |
| "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)" |
| } |
| } |
