| { |
| "source": "/workspace/svd_safety/assets/models/Llama-2-7b-chat-hf", |
| "upstream": "hiahei/Swift-SVD (compression) + AIoT-MLSys-Lab/SVD-LLM utils/LoRA.py (recovery)", |
| "stage": "finetuned", |
| "ratio_removed": 0.4, |
| "internal_ratio_kept": 0.6, |
| "from_pt": "/workspace/svd_safety/results/swift_ft/cells/l2_swift_remove40/finetune_v/merge.pt", |
| "mean_rel_err": 0.5146559384718005, |
| "max_rel_err": 0.631554126739502, |
| "param_fraction": 0.599840865851684, |
| "ranks": { |
| "0.q_proj": 1454, |
| "0.k_proj": 1441, |
| "0.v_proj": 1375, |
| "0.o_proj": 1391, |
| "0.gate_proj": 1832, |
| "0.up_proj": 1833, |
| "0.down_proj": 2049, |
| "1.q_proj": 1344, |
| "1.k_proj": 1341, |
| "1.v_proj": 1293, |
| "1.o_proj": 1289, |
| "1.gate_proj": 1812, |
| "1.up_proj": 1811, |
| "1.down_proj": 2131, |
| "2.q_proj": 1226, |
| "2.k_proj": 1226, |
| "2.v_proj": 1206, |
| "2.o_proj": 1178, |
| "2.gate_proj": 1785, |
| "2.up_proj": 1785, |
| "2.down_proj": 1685, |
| "3.q_proj": 1234, |
| "3.k_proj": 1234, |
| "3.v_proj": 1225, |
| "3.o_proj": 1197, |
| "3.gate_proj": 1789, |
| "3.up_proj": 1788, |
| "3.down_proj": 1720, |
| "4.q_proj": 1261, |
| "4.k_proj": 1262, |
| "4.v_proj": 1252, |
| "4.o_proj": 1230, |
| "4.gate_proj": 1796, |
| "4.up_proj": 1796, |
| "4.down_proj": 1791, |
| "5.q_proj": 1236, |
| "5.k_proj": 1237, |
| "5.v_proj": 1228, |
| "5.o_proj": 1212, |
| "5.gate_proj": 1791, |
| "5.up_proj": 1791, |
| "5.down_proj": 1754, |
| "6.q_proj": 1256, |
| "6.k_proj": 1256, |
| "6.v_proj": 1252, |
| "6.o_proj": 1239, |
| "6.gate_proj": 1797, |
| "6.up_proj": 1796, |
| "6.down_proj": 1801, |
| "7.q_proj": 1229, |
| "7.k_proj": 1229, |
| "7.v_proj": 1227, |
| "7.o_proj": 1214, |
| "7.gate_proj": 1791, |
| "7.up_proj": 1790, |
| "7.down_proj": 1758, |
| "8.q_proj": 1239, |
| "8.k_proj": 1240, |
| "8.v_proj": 1239, |
| "8.o_proj": 1231, |
| "8.gate_proj": 1794, |
| "8.up_proj": 1793, |
| "8.down_proj": 1783, |
| "9.q_proj": 1219, |
| "9.k_proj": 1221, |
| "9.v_proj": 1220, |
| "9.o_proj": 1215, |
| "9.gate_proj": 1789, |
| "9.up_proj": 1789, |
| "9.down_proj": 1751, |
| "10.q_proj": 1209, |
| "10.k_proj": 1211, |
| "10.v_proj": 1210, |
| "10.o_proj": 1209, |
| "10.gate_proj": 1787, |
| "10.up_proj": 1786, |
| "10.down_proj": 1735, |
| "11.q_proj": 1209, |
| "11.k_proj": 1210, |
| "11.v_proj": 1212, |
| "11.o_proj": 1207, |
| "11.gate_proj": 1787, |
| "11.up_proj": 1786, |
| "11.down_proj": 1735, |
| "12.q_proj": 1203, |
| "12.k_proj": 1204, |
| "12.v_proj": 1206, |
| "12.o_proj": 1204, |
| "12.gate_proj": 1785, |
| "12.up_proj": 1785, |
| "12.down_proj": 1729, |
| "13.q_proj": 1208, |
| "13.k_proj": 1209, |
| "13.v_proj": 1211, |
| "13.o_proj": 1211, |
| "13.gate_proj": 1787, |
| "13.up_proj": 1786, |
| "13.down_proj": 1741, |
| "14.q_proj": 1203, |
| "14.k_proj": 1204, |
| "14.v_proj": 1206, |
| "14.o_proj": 1207, |
| "14.gate_proj": 1785, |
| "14.up_proj": 1785, |
| "14.down_proj": 1736, |
| "15.q_proj": 1205, |
| "15.k_proj": 1205, |
| "15.v_proj": 1208, |
| "15.o_proj": 1211, |
| "15.gate_proj": 1787, |
| "15.up_proj": 1786, |
| "15.down_proj": 1744, |
| "16.q_proj": 1205, |
| "16.k_proj": 1207, |
| "16.v_proj": 1209, |
| "16.o_proj": 1216, |
| "16.gate_proj": 1787, |
| "16.up_proj": 1786, |
| "16.down_proj": 1751, |
| "17.q_proj": 1201, |
| "17.k_proj": 1202, |
| "17.v_proj": 1206, |
| "17.o_proj": 1209, |
| "17.gate_proj": 1785, |
| "17.up_proj": 1785, |
| "17.down_proj": 1748, |
| "18.q_proj": 1200, |
| "18.k_proj": 1200, |
| "18.v_proj": 1206, |
| "18.o_proj": 1205, |
| "18.gate_proj": 1785, |
| "18.up_proj": 1785, |
| "18.down_proj": 1747, |
| "19.q_proj": 1193, |
| "19.k_proj": 1194, |
| "19.v_proj": 1200, |
| "19.o_proj": 1199, |
| "19.gate_proj": 1784, |
| "19.up_proj": 1784, |
| "19.down_proj": 1737, |
| "20.q_proj": 1192, |
| "20.k_proj": 1192, |
| "20.v_proj": 1198, |
| "20.o_proj": 1203, |
| "20.gate_proj": 1784, |
| "20.up_proj": 1784, |
| "20.down_proj": 1740, |
| "21.q_proj": 1189, |
| "21.k_proj": 1189, |
| "21.v_proj": 1197, |
| "21.o_proj": 1196, |
| "21.gate_proj": 1783, |
| "21.up_proj": 1783, |
| "21.down_proj": 1736, |
| "22.q_proj": 1189, |
| "22.k_proj": 1189, |
| "22.v_proj": 1197, |
| "22.o_proj": 1202, |
| "22.gate_proj": 1783, |
| "22.up_proj": 1783, |
| "22.down_proj": 1737, |
| "23.q_proj": 1186, |
| "23.k_proj": 1186, |
| "23.v_proj": 1198, |
| "23.o_proj": 1198, |
| "23.gate_proj": 1782, |
| "23.up_proj": 1783, |
| "23.down_proj": 1735, |
| "24.q_proj": 1185, |
| "24.k_proj": 1186, |
| "24.v_proj": 1196, |
| "24.o_proj": 1201, |
| "24.gate_proj": 1782, |
| "24.up_proj": 1783, |
| "24.down_proj": 1735, |
| "25.q_proj": 1186, |
| "25.k_proj": 1185, |
| "25.v_proj": 1199, |
| "25.o_proj": 1207, |
| "25.gate_proj": 1783, |
| "25.up_proj": 1783, |
| "25.down_proj": 1740, |
| "26.q_proj": 1187, |
| "26.k_proj": 1187, |
| "26.v_proj": 1199, |
| "26.o_proj": 1209, |
| "26.gate_proj": 1783, |
| "26.up_proj": 1783, |
| "26.down_proj": 1742, |
| "27.q_proj": 1188, |
| "27.k_proj": 1188, |
| "27.v_proj": 1201, |
| "27.o_proj": 1206, |
| "27.gate_proj": 1784, |
| "27.up_proj": 1784, |
| "27.down_proj": 1749, |
| "28.q_proj": 1189, |
| "28.k_proj": 1189, |
| "28.v_proj": 1203, |
| "28.o_proj": 1214, |
| "28.gate_proj": 1784, |
| "28.up_proj": 1785, |
| "28.down_proj": 1755, |
| "29.q_proj": 1189, |
| "29.k_proj": 1190, |
| "29.v_proj": 1203, |
| "29.o_proj": 1220, |
| "29.gate_proj": 1785, |
| "29.up_proj": 1786, |
| "29.down_proj": 1763, |
| "30.q_proj": 1202, |
| "30.k_proj": 1201, |
| "30.v_proj": 1218, |
| "30.o_proj": 1230, |
| "30.gate_proj": 1789, |
| "30.up_proj": 1791, |
| "30.down_proj": 1831, |
| "31.q_proj": 1480, |
| "31.k_proj": 1481, |
| "31.v_proj": 1496, |
| "31.o_proj": 1536, |
| "31.gate_proj": 1855, |
| "31.up_proj": 1858, |
| "31.down_proj": 2423 |
| }, |
| "rel_err": { |
| "0.q_proj": 0.14318764209747314, |
| "0.k_proj": 0.11331941187381744, |
| "0.v_proj": 0.3313101828098297, |
| "0.o_proj": 0.3334074020385742, |
| "0.gate_proj": 0.4394051432609558, |
| "0.up_proj": 0.45333534479141235, |
| "0.down_proj": 0.4774848520755768, |
| "1.q_proj": 0.1363212615251541, |
| "1.k_proj": 0.12104111164808273, |
| "1.v_proj": 0.47113242745399475, |
| "1.o_proj": 0.39778363704681396, |
| "1.gate_proj": 0.42443057894706726, |
| "1.up_proj": 0.4789886474609375, |
| "1.down_proj": 0.5252040028572083, |
| "2.q_proj": 0.29307156801223755, |
| "2.k_proj": 0.2808840572834015, |
| "2.v_proj": 0.5237805247306824, |
| "2.o_proj": 0.5878390669822693, |
| "2.gate_proj": 0.4957614839076996, |
| "2.up_proj": 0.5317240357398987, |
| "2.down_proj": 0.6058598160743713, |
| "3.q_proj": 0.39533179998397827, |
| "3.k_proj": 0.39482513070106506, |
| "3.v_proj": 0.5370124578475952, |
| "3.o_proj": 0.5734050869941711, |
| "3.gate_proj": 0.5502407550811768, |
| "3.up_proj": 0.5727823972702026, |
| "3.down_proj": 0.5635493993759155, |
| "4.q_proj": 0.37751975655555725, |
| "4.k_proj": 0.38127243518829346, |
| "4.v_proj": 0.5265920758247375, |
| "4.o_proj": 0.5677705407142639, |
| "4.gate_proj": 0.5490803122520447, |
| "4.up_proj": 0.5769976377487183, |
| "4.down_proj": 0.5522221326828003, |
| "5.q_proj": 0.3945859670639038, |
| "5.k_proj": 0.3935074508190155, |
| "5.v_proj": 0.536858081817627, |
| "5.o_proj": 0.5777600407600403, |
| "5.gate_proj": 0.5458164215087891, |
| "5.up_proj": 0.5791149139404297, |
| "5.down_proj": 0.5521062612533569, |
| "6.q_proj": 0.375798761844635, |
| "6.k_proj": 0.382950097322464, |
| "6.v_proj": 0.5131592750549316, |
| "6.o_proj": 0.5379549264907837, |
| "6.gate_proj": 0.5389353632926941, |
| "6.up_proj": 0.5784068703651428, |
| "6.down_proj": 0.5407786965370178, |
| "7.q_proj": 0.3880448639392853, |
| "7.k_proj": 0.39485877752304077, |
| "7.v_proj": 0.5132298469543457, |
| "7.o_proj": 0.5630982518196106, |
| "7.gate_proj": 0.5367270112037659, |
| "7.up_proj": 0.5791166424751282, |
| "7.down_proj": 0.5435407161712646, |
| "8.q_proj": 0.3958355188369751, |
| "8.k_proj": 0.40927451848983765, |
| "8.v_proj": 0.4954420328140259, |
| "8.o_proj": 0.5553522706031799, |
| "8.gate_proj": 0.533244252204895, |
| "8.up_proj": 0.5733633637428284, |
| "8.down_proj": 0.5326294302940369, |
| "9.q_proj": 0.4168955981731415, |
| "9.k_proj": 0.422296404838562, |
| "9.v_proj": 0.4987483322620392, |
| "9.o_proj": 0.5611084699630737, |
| "9.gate_proj": 0.5303514003753662, |
| "9.up_proj": 0.5685946941375732, |
| "9.down_proj": 0.5385213494300842, |
| "10.q_proj": 0.38947921991348267, |
| "10.k_proj": 0.40109699964523315, |
| "10.v_proj": 0.4944608509540558, |
| "10.o_proj": 0.5274718403816223, |
| "10.gate_proj": 0.5256984233856201, |
| "10.up_proj": 0.5644442439079285, |
| "10.down_proj": 0.5384189486503601, |
| "11.q_proj": 0.3771919906139374, |
| "11.k_proj": 0.3906274139881134, |
| "11.v_proj": 0.47947946190834045, |
| "11.o_proj": 0.5166332721710205, |
| "11.gate_proj": 0.5260279178619385, |
| "11.up_proj": 0.5649887323379517, |
| "11.down_proj": 0.5448550581932068, |
| "12.q_proj": 0.41809821128845215, |
| "12.k_proj": 0.4165879487991333, |
| "12.v_proj": 0.5010970234870911, |
| "12.o_proj": 0.5586170554161072, |
| "12.gate_proj": 0.5309696793556213, |
| "12.up_proj": 0.565258800983429, |
| "12.down_proj": 0.5461880564689636, |
| "13.q_proj": 0.42253896594047546, |
| "13.k_proj": 0.4398796558380127, |
| "13.v_proj": 0.5294481515884399, |
| "13.o_proj": 0.546043336391449, |
| "13.gate_proj": 0.5335360765457153, |
| "13.up_proj": 0.5683612823486328, |
| "13.down_proj": 0.5552169680595398, |
| "14.q_proj": 0.403841495513916, |
| "14.k_proj": 0.40865302085876465, |
| "14.v_proj": 0.52288419008255, |
| "14.o_proj": 0.5442802906036377, |
| "14.gate_proj": 0.5336455702781677, |
| "14.up_proj": 0.5672312378883362, |
| "14.down_proj": 0.5583865642547607, |
| "15.q_proj": 0.425073504447937, |
| "15.k_proj": 0.4273808002471924, |
| "15.v_proj": 0.5283194184303284, |
| "15.o_proj": 0.5536724925041199, |
| "15.gate_proj": 0.537350594997406, |
| "15.up_proj": 0.5708211660385132, |
| "15.down_proj": 0.5700202584266663, |
| "16.q_proj": 0.4221418499946594, |
| "16.k_proj": 0.42537611722946167, |
| "16.v_proj": 0.5565226674079895, |
| "16.o_proj": 0.5793879628181458, |
| "16.gate_proj": 0.5364647507667542, |
| "16.up_proj": 0.568230152130127, |
| "16.down_proj": 0.5813849568367004, |
| "17.q_proj": 0.4647054970264435, |
| "17.k_proj": 0.4621625244617462, |
| "17.v_proj": 0.5715993642807007, |
| "17.o_proj": 0.5927662253379822, |
| "17.gate_proj": 0.5394077897071838, |
| "17.up_proj": 0.5690187811851501, |
| "17.down_proj": 0.5798382759094238, |
| "18.q_proj": 0.48962292075157166, |
| "18.k_proj": 0.4945559501647949, |
| "18.v_proj": 0.5778101682662964, |
| "18.o_proj": 0.6191045045852661, |
| "18.gate_proj": 0.5480973720550537, |
| "18.up_proj": 0.5750609636306763, |
| "18.down_proj": 0.5799038410186768, |
| "19.q_proj": 0.47487759590148926, |
| "19.k_proj": 0.4706522524356842, |
| "19.v_proj": 0.5633808970451355, |
| "19.o_proj": 0.608212411403656, |
| "19.gate_proj": 0.5518002510070801, |
| "19.up_proj": 0.5758308172225952, |
| "19.down_proj": 0.5820042490959167, |
| "20.q_proj": 0.4659927785396576, |
| "20.k_proj": 0.46770182251930237, |
| "20.v_proj": 0.5783191919326782, |
| "20.o_proj": 0.6094985008239746, |
| "20.gate_proj": 0.5498113036155701, |
| "20.up_proj": 0.5726807117462158, |
| "20.down_proj": 0.5921808481216431, |
| "21.q_proj": 0.48177552223205566, |
| "21.k_proj": 0.48396918177604675, |
| "21.v_proj": 0.5714066028594971, |
| "21.o_proj": 0.6256420016288757, |
| "21.gate_proj": 0.5510643124580383, |
| "21.up_proj": 0.5723118782043457, |
| "21.down_proj": 0.5887574553489685, |
| "22.q_proj": 0.49377286434173584, |
| "22.k_proj": 0.49821627140045166, |
| "22.v_proj": 0.5825276970863342, |
| "22.o_proj": 0.6214143633842468, |
| "22.gate_proj": 0.5508750677108765, |
| "22.up_proj": 0.5714125037193298, |
| "22.down_proj": 0.5857972502708435, |
| "23.q_proj": 0.5258812308311462, |
| "23.k_proj": 0.5253252983093262, |
| "23.v_proj": 0.5805611610412598, |
| "23.o_proj": 0.6131620407104492, |
| "23.gate_proj": 0.5545095801353455, |
| "23.up_proj": 0.5731971859931946, |
| "23.down_proj": 0.5782360434532166, |
| "24.q_proj": 0.4733271896839142, |
| "24.k_proj": 0.4773584306240082, |
| "24.v_proj": 0.5734611749649048, |
| "24.o_proj": 0.617493748664856, |
| "24.gate_proj": 0.5553052425384521, |
| "24.up_proj": 0.573965311050415, |
| "24.down_proj": 0.5709168314933777, |
| "25.q_proj": 0.51483154296875, |
| "25.k_proj": 0.522794783115387, |
| "25.v_proj": 0.5812080502510071, |
| "25.o_proj": 0.60832679271698, |
| "25.gate_proj": 0.558302640914917, |
| "25.up_proj": 0.5760963559150696, |
| "25.down_proj": 0.5624425411224365, |
| "26.q_proj": 0.484011173248291, |
| "26.k_proj": 0.49306154251098633, |
| "26.v_proj": 0.5701485872268677, |
| "26.o_proj": 0.631554126739502, |
| "26.gate_proj": 0.559934675693512, |
| "26.up_proj": 0.5781747102737427, |
| "26.down_proj": 0.5602895021438599, |
| "27.q_proj": 0.5331695675849915, |
| "27.k_proj": 0.5409024953842163, |
| "27.v_proj": 0.5903238654136658, |
| "27.o_proj": 0.6105265617370605, |
| "27.gate_proj": 0.5627158284187317, |
| "27.up_proj": 0.5795223712921143, |
| "27.down_proj": 0.556897759437561, |
| "28.q_proj": 0.5141112208366394, |
| "28.k_proj": 0.5246506333351135, |
| "28.v_proj": 0.5859890580177307, |
| "28.o_proj": 0.6046826243400574, |
| "28.gate_proj": 0.5626722574234009, |
| "28.up_proj": 0.5781853795051575, |
| "28.down_proj": 0.5647644996643066, |
| "29.q_proj": 0.4800718128681183, |
| "29.k_proj": 0.48807916045188904, |
| "29.v_proj": 0.5776322484016418, |
| "29.o_proj": 0.6057113409042358, |
| "29.gate_proj": 0.5614248514175415, |
| "29.up_proj": 0.5739281177520752, |
| "29.down_proj": 0.570681095123291, |
| "30.q_proj": 0.5097528100013733, |
| "30.k_proj": 0.5179398655891418, |
| "30.v_proj": 0.5861728191375732, |
| "30.o_proj": 0.6103938817977905, |
| "30.gate_proj": 0.5546872615814209, |
| "30.up_proj": 0.5617015361785889, |
| "30.down_proj": 0.5654463171958923, |
| "31.q_proj": 0.37722048163414, |
| "31.k_proj": 0.3839050531387329, |
| "31.v_proj": 0.4845984876155853, |
| "31.o_proj": 0.48935359716415405, |
| "31.gate_proj": 0.5315423011779785, |
| "31.up_proj": 0.5343829989433289, |
| "31.down_proj": 0.43410614132881165 |
| }, |
| "step": "swift_svd_dynamic_finetuned", |
| "compression": { |
| "upstream": "hiahei/Swift-SVD", |
| "step": "swift_svd_dynamic", |
| "alloc": "dynamic", |
| "alpha": 0.6, |
| "nsamples": 256, |
| "seqlen": 2048, |
| "dataset": "WikiText2", |
| "seed": 42, |
| "gqa_budget": null, |
| "param_fraction": 0.599840865851684, |
| "mean_rel_err": 0.5142416990295585, |
| "max_rel_err": 0.6316028237342834 |
| }, |
| "factorisation": { |
| "max_rel_err": 0.0014253825647756457, |
| "factor_params": 3884572672, |
| "dense_params": 6476005376 |
| }, |
| "finetune": { |
| "protocol": "svdllm_stage2_u_then_v", |
| "script": "gitclones/SVD-LLM/utils/LoRA.py", |
| "data": "alpaca-cleaned", |
| "epochs_per_half": 2, |
| "learning_rate": 0.0001, |
| "batch_size": 64, |
| "micro_batch_size": 4, |
| "lora_r": 8, |
| "lora_alpha": 16, |
| "lora_dropout": 0.05, |
| "cutoff_len": 256, |
| "val_set_size": 2000, |
| "max_train_samples": "all", |
| "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)" |
| } |
| } |
