| { |
| "source": "/workspace/svd_safety/assets/models/Llama-3-8B-Instruct", |
| "upstream": "hiahei/Swift-SVD (compression) + AIoT-MLSys-Lab/SVD-LLM utils/LoRA.py (recovery)", |
| "stage": "finetuned", |
| "ratio_removed": 0.5, |
| "internal_ratio_kept": 0.5, |
| "from_pt": "/workspace/svd_safety/results/swift_ft/cells/l3_swift_remove50/finetune_v/merge.pt", |
| "mean_rel_err": 0.5984087092136698, |
| "max_rel_err": 0.7104170918464661, |
| "param_fraction": 0.5002723106971154, |
| "ranks": { |
| "0.q_proj": 1153, |
| "0.k_proj": 486, |
| "0.v_proj": 455, |
| "0.o_proj": 1097, |
| "0.gate_proj": 1680, |
| "0.up_proj": 1680, |
| "0.down_proj": 1764, |
| "1.q_proj": 1001, |
| "1.k_proj": 405, |
| "1.v_proj": 389, |
| "1.o_proj": 959, |
| "1.gate_proj": 1680, |
| "1.up_proj": 1680, |
| "1.down_proj": 1618, |
| "2.q_proj": 993, |
| "2.k_proj": 401, |
| "2.v_proj": 396, |
| "2.o_proj": 962, |
| "2.gate_proj": 1680, |
| "2.up_proj": 1680, |
| "2.down_proj": 1484, |
| "3.q_proj": 979, |
| "3.k_proj": 394, |
| "3.v_proj": 392, |
| "3.o_proj": 954, |
| "3.gate_proj": 1680, |
| "3.up_proj": 1680, |
| "3.down_proj": 1474, |
| "4.q_proj": 987, |
| "4.k_proj": 399, |
| "4.v_proj": 396, |
| "4.o_proj": 970, |
| "4.gate_proj": 1680, |
| "4.up_proj": 1680, |
| "4.down_proj": 1506, |
| "5.q_proj": 979, |
| "5.k_proj": 395, |
| "5.v_proj": 391, |
| "5.o_proj": 966, |
| "5.gate_proj": 1680, |
| "5.up_proj": 1680, |
| "5.down_proj": 1501, |
| "6.q_proj": 970, |
| "6.k_proj": 390, |
| "6.v_proj": 388, |
| "6.o_proj": 963, |
| "6.gate_proj": 1680, |
| "6.up_proj": 1680, |
| "6.down_proj": 1490, |
| "7.q_proj": 969, |
| "7.k_proj": 390, |
| "7.v_proj": 388, |
| "7.o_proj": 967, |
| "7.gate_proj": 1680, |
| "7.up_proj": 1680, |
| "7.down_proj": 1489, |
| "8.q_proj": 959, |
| "8.k_proj": 385, |
| "8.v_proj": 383, |
| "8.o_proj": 958, |
| "8.gate_proj": 1680, |
| "8.up_proj": 1680, |
| "8.down_proj": 1472, |
| "9.q_proj": 957, |
| "9.k_proj": 384, |
| "9.v_proj": 384, |
| "9.o_proj": 957, |
| "9.gate_proj": 1680, |
| "9.up_proj": 1680, |
| "9.down_proj": 1471, |
| "10.q_proj": 949, |
| "10.k_proj": 379, |
| "10.v_proj": 378, |
| "10.o_proj": 949, |
| "10.gate_proj": 1680, |
| "10.up_proj": 1680, |
| "10.down_proj": 1457, |
| "11.q_proj": 955, |
| "11.k_proj": 382, |
| "11.v_proj": 383, |
| "11.o_proj": 956, |
| "11.gate_proj": 1680, |
| "11.up_proj": 1680, |
| "11.down_proj": 1471, |
| "12.q_proj": 960, |
| "12.k_proj": 384, |
| "12.v_proj": 385, |
| "12.o_proj": 963, |
| "12.gate_proj": 1680, |
| "12.up_proj": 1680, |
| "12.down_proj": 1480, |
| "13.q_proj": 956, |
| "13.k_proj": 384, |
| "13.v_proj": 384, |
| "13.o_proj": 962, |
| "13.gate_proj": 1680, |
| "13.up_proj": 1680, |
| "13.down_proj": 1479, |
| "14.q_proj": 956, |
| "14.k_proj": 384, |
| "14.v_proj": 384, |
| "14.o_proj": 962, |
| "14.gate_proj": 1680, |
| "14.up_proj": 1680, |
| "14.down_proj": 1481, |
| "15.q_proj": 959, |
| "15.k_proj": 384, |
| "15.v_proj": 384, |
| "15.o_proj": 963, |
| "15.gate_proj": 1680, |
| "15.up_proj": 1680, |
| "15.down_proj": 1490, |
| "16.q_proj": 957, |
| "16.k_proj": 383, |
| "16.v_proj": 382, |
| "16.o_proj": 963, |
| "16.gate_proj": 1680, |
| "16.up_proj": 1680, |
| "16.down_proj": 1488, |
| "17.q_proj": 946, |
| "17.k_proj": 376, |
| "17.v_proj": 376, |
| "17.o_proj": 949, |
| "17.gate_proj": 1680, |
| "17.up_proj": 1680, |
| "17.down_proj": 1468, |
| "18.q_proj": 936, |
| "18.k_proj": 371, |
| "18.v_proj": 372, |
| "18.o_proj": 935, |
| "18.gate_proj": 1680, |
| "18.up_proj": 1680, |
| "18.down_proj": 1450, |
| "19.q_proj": 931, |
| "19.k_proj": 368, |
| "19.v_proj": 370, |
| "19.o_proj": 927, |
| "19.gate_proj": 1680, |
| "19.up_proj": 1680, |
| "19.down_proj": 1440, |
| "20.q_proj": 928, |
| "20.k_proj": 367, |
| "20.v_proj": 369, |
| "20.o_proj": 925, |
| "20.gate_proj": 1680, |
| "20.up_proj": 1680, |
| "20.down_proj": 1436, |
| "21.q_proj": 931, |
| "21.k_proj": 368, |
| "21.v_proj": 370, |
| "21.o_proj": 935, |
| "21.gate_proj": 1680, |
| "21.up_proj": 1680, |
| "21.down_proj": 1444, |
| "22.q_proj": 925, |
| "22.k_proj": 365, |
| "22.v_proj": 368, |
| "22.o_proj": 925, |
| "22.gate_proj": 1680, |
| "22.up_proj": 1680, |
| "22.down_proj": 1436, |
| "23.q_proj": 924, |
| "23.k_proj": 364, |
| "23.v_proj": 367, |
| "23.o_proj": 923, |
| "23.gate_proj": 1680, |
| "23.up_proj": 1680, |
| "23.down_proj": 1433, |
| "24.q_proj": 923, |
| "24.k_proj": 363, |
| "24.v_proj": 368, |
| "24.o_proj": 926, |
| "24.gate_proj": 1680, |
| "24.up_proj": 1680, |
| "24.down_proj": 1431, |
| "25.q_proj": 925, |
| "25.k_proj": 364, |
| "25.v_proj": 369, |
| "25.o_proj": 930, |
| "25.gate_proj": 1680, |
| "25.up_proj": 1680, |
| "25.down_proj": 1434, |
| "26.q_proj": 924, |
| "26.k_proj": 364, |
| "26.v_proj": 368, |
| "26.o_proj": 939, |
| "26.gate_proj": 1680, |
| "26.up_proj": 1680, |
| "26.down_proj": 1440, |
| "27.q_proj": 925, |
| "27.k_proj": 364, |
| "27.v_proj": 371, |
| "27.o_proj": 944, |
| "27.gate_proj": 1680, |
| "27.up_proj": 1680, |
| "27.down_proj": 1448, |
| "28.q_proj": 928, |
| "28.k_proj": 366, |
| "28.v_proj": 372, |
| "28.o_proj": 945, |
| "28.gate_proj": 1680, |
| "28.up_proj": 1680, |
| "28.down_proj": 1462, |
| "29.q_proj": 933, |
| "29.k_proj": 369, |
| "29.v_proj": 376, |
| "29.o_proj": 961, |
| "29.gate_proj": 1680, |
| "29.up_proj": 1680, |
| "29.down_proj": 1482, |
| "30.q_proj": 944, |
| "30.k_proj": 374, |
| "30.v_proj": 382, |
| "30.o_proj": 978, |
| "30.gate_proj": 1680, |
| "30.up_proj": 1680, |
| "30.down_proj": 1527, |
| "31.q_proj": 1058, |
| "31.k_proj": 436, |
| "31.v_proj": 448, |
| "31.o_proj": 1107, |
| "31.gate_proj": 1680, |
| "31.up_proj": 1680, |
| "31.down_proj": 1830 |
| }, |
| "rel_err": { |
| "0.q_proj": 0.1555257886648178, |
| "0.k_proj": 0.11019963026046753, |
| "0.v_proj": 0.6514365673065186, |
| "0.o_proj": 0.558626651763916, |
| "0.gate_proj": 0.5772461295127869, |
| "0.up_proj": 0.6381028890609741, |
| "0.down_proj": 0.6298868060112, |
| "1.q_proj": 0.3837055265903473, |
| "1.k_proj": 0.340504914522171, |
| "1.v_proj": 0.6825932860374451, |
| "1.o_proj": 0.6070690155029297, |
| "1.gate_proj": 0.5934946537017822, |
| "1.up_proj": 0.6246401071548462, |
| "1.down_proj": 0.6702373623847961, |
| "2.q_proj": 0.5115190148353577, |
| "2.k_proj": 0.46816286444664, |
| "2.v_proj": 0.6202782988548279, |
| "2.o_proj": 0.5873217582702637, |
| "2.gate_proj": 0.6200411319732666, |
| "2.up_proj": 0.6416133642196655, |
| "2.down_proj": 0.6713889241218567, |
| "3.q_proj": 0.5343612432479858, |
| "3.k_proj": 0.48675867915153503, |
| "3.v_proj": 0.6520168781280518, |
| "3.o_proj": 0.6289046406745911, |
| "3.gate_proj": 0.6113771796226501, |
| "3.up_proj": 0.649507462978363, |
| "3.down_proj": 0.664247989654541, |
| "4.q_proj": 0.5104104280471802, |
| "4.k_proj": 0.4806947410106659, |
| "4.v_proj": 0.6609205603599548, |
| "4.o_proj": 0.6258541941642761, |
| "4.gate_proj": 0.6113108992576599, |
| "4.up_proj": 0.6584666967391968, |
| "4.down_proj": 0.6677454113960266, |
| "5.q_proj": 0.4832381010055542, |
| "5.k_proj": 0.49522989988327026, |
| "5.v_proj": 0.6369020938873291, |
| "5.o_proj": 0.6074650883674622, |
| "5.gate_proj": 0.6114932298660278, |
| "5.up_proj": 0.6566051840782166, |
| "5.down_proj": 0.663334310054779, |
| "6.q_proj": 0.4991042912006378, |
| "6.k_proj": 0.4993811845779419, |
| "6.v_proj": 0.6485827565193176, |
| "6.o_proj": 0.6202296018600464, |
| "6.gate_proj": 0.6068925857543945, |
| "6.up_proj": 0.6527253985404968, |
| "6.down_proj": 0.6605090498924255, |
| "7.q_proj": 0.4872052073478699, |
| "7.k_proj": 0.485134094953537, |
| "7.v_proj": 0.6055662631988525, |
| "7.o_proj": 0.6058576107025146, |
| "7.gate_proj": 0.6032496094703674, |
| "7.up_proj": 0.6502962112426758, |
| "7.down_proj": 0.641866147518158, |
| "8.q_proj": 0.4624485671520233, |
| "8.k_proj": 0.46426060795783997, |
| "8.v_proj": 0.6231876611709595, |
| "8.o_proj": 0.5961927771568298, |
| "8.gate_proj": 0.5874994993209839, |
| "8.up_proj": 0.6398731470108032, |
| "8.down_proj": 0.6440942883491516, |
| "9.q_proj": 0.4597213566303253, |
| "9.k_proj": 0.46052107214927673, |
| "9.v_proj": 0.572304368019104, |
| "9.o_proj": 0.6035570502281189, |
| "9.gate_proj": 0.5833413600921631, |
| "9.up_proj": 0.6350976824760437, |
| "9.down_proj": 0.6436678767204285, |
| "10.q_proj": 0.46521714329719543, |
| "10.k_proj": 0.478322833776474, |
| "10.v_proj": 0.5912260413169861, |
| "10.o_proj": 0.6270232796669006, |
| "10.gate_proj": 0.5851974487304688, |
| "10.up_proj": 0.6311303377151489, |
| "10.down_proj": 0.6381176710128784, |
| "11.q_proj": 0.47614750266075134, |
| "11.k_proj": 0.4854636788368225, |
| "11.v_proj": 0.5704461932182312, |
| "11.o_proj": 0.60197913646698, |
| "11.gate_proj": 0.5863650441169739, |
| "11.up_proj": 0.6343603134155273, |
| "11.down_proj": 0.6290261745452881, |
| "12.q_proj": 0.4840157628059387, |
| "12.k_proj": 0.4766708314418793, |
| "12.v_proj": 0.6087822318077087, |
| "12.o_proj": 0.6030346751213074, |
| "12.gate_proj": 0.5890772342681885, |
| "12.up_proj": 0.6384384036064148, |
| "12.down_proj": 0.6522207856178284, |
| "13.q_proj": 0.43680158257484436, |
| "13.k_proj": 0.4298385679721832, |
| "13.v_proj": 0.6261775493621826, |
| "13.o_proj": 0.5952807664871216, |
| "13.gate_proj": 0.5872403383255005, |
| "13.up_proj": 0.6357120275497437, |
| "13.down_proj": 0.6553712487220764, |
| "14.q_proj": 0.45404571294784546, |
| "14.k_proj": 0.4589211046695709, |
| "14.v_proj": 0.6460106372833252, |
| "14.o_proj": 0.6136132478713989, |
| "14.gate_proj": 0.5913381576538086, |
| "14.up_proj": 0.647808849811554, |
| "14.down_proj": 0.6647049784660339, |
| "15.q_proj": 0.45149752497673035, |
| "15.k_proj": 0.4706929326057434, |
| "15.v_proj": 0.6710425615310669, |
| "15.o_proj": 0.6365965604782104, |
| "15.gate_proj": 0.5898893475532532, |
| "15.up_proj": 0.6483412384986877, |
| "15.down_proj": 0.6726868748664856, |
| "16.q_proj": 0.49306097626686096, |
| "16.k_proj": 0.4802592396736145, |
| "16.v_proj": 0.6781746745109558, |
| "16.o_proj": 0.6331717371940613, |
| "16.gate_proj": 0.5992128849029541, |
| "16.up_proj": 0.6493324637413025, |
| "16.down_proj": 0.6702439785003662, |
| "17.q_proj": 0.4995686411857605, |
| "17.k_proj": 0.49193471670150757, |
| "17.v_proj": 0.6817148923873901, |
| "17.o_proj": 0.6546281576156616, |
| "17.gate_proj": 0.6066341996192932, |
| "17.up_proj": 0.6509790420532227, |
| "17.down_proj": 0.6842725276947021, |
| "18.q_proj": 0.5473029017448425, |
| "18.k_proj": 0.5079021453857422, |
| "18.v_proj": 0.6422337293624878, |
| "18.o_proj": 0.6411892771720886, |
| "18.gate_proj": 0.6166319251060486, |
| "18.up_proj": 0.6503247022628784, |
| "18.down_proj": 0.6794316172599792, |
| "19.q_proj": 0.5715011358261108, |
| "19.k_proj": 0.5608424544334412, |
| "19.v_proj": 0.6600127816200256, |
| "19.o_proj": 0.6745914220809937, |
| "19.gate_proj": 0.6198409795761108, |
| "19.up_proj": 0.647895872592926, |
| "19.down_proj": 0.6834138035774231, |
| "20.q_proj": 0.568836510181427, |
| "20.k_proj": 0.5440749526023865, |
| "20.v_proj": 0.6928178071975708, |
| "20.o_proj": 0.6790831685066223, |
| "20.gate_proj": 0.6240135431289673, |
| "20.up_proj": 0.648634672164917, |
| "20.down_proj": 0.6841005086898804, |
| "21.q_proj": 0.5723114609718323, |
| "21.k_proj": 0.5470589995384216, |
| "21.v_proj": 0.6950841546058655, |
| "21.o_proj": 0.6883072257041931, |
| "21.gate_proj": 0.6248385906219482, |
| "21.up_proj": 0.6486678123474121, |
| "21.down_proj": 0.6916183233261108, |
| "22.q_proj": 0.5811800956726074, |
| "22.k_proj": 0.5494594573974609, |
| "22.v_proj": 0.6604698896408081, |
| "22.o_proj": 0.6805965304374695, |
| "22.gate_proj": 0.6257846355438232, |
| "22.up_proj": 0.6461135745048523, |
| "22.down_proj": 0.6894308924674988, |
| "23.q_proj": 0.6023896336555481, |
| "23.k_proj": 0.5706185102462769, |
| "23.v_proj": 0.6682502031326294, |
| "23.o_proj": 0.6862603425979614, |
| "23.gate_proj": 0.6281367540359497, |
| "23.up_proj": 0.6450577974319458, |
| "23.down_proj": 0.6818817257881165, |
| "24.q_proj": 0.5854701995849609, |
| "24.k_proj": 0.5466554164886475, |
| "24.v_proj": 0.6416366100311279, |
| "24.o_proj": 0.6976205110549927, |
| "24.gate_proj": 0.6292465925216675, |
| "24.up_proj": 0.6437935829162598, |
| "24.down_proj": 0.6720544099807739, |
| "25.q_proj": 0.5890008211135864, |
| "25.k_proj": 0.573958694934845, |
| "25.v_proj": 0.6611844897270203, |
| "25.o_proj": 0.6952077150344849, |
| "25.gate_proj": 0.6287244558334351, |
| "25.up_proj": 0.6435160636901855, |
| "25.down_proj": 0.6599352359771729, |
| "26.q_proj": 0.5816289186477661, |
| "26.k_proj": 0.5328107476234436, |
| "26.v_proj": 0.7104170918464661, |
| "26.o_proj": 0.6926466822624207, |
| "26.gate_proj": 0.6278991103172302, |
| "26.up_proj": 0.6444916129112244, |
| "26.down_proj": 0.6558243036270142, |
| "27.q_proj": 0.5827873945236206, |
| "27.k_proj": 0.5452601313591003, |
| "27.v_proj": 0.6339237093925476, |
| "27.o_proj": 0.6614390015602112, |
| "27.gate_proj": 0.6244891881942749, |
| "27.up_proj": 0.6438120603561401, |
| "27.down_proj": 0.649568498134613, |
| "28.q_proj": 0.5894160270690918, |
| "28.k_proj": 0.5554025173187256, |
| "28.v_proj": 0.7103909254074097, |
| "28.o_proj": 0.651461124420166, |
| "28.gate_proj": 0.6244070529937744, |
| "28.up_proj": 0.6442367434501648, |
| "28.down_proj": 0.6564620733261108, |
| "29.q_proj": 0.6010609269142151, |
| "29.k_proj": 0.5750370621681213, |
| "29.v_proj": 0.6888416409492493, |
| "29.o_proj": 0.6581999063491821, |
| "29.gate_proj": 0.6179629564285278, |
| "29.up_proj": 0.639843761920929, |
| "29.down_proj": 0.6715606451034546, |
| "30.q_proj": 0.48527583479881287, |
| "30.k_proj": 0.4483523666858673, |
| "30.v_proj": 0.6323948502540588, |
| "30.o_proj": 0.6071786284446716, |
| "30.gate_proj": 0.6107315421104431, |
| "30.up_proj": 0.6275578141212463, |
| "30.down_proj": 0.6769006848335266, |
| "31.q_proj": 0.5020249485969543, |
| "31.k_proj": 0.45986607670783997, |
| "31.v_proj": 0.5856180787086487, |
| "31.o_proj": 0.606295108795166, |
| "31.gate_proj": 0.6043108105659485, |
| "31.up_proj": 0.6049404740333557, |
| "31.down_proj": 0.632754921913147 |
| }, |
| "step": "swift_svd_dynamic_finetuned", |
| "compression": { |
| "upstream": "hiahei/Swift-SVD", |
| "step": "swift_svd_dynamic", |
| "alloc": "dynamic", |
| "alpha": 0.6, |
| "nsamples": 256, |
| "seqlen": 2048, |
| "dataset": "WikiText2", |
| "seed": 42, |
| "gqa_budget": "shape", |
| "param_fraction": 0.5002723106971154, |
| "mean_rel_err": 0.5978308144424643, |
| "max_rel_err": 0.7103155851364136 |
| }, |
| "factorisation": { |
| "max_rel_err": 0.001514429459348321, |
| "factor_params": 3491561472, |
| "dense_params": 6979321856 |
| }, |
| "finetune": { |
| "protocol": "svdllm_stage2_u_then_v", |
| "script": "gitclones/SVD-LLM/utils/LoRA.py", |
| "data": "alpaca-cleaned", |
| "epochs_per_half": 2, |
| "learning_rate": 0.0001, |
| "batch_size": 64, |
| "micro_batch_size": 4, |
| "lora_r": 8, |
| "lora_alpha": 16, |
| "lora_dropout": 0.05, |
| "cutoff_len": 256, |
| "val_set_size": 2000, |
| "max_train_samples": "all", |
| "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)" |
| } |
| } |
