| { |
| "source": "/workspace/svd_safety/assets/models/Mistral-7B-Instruct-v0.2", |
| "upstream": "hiahei/Swift-SVD (compression) + AIoT-MLSys-Lab/SVD-LLM utils/LoRA.py (recovery)", |
| "stage": "finetuned", |
| "ratio_removed": 0.3, |
| "internal_ratio_kept": 0.7, |
| "from_pt": "/workspace/svd_safety/results/swift_ft/cells/mis7_swift_remove30/finetune_v/merge.pt", |
| "mean_rel_err": 0.46171893551945686, |
| "max_rel_err": 0.5801312327384949, |
| "param_fraction": 0.7003361628605769, |
| "ranks": { |
| "0.q_proj": 1613, |
| "0.k_proj": 679, |
| "0.v_proj": 617, |
| "0.o_proj": 1473, |
| "0.gate_proj": 2352, |
| "0.up_proj": 2352, |
| "0.down_proj": 2382, |
| "1.q_proj": 1619, |
| "1.k_proj": 690, |
| "1.v_proj": 650, |
| "1.o_proj": 1517, |
| "1.gate_proj": 2352, |
| "1.up_proj": 2352, |
| "1.down_proj": 2765, |
| "2.q_proj": 1319, |
| "2.k_proj": 523, |
| "2.v_proj": 513, |
| "2.o_proj": 1270, |
| "2.gate_proj": 2352, |
| "2.up_proj": 2352, |
| "2.down_proj": 1892, |
| "3.q_proj": 1316, |
| "3.k_proj": 520, |
| "3.v_proj": 508, |
| "3.o_proj": 1257, |
| "3.gate_proj": 2352, |
| "3.up_proj": 2352, |
| "3.down_proj": 1895, |
| "4.q_proj": 1314, |
| "4.k_proj": 520, |
| "4.v_proj": 512, |
| "4.o_proj": 1264, |
| "4.gate_proj": 2352, |
| "4.up_proj": 2352, |
| "4.down_proj": 1919, |
| "5.q_proj": 1313, |
| "5.k_proj": 520, |
| "5.v_proj": 513, |
| "5.o_proj": 1274, |
| "5.gate_proj": 2352, |
| "5.up_proj": 2352, |
| "5.down_proj": 1938, |
| "6.q_proj": 1315, |
| "6.k_proj": 522, |
| "6.v_proj": 514, |
| "6.o_proj": 1285, |
| "6.gate_proj": 2352, |
| "6.up_proj": 2352, |
| "6.down_proj": 1953, |
| "7.q_proj": 1313, |
| "7.k_proj": 522, |
| "7.v_proj": 515, |
| "7.o_proj": 1295, |
| "7.gate_proj": 2352, |
| "7.up_proj": 2352, |
| "7.down_proj": 1969, |
| "8.q_proj": 1310, |
| "8.k_proj": 519, |
| "8.v_proj": 515, |
| "8.o_proj": 1294, |
| "8.gate_proj": 2352, |
| "8.up_proj": 2352, |
| "8.down_proj": 1965, |
| "9.q_proj": 1308, |
| "9.k_proj": 518, |
| "9.v_proj": 516, |
| "9.o_proj": 1294, |
| "9.gate_proj": 2352, |
| "9.up_proj": 2352, |
| "9.down_proj": 1973, |
| "10.q_proj": 1309, |
| "10.k_proj": 518, |
| "10.v_proj": 516, |
| "10.o_proj": 1305, |
| "10.gate_proj": 2352, |
| "10.up_proj": 2352, |
| "10.down_proj": 1974, |
| "11.q_proj": 1307, |
| "11.k_proj": 518, |
| "11.v_proj": 518, |
| "11.o_proj": 1308, |
| "11.gate_proj": 2352, |
| "11.up_proj": 2352, |
| "11.down_proj": 1977, |
| "12.q_proj": 1315, |
| "12.k_proj": 523, |
| "12.v_proj": 522, |
| "12.o_proj": 1318, |
| "12.gate_proj": 2352, |
| "12.up_proj": 2352, |
| "12.down_proj": 1999, |
| "13.q_proj": 1311, |
| "13.k_proj": 521, |
| "13.v_proj": 520, |
| "13.o_proj": 1315, |
| "13.gate_proj": 2352, |
| "13.up_proj": 2352, |
| "13.down_proj": 1999, |
| "14.q_proj": 1321, |
| "14.k_proj": 526, |
| "14.v_proj": 529, |
| "14.o_proj": 1328, |
| "14.gate_proj": 2352, |
| "14.up_proj": 2352, |
| "14.down_proj": 2032, |
| "15.q_proj": 1336, |
| "15.k_proj": 535, |
| "15.v_proj": 538, |
| "15.o_proj": 1350, |
| "15.gate_proj": 2352, |
| "15.up_proj": 2352, |
| "15.down_proj": 2073, |
| "16.q_proj": 1343, |
| "16.k_proj": 538, |
| "16.v_proj": 540, |
| "16.o_proj": 1355, |
| "16.gate_proj": 2352, |
| "16.up_proj": 2352, |
| "16.down_proj": 2099, |
| "17.q_proj": 1370, |
| "17.k_proj": 552, |
| "17.v_proj": 555, |
| "17.o_proj": 1386, |
| "17.gate_proj": 2352, |
| "17.up_proj": 2352, |
| "17.down_proj": 2160, |
| "18.q_proj": 1369, |
| "18.k_proj": 551, |
| "18.v_proj": 556, |
| "18.o_proj": 1390, |
| "18.gate_proj": 2352, |
| "18.up_proj": 2352, |
| "18.down_proj": 2176, |
| "19.q_proj": 1351, |
| "19.k_proj": 542, |
| "19.v_proj": 547, |
| "19.o_proj": 1376, |
| "19.gate_proj": 2352, |
| "19.up_proj": 2352, |
| "19.down_proj": 2151, |
| "20.q_proj": 1318, |
| "20.k_proj": 524, |
| "20.v_proj": 530, |
| "20.o_proj": 1337, |
| "20.gate_proj": 2352, |
| "20.up_proj": 2352, |
| "20.down_proj": 2083, |
| "21.q_proj": 1310, |
| "21.k_proj": 518, |
| "21.v_proj": 526, |
| "21.o_proj": 1325, |
| "21.gate_proj": 2352, |
| "21.up_proj": 2352, |
| "21.down_proj": 2067, |
| "22.q_proj": 1297, |
| "22.k_proj": 510, |
| "22.v_proj": 519, |
| "22.o_proj": 1308, |
| "22.gate_proj": 2352, |
| "22.up_proj": 2352, |
| "22.down_proj": 2040, |
| "23.q_proj": 1298, |
| "23.k_proj": 511, |
| "23.v_proj": 519, |
| "23.o_proj": 1317, |
| "23.gate_proj": 2352, |
| "23.up_proj": 2352, |
| "23.down_proj": 2044, |
| "24.q_proj": 1297, |
| "24.k_proj": 510, |
| "24.v_proj": 519, |
| "24.o_proj": 1321, |
| "24.gate_proj": 2352, |
| "24.up_proj": 2352, |
| "24.down_proj": 2045, |
| "25.q_proj": 1300, |
| "25.k_proj": 512, |
| "25.v_proj": 523, |
| "25.o_proj": 1330, |
| "25.gate_proj": 2352, |
| "25.up_proj": 2352, |
| "25.down_proj": 2059, |
| "26.q_proj": 1306, |
| "26.k_proj": 515, |
| "26.v_proj": 528, |
| "26.o_proj": 1340, |
| "26.gate_proj": 2352, |
| "26.up_proj": 2352, |
| "26.down_proj": 2076, |
| "27.q_proj": 1311, |
| "27.k_proj": 518, |
| "27.v_proj": 530, |
| "27.o_proj": 1341, |
| "27.gate_proj": 2352, |
| "27.up_proj": 2352, |
| "27.down_proj": 2090, |
| "28.q_proj": 1328, |
| "28.k_proj": 528, |
| "28.v_proj": 541, |
| "28.o_proj": 1366, |
| "28.gate_proj": 2352, |
| "28.up_proj": 2352, |
| "28.down_proj": 2144, |
| "29.q_proj": 1339, |
| "29.k_proj": 533, |
| "29.v_proj": 548, |
| "29.o_proj": 1392, |
| "29.gate_proj": 2352, |
| "29.up_proj": 2352, |
| "29.down_proj": 2170, |
| "30.q_proj": 1359, |
| "30.k_proj": 543, |
| "30.v_proj": 561, |
| "30.o_proj": 1418, |
| "30.gate_proj": 2352, |
| "30.up_proj": 2352, |
| "30.down_proj": 2230, |
| "31.q_proj": 1473, |
| "31.k_proj": 605, |
| "31.v_proj": 626, |
| "31.o_proj": 1559, |
| "31.gate_proj": 2352, |
| "31.up_proj": 2352, |
| "31.down_proj": 2541 |
| }, |
| "rel_err": { |
| "0.q_proj": 0.08783026784658432, |
| "0.k_proj": 0.076280377805233, |
| "0.v_proj": 0.28279638290405273, |
| "0.o_proj": 0.3450925052165985, |
| "0.gate_proj": 0.3139413893222809, |
| "0.up_proj": 0.34659841656684875, |
| "0.down_proj": 0.4610687792301178, |
| "1.q_proj": 0.14944231510162354, |
| "1.k_proj": 0.08638541400432587, |
| "1.v_proj": 0.42672011256217957, |
| "1.o_proj": 0.4614779055118561, |
| "1.gate_proj": 0.37372663617134094, |
| "1.up_proj": 0.4030474126338959, |
| "1.down_proj": 0.4160686731338501, |
| "2.q_proj": 0.40826842188835144, |
| "2.k_proj": 0.38183024525642395, |
| "2.v_proj": 0.5488172769546509, |
| "2.o_proj": 0.5261715054512024, |
| "2.gate_proj": 0.44494783878326416, |
| "2.up_proj": 0.46598997712135315, |
| "2.down_proj": 0.5656055808067322, |
| "3.q_proj": 0.37075719237327576, |
| "3.k_proj": 0.3103460371494293, |
| "3.v_proj": 0.5801312327384949, |
| "3.o_proj": 0.570527195930481, |
| "3.gate_proj": 0.45506104826927185, |
| "3.up_proj": 0.48117780685424805, |
| "3.down_proj": 0.5562737584114075, |
| "4.q_proj": 0.3778054118156433, |
| "4.k_proj": 0.33330559730529785, |
| "4.v_proj": 0.5392099618911743, |
| "4.o_proj": 0.5162532329559326, |
| "4.gate_proj": 0.4575117528438568, |
| "4.up_proj": 0.49015605449676514, |
| "4.down_proj": 0.5629571676254272, |
| "5.q_proj": 0.4073948264122009, |
| "5.k_proj": 0.3724959194660187, |
| "5.v_proj": 0.5715569853782654, |
| "5.o_proj": 0.549954354763031, |
| "5.gate_proj": 0.45217984914779663, |
| "5.up_proj": 0.49232661724090576, |
| "5.down_proj": 0.5492331981658936, |
| "6.q_proj": 0.39479202032089233, |
| "6.k_proj": 0.3444615304470062, |
| "6.v_proj": 0.5562271475791931, |
| "6.o_proj": 0.5352969169616699, |
| "6.gate_proj": 0.46358194947242737, |
| "6.up_proj": 0.49562251567840576, |
| "6.down_proj": 0.5394559502601624, |
| "7.q_proj": 0.3770744204521179, |
| "7.k_proj": 0.32297107577323914, |
| "7.v_proj": 0.5488503575325012, |
| "7.o_proj": 0.5085676908493042, |
| "7.gate_proj": 0.4547872841358185, |
| "7.up_proj": 0.48921388387680054, |
| "7.down_proj": 0.5321547985076904, |
| "8.q_proj": 0.3834538757801056, |
| "8.k_proj": 0.32134145498275757, |
| "8.v_proj": 0.5088310837745667, |
| "8.o_proj": 0.4981762766838074, |
| "8.gate_proj": 0.4569913148880005, |
| "8.up_proj": 0.48666849732398987, |
| "8.down_proj": 0.5265653729438782, |
| "9.q_proj": 0.3789723515510559, |
| "9.k_proj": 0.3307971656322479, |
| "9.v_proj": 0.4929361045360565, |
| "9.o_proj": 0.5009838342666626, |
| "9.gate_proj": 0.453980952501297, |
| "9.up_proj": 0.4856830835342407, |
| "9.down_proj": 0.5116001963615417, |
| "10.q_proj": 0.3785707354545593, |
| "10.k_proj": 0.3373726010322571, |
| "10.v_proj": 0.4934564232826233, |
| "10.o_proj": 0.5008266568183899, |
| "10.gate_proj": 0.4539565443992615, |
| "10.up_proj": 0.48363447189331055, |
| "10.down_proj": 0.5184407234191895, |
| "11.q_proj": 0.3743845522403717, |
| "11.k_proj": 0.3172908425331116, |
| "11.v_proj": 0.465146541595459, |
| "11.o_proj": 0.5011467337608337, |
| "11.gate_proj": 0.4532534182071686, |
| "11.up_proj": 0.48237940669059753, |
| "11.down_proj": 0.516564130783081, |
| "12.q_proj": 0.36303064227104187, |
| "12.k_proj": 0.30660074949264526, |
| "12.v_proj": 0.5052470564842224, |
| "12.o_proj": 0.4829132556915283, |
| "12.gate_proj": 0.44975727796554565, |
| "12.up_proj": 0.47766903042793274, |
| "12.down_proj": 0.5123953223228455, |
| "13.q_proj": 0.38796108961105347, |
| "13.k_proj": 0.3449362516403198, |
| "13.v_proj": 0.5052615404129028, |
| "13.o_proj": 0.48829060792922974, |
| "13.gate_proj": 0.4554385840892792, |
| "13.up_proj": 0.47726166248321533, |
| "13.down_proj": 0.5091195106506348, |
| "14.q_proj": 0.36075472831726074, |
| "14.k_proj": 0.32303425669670105, |
| "14.v_proj": 0.5109939575195312, |
| "14.o_proj": 0.49242928624153137, |
| "14.gate_proj": 0.4523519277572632, |
| "14.up_proj": 0.48172876238822937, |
| "14.down_proj": 0.532478928565979, |
| "15.q_proj": 0.35985133051872253, |
| "15.k_proj": 0.30060359835624695, |
| "15.v_proj": 0.5368859767913818, |
| "15.o_proj": 0.49520623683929443, |
| "15.gate_proj": 0.44976216554641724, |
| "15.up_proj": 0.47884488105773926, |
| "15.down_proj": 0.527091383934021, |
| "16.q_proj": 0.3671673536300659, |
| "16.k_proj": 0.319665789604187, |
| "16.v_proj": 0.5345363020896912, |
| "16.o_proj": 0.48225852847099304, |
| "16.gate_proj": 0.4529358446598053, |
| "16.up_proj": 0.486669659614563, |
| "16.down_proj": 0.5261300206184387, |
| "17.q_proj": 0.39087575674057007, |
| "17.k_proj": 0.3505445420742035, |
| "17.v_proj": 0.5095366835594177, |
| "17.o_proj": 0.4995217025279999, |
| "17.gate_proj": 0.46473342180252075, |
| "17.up_proj": 0.496961385011673, |
| "17.down_proj": 0.5169269442558289, |
| "18.q_proj": 0.3686104118824005, |
| "18.k_proj": 0.3233947157859802, |
| "18.v_proj": 0.5597348213195801, |
| "18.o_proj": 0.47679799795150757, |
| "18.gate_proj": 0.4632626473903656, |
| "18.up_proj": 0.49456414580345154, |
| "18.down_proj": 0.5329453349113464, |
| "19.q_proj": 0.4161927103996277, |
| "19.k_proj": 0.3401549458503723, |
| "19.v_proj": 0.5778656601905823, |
| "19.o_proj": 0.49888941645622253, |
| "19.gate_proj": 0.4682750403881073, |
| "19.up_proj": 0.4958019256591797, |
| "19.down_proj": 0.548184871673584, |
| "20.q_proj": 0.438224732875824, |
| "20.k_proj": 0.380808025598526, |
| "20.v_proj": 0.5477811098098755, |
| "20.o_proj": 0.5176671147346497, |
| "20.gate_proj": 0.4731369912624359, |
| "20.up_proj": 0.49586376547813416, |
| "20.down_proj": 0.5577734112739563, |
| "21.q_proj": 0.4684925973415375, |
| "21.k_proj": 0.40865975618362427, |
| "21.v_proj": 0.522392213344574, |
| "21.o_proj": 0.5564069747924805, |
| "21.gate_proj": 0.4738660454750061, |
| "21.up_proj": 0.49745243787765503, |
| "21.down_proj": 0.5537582039833069, |
| "22.q_proj": 0.5066864490509033, |
| "22.k_proj": 0.4351329803466797, |
| "22.v_proj": 0.5337282419204712, |
| "22.o_proj": 0.5765807628631592, |
| "22.gate_proj": 0.47878557443618774, |
| "22.up_proj": 0.4989929795265198, |
| "22.down_proj": 0.5528429746627808, |
| "23.q_proj": 0.49622949957847595, |
| "23.k_proj": 0.4328460097312927, |
| "23.v_proj": 0.5585790872573853, |
| "23.o_proj": 0.554423451423645, |
| "23.gate_proj": 0.48155754804611206, |
| "23.up_proj": 0.50072181224823, |
| "23.down_proj": 0.5464255213737488, |
| "24.q_proj": 0.5014330148696899, |
| "24.k_proj": 0.45270365476608276, |
| "24.v_proj": 0.569066047668457, |
| "24.o_proj": 0.5753594040870667, |
| "24.gate_proj": 0.4825673997402191, |
| "24.up_proj": 0.5023998618125916, |
| "24.down_proj": 0.5373293161392212, |
| "25.q_proj": 0.5077980756759644, |
| "25.k_proj": 0.4391773045063019, |
| "25.v_proj": 0.5462435483932495, |
| "25.o_proj": 0.5570780634880066, |
| "25.gate_proj": 0.4843604266643524, |
| "25.up_proj": 0.5035067796707153, |
| "25.down_proj": 0.5260589122772217, |
| "26.q_proj": 0.5026710033416748, |
| "26.k_proj": 0.4264448285102844, |
| "26.v_proj": 0.5314569473266602, |
| "26.o_proj": 0.537655234336853, |
| "26.gate_proj": 0.4852832555770874, |
| "26.up_proj": 0.5033767223358154, |
| "26.down_proj": 0.5124109387397766, |
| "27.q_proj": 0.5120652914047241, |
| "27.k_proj": 0.4639483690261841, |
| "27.v_proj": 0.5471333265304565, |
| "27.o_proj": 0.5303576588630676, |
| "27.gate_proj": 0.4886907935142517, |
| "27.up_proj": 0.5048775672912598, |
| "27.down_proj": 0.5119288563728333, |
| "28.q_proj": 0.4723275899887085, |
| "28.k_proj": 0.39252132177352905, |
| "28.v_proj": 0.5004826188087463, |
| "28.o_proj": 0.5055510997772217, |
| "28.gate_proj": 0.4891432821750641, |
| "28.up_proj": 0.5027062296867371, |
| "28.down_proj": 0.5162467956542969, |
| "29.q_proj": 0.45425698161125183, |
| "29.k_proj": 0.39296162128448486, |
| "29.v_proj": 0.529412031173706, |
| "29.o_proj": 0.4873389005661011, |
| "29.gate_proj": 0.4842919409275055, |
| "29.up_proj": 0.4974598288536072, |
| "29.down_proj": 0.5207532644271851, |
| "30.q_proj": 0.4333595335483551, |
| "30.k_proj": 0.35803964734077454, |
| "30.v_proj": 0.4631883203983307, |
| "30.o_proj": 0.47360387444496155, |
| "30.gate_proj": 0.4840942919254303, |
| "30.up_proj": 0.48931118845939636, |
| "30.down_proj": 0.5056915283203125, |
| "31.q_proj": 0.38473233580589294, |
| "31.k_proj": 0.3066442310810089, |
| "31.v_proj": 0.4090970456600189, |
| "31.o_proj": 0.3971359431743622, |
| "31.gate_proj": 0.4917009174823761, |
| "31.up_proj": 0.4686800241470337, |
| "31.down_proj": 0.4488861858844757 |
| }, |
| "step": "swift_svd_dynamic_finetuned", |
| "compression": { |
| "upstream": "hiahei/Swift-SVD", |
| "step": "swift_svd_dynamic", |
| "alloc": "dynamic", |
| "alpha": 0.6, |
| "nsamples": 256, |
| "seqlen": 2048, |
| "dataset": "WikiText2", |
| "seed": 42, |
| "gqa_budget": "shape", |
| "param_fraction": 0.7003361628605769, |
| "mean_rel_err": 0.46043376145618303, |
| "max_rel_err": 0.5793784856796265 |
| }, |
| "factorisation": { |
| "max_rel_err": 0.00140141521114856, |
| "factor_params": 4887871488, |
| "dense_params": 6979321856 |
| }, |
| "finetune": { |
| "protocol": "svdllm_stage2_u_then_v", |
| "script": "gitclones/SVD-LLM/utils/LoRA.py", |
| "data": "alpaca-cleaned", |
| "epochs_per_half": 2, |
| "learning_rate": 0.0001, |
| "batch_size": 64, |
| "micro_batch_size": 4, |
| "lora_r": 8, |
| "lora_alpha": 16, |
| "lora_dropout": 0.05, |
| "cutoff_len": 256, |
| "val_set_size": 2000, |
| "max_train_samples": "all", |
| "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)" |
| } |
| } |
