| { |
| "source": "/workspace/svd_safety/assets/models/Mistral-7B-Instruct-v0.2", |
| "upstream": "hiahei/Swift-SVD (compression) + AIoT-MLSys-Lab/SVD-LLM utils/LoRA.py (recovery)", |
| "stage": "finetuned", |
| "ratio_removed": 0.5, |
| "internal_ratio_kept": 0.5, |
| "from_pt": "/workspace/svd_safety/results/swift_ft/cells/mis7_swift_remove50/finetune_v/merge.pt", |
| "mean_rel_err": 0.5888538884464651, |
| "max_rel_err": 0.7068923115730286, |
| "param_fraction": 0.5002723106971154, |
| "ranks": { |
| "0.q_proj": 1152, |
| "0.k_proj": 485, |
| "0.v_proj": 442, |
| "0.o_proj": 1052, |
| "0.gate_proj": 1680, |
| "0.up_proj": 1680, |
| "0.down_proj": 1702, |
| "1.q_proj": 1157, |
| "1.k_proj": 493, |
| "1.v_proj": 465, |
| "1.o_proj": 1084, |
| "1.gate_proj": 1680, |
| "1.up_proj": 1680, |
| "1.down_proj": 1976, |
| "2.q_proj": 942, |
| "2.k_proj": 374, |
| "2.v_proj": 367, |
| "2.o_proj": 907, |
| "2.gate_proj": 1680, |
| "2.up_proj": 1680, |
| "2.down_proj": 1351, |
| "3.q_proj": 940, |
| "3.k_proj": 372, |
| "3.v_proj": 363, |
| "3.o_proj": 898, |
| "3.gate_proj": 1680, |
| "3.up_proj": 1680, |
| "3.down_proj": 1353, |
| "4.q_proj": 938, |
| "4.k_proj": 371, |
| "4.v_proj": 366, |
| "4.o_proj": 903, |
| "4.gate_proj": 1680, |
| "4.up_proj": 1680, |
| "4.down_proj": 1371, |
| "5.q_proj": 938, |
| "5.k_proj": 372, |
| "5.v_proj": 366, |
| "5.o_proj": 910, |
| "5.gate_proj": 1680, |
| "5.up_proj": 1680, |
| "5.down_proj": 1384, |
| "6.q_proj": 939, |
| "6.k_proj": 373, |
| "6.v_proj": 368, |
| "6.o_proj": 917, |
| "6.gate_proj": 1680, |
| "6.up_proj": 1680, |
| "6.down_proj": 1395, |
| "7.q_proj": 938, |
| "7.k_proj": 373, |
| "7.v_proj": 368, |
| "7.o_proj": 925, |
| "7.gate_proj": 1680, |
| "7.up_proj": 1680, |
| "7.down_proj": 1406, |
| "8.q_proj": 935, |
| "8.k_proj": 371, |
| "8.v_proj": 368, |
| "8.o_proj": 924, |
| "8.gate_proj": 1680, |
| "8.up_proj": 1680, |
| "8.down_proj": 1403, |
| "9.q_proj": 934, |
| "9.k_proj": 370, |
| "9.v_proj": 369, |
| "9.o_proj": 924, |
| "9.gate_proj": 1680, |
| "9.up_proj": 1680, |
| "9.down_proj": 1409, |
| "10.q_proj": 935, |
| "10.k_proj": 371, |
| "10.v_proj": 369, |
| "10.o_proj": 932, |
| "10.gate_proj": 1680, |
| "10.up_proj": 1680, |
| "10.down_proj": 1410, |
| "11.q_proj": 933, |
| "11.k_proj": 370, |
| "11.v_proj": 370, |
| "11.o_proj": 934, |
| "11.gate_proj": 1680, |
| "11.up_proj": 1680, |
| "11.down_proj": 1412, |
| "12.q_proj": 939, |
| "12.k_proj": 374, |
| "12.v_proj": 373, |
| "12.o_proj": 941, |
| "12.gate_proj": 1680, |
| "12.up_proj": 1680, |
| "12.down_proj": 1428, |
| "13.q_proj": 936, |
| "13.k_proj": 372, |
| "13.v_proj": 372, |
| "13.o_proj": 939, |
| "13.gate_proj": 1680, |
| "13.up_proj": 1680, |
| "13.down_proj": 1428, |
| "14.q_proj": 944, |
| "14.k_proj": 376, |
| "14.v_proj": 378, |
| "14.o_proj": 949, |
| "14.gate_proj": 1680, |
| "14.up_proj": 1680, |
| "14.down_proj": 1451, |
| "15.q_proj": 955, |
| "15.k_proj": 383, |
| "15.v_proj": 385, |
| "15.o_proj": 964, |
| "15.gate_proj": 1680, |
| "15.up_proj": 1680, |
| "15.down_proj": 1481, |
| "16.q_proj": 959, |
| "16.k_proj": 385, |
| "16.v_proj": 387, |
| "16.o_proj": 968, |
| "16.gate_proj": 1680, |
| "16.up_proj": 1680, |
| "16.down_proj": 1500, |
| "17.q_proj": 978, |
| "17.k_proj": 395, |
| "17.v_proj": 397, |
| "17.o_proj": 990, |
| "17.gate_proj": 1680, |
| "17.up_proj": 1680, |
| "17.down_proj": 1543, |
| "18.q_proj": 978, |
| "18.k_proj": 394, |
| "18.v_proj": 398, |
| "18.o_proj": 993, |
| "18.gate_proj": 1680, |
| "18.up_proj": 1680, |
| "18.down_proj": 1555, |
| "19.q_proj": 965, |
| "19.k_proj": 388, |
| "19.v_proj": 391, |
| "19.o_proj": 983, |
| "19.gate_proj": 1680, |
| "19.up_proj": 1680, |
| "19.down_proj": 1537, |
| "20.q_proj": 942, |
| "20.k_proj": 375, |
| "20.v_proj": 379, |
| "20.o_proj": 955, |
| "20.gate_proj": 1680, |
| "20.up_proj": 1680, |
| "20.down_proj": 1488, |
| "21.q_proj": 936, |
| "21.k_proj": 371, |
| "21.v_proj": 376, |
| "21.o_proj": 947, |
| "21.gate_proj": 1680, |
| "21.up_proj": 1680, |
| "21.down_proj": 1477, |
| "22.q_proj": 926, |
| "22.k_proj": 365, |
| "22.v_proj": 371, |
| "22.o_proj": 934, |
| "22.gate_proj": 1680, |
| "22.up_proj": 1680, |
| "22.down_proj": 1457, |
| "23.q_proj": 927, |
| "23.k_proj": 365, |
| "23.v_proj": 371, |
| "23.o_proj": 940, |
| "23.gate_proj": 1680, |
| "23.up_proj": 1680, |
| "23.down_proj": 1460, |
| "24.q_proj": 926, |
| "24.k_proj": 365, |
| "24.v_proj": 371, |
| "24.o_proj": 944, |
| "24.gate_proj": 1680, |
| "24.up_proj": 1680, |
| "24.down_proj": 1461, |
| "25.q_proj": 929, |
| "25.k_proj": 366, |
| "25.v_proj": 374, |
| "25.o_proj": 951, |
| "25.gate_proj": 1680, |
| "25.up_proj": 1680, |
| "25.down_proj": 1471, |
| "26.q_proj": 933, |
| "26.k_proj": 368, |
| "26.v_proj": 377, |
| "26.o_proj": 957, |
| "26.gate_proj": 1680, |
| "26.up_proj": 1680, |
| "26.down_proj": 1483, |
| "27.q_proj": 936, |
| "27.k_proj": 370, |
| "27.v_proj": 379, |
| "27.o_proj": 958, |
| "27.gate_proj": 1680, |
| "27.up_proj": 1680, |
| "27.down_proj": 1493, |
| "28.q_proj": 949, |
| "28.k_proj": 378, |
| "28.v_proj": 387, |
| "28.o_proj": 976, |
| "28.gate_proj": 1680, |
| "28.up_proj": 1680, |
| "28.down_proj": 1532, |
| "29.q_proj": 957, |
| "29.k_proj": 381, |
| "29.v_proj": 392, |
| "29.o_proj": 994, |
| "29.gate_proj": 1680, |
| "29.up_proj": 1680, |
| "29.down_proj": 1550, |
| "30.q_proj": 971, |
| "30.k_proj": 389, |
| "30.v_proj": 401, |
| "30.o_proj": 1013, |
| "30.gate_proj": 1680, |
| "30.up_proj": 1680, |
| "30.down_proj": 1593, |
| "31.q_proj": 1053, |
| "31.k_proj": 433, |
| "31.v_proj": 448, |
| "31.o_proj": 1114, |
| "31.gate_proj": 1680, |
| "31.up_proj": 1680, |
| "31.down_proj": 1816 |
| }, |
| "rel_err": { |
| "0.q_proj": 0.10284276306629181, |
| "0.k_proj": 0.09011625498533249, |
| "0.v_proj": 0.4146844744682312, |
| "0.o_proj": 0.4441743791103363, |
| "0.gate_proj": 0.4323641359806061, |
| "0.up_proj": 0.4812891185283661, |
| "0.down_proj": 0.6020375490188599, |
| "1.q_proj": 0.1907663345336914, |
| "1.k_proj": 0.13933585584163666, |
| "1.v_proj": 0.5992577075958252, |
| "1.o_proj": 0.5850644111633301, |
| "1.gate_proj": 0.5018978714942932, |
| "1.up_proj": 0.5401790142059326, |
| "1.down_proj": 0.5769923329353333, |
| "2.q_proj": 0.5321782827377319, |
| "2.k_proj": 0.5425702929496765, |
| "2.v_proj": 0.682027280330658, |
| "2.o_proj": 0.6367513537406921, |
| "2.gate_proj": 0.5818910598754883, |
| "2.up_proj": 0.6077799201011658, |
| "2.down_proj": 0.681037187576294, |
| "3.q_proj": 0.48435133695602417, |
| "3.k_proj": 0.4475176930427551, |
| "3.v_proj": 0.7068923115730286, |
| "3.o_proj": 0.677703857421875, |
| "3.gate_proj": 0.5896062254905701, |
| "3.up_proj": 0.6225286722183228, |
| "3.down_proj": 0.6739526987075806, |
| "4.q_proj": 0.4879390597343445, |
| "4.k_proj": 0.4703967571258545, |
| "4.v_proj": 0.6710249185562134, |
| "4.o_proj": 0.6244789361953735, |
| "4.gate_proj": 0.5910874605178833, |
| "4.up_proj": 0.6321871876716614, |
| "4.down_proj": 0.6802555918693542, |
| "5.q_proj": 0.5210317969322205, |
| "5.k_proj": 0.5119411945343018, |
| "5.v_proj": 0.6989429593086243, |
| "5.o_proj": 0.6632655262947083, |
| "5.gate_proj": 0.5846965312957764, |
| "5.up_proj": 0.6342030763626099, |
| "5.down_proj": 0.6687344908714294, |
| "6.q_proj": 0.505556583404541, |
| "6.k_proj": 0.48692795634269714, |
| "6.v_proj": 0.6872745752334595, |
| "6.o_proj": 0.646428108215332, |
| "6.gate_proj": 0.5995981693267822, |
| "6.up_proj": 0.6370501518249512, |
| "6.down_proj": 0.6577861309051514, |
| "7.q_proj": 0.4832315444946289, |
| "7.k_proj": 0.4549109935760498, |
| "7.v_proj": 0.6812716722488403, |
| "7.o_proj": 0.6252717971801758, |
| "7.gate_proj": 0.5871505737304688, |
| "7.up_proj": 0.6294843554496765, |
| "7.down_proj": 0.654327392578125, |
| "8.q_proj": 0.49038824439048767, |
| "8.k_proj": 0.4526974856853485, |
| "8.v_proj": 0.6452953219413757, |
| "8.o_proj": 0.6178920269012451, |
| "8.gate_proj": 0.5910043716430664, |
| "8.up_proj": 0.6263056993484497, |
| "8.down_proj": 0.647315502166748, |
| "9.q_proj": 0.48501771688461304, |
| "9.k_proj": 0.4705408811569214, |
| "9.v_proj": 0.6292407512664795, |
| "9.o_proj": 0.6174561977386475, |
| "9.gate_proj": 0.5865891575813293, |
| "9.up_proj": 0.6233162879943848, |
| "9.down_proj": 0.6318148970603943, |
| "10.q_proj": 0.4836317300796509, |
| "10.k_proj": 0.4697504937648773, |
| "10.v_proj": 0.6263429522514343, |
| "10.o_proj": 0.6223848462104797, |
| "10.gate_proj": 0.5854211449623108, |
| "10.up_proj": 0.6197803020477295, |
| "10.down_proj": 0.6372787356376648, |
| "11.q_proj": 0.4799323081970215, |
| "11.k_proj": 0.4386899471282959, |
| "11.v_proj": 0.6085149049758911, |
| "11.o_proj": 0.6194866895675659, |
| "11.gate_proj": 0.585040271282196, |
| "11.up_proj": 0.6186556816101074, |
| "11.down_proj": 0.6358811259269714, |
| "12.q_proj": 0.46628451347351074, |
| "12.k_proj": 0.4467337131500244, |
| "12.v_proj": 0.6398075222969055, |
| "12.o_proj": 0.602120578289032, |
| "12.gate_proj": 0.5809478759765625, |
| "12.up_proj": 0.6136550307273865, |
| "12.down_proj": 0.6333811283111572, |
| "13.q_proj": 0.49495774507522583, |
| "13.k_proj": 0.4739055931568146, |
| "13.v_proj": 0.6350283026695251, |
| "13.o_proj": 0.6104384660720825, |
| "13.gate_proj": 0.5884830951690674, |
| "13.up_proj": 0.6139264702796936, |
| "13.down_proj": 0.6283634305000305, |
| "14.q_proj": 0.4639151692390442, |
| "14.k_proj": 0.44637247920036316, |
| "14.v_proj": 0.6475556492805481, |
| "14.o_proj": 0.6104862093925476, |
| "14.gate_proj": 0.5859494209289551, |
| "14.up_proj": 0.6189398169517517, |
| "14.down_proj": 0.6564309597015381, |
| "15.q_proj": 0.46387919783592224, |
| "15.k_proj": 0.42992931604385376, |
| "15.v_proj": 0.675900399684906, |
| "15.o_proj": 0.6175917387008667, |
| "15.gate_proj": 0.5837653279304504, |
| "15.up_proj": 0.6173660159111023, |
| "15.down_proj": 0.6548573970794678, |
| "16.q_proj": 0.4708837568759918, |
| "16.k_proj": 0.4432294964790344, |
| "16.v_proj": 0.6696433424949646, |
| "16.o_proj": 0.6041324138641357, |
| "16.gate_proj": 0.5891684293746948, |
| "16.up_proj": 0.626923680305481, |
| "16.down_proj": 0.656101644039154, |
| "17.q_proj": 0.5025116801261902, |
| "17.k_proj": 0.48880496621131897, |
| "17.v_proj": 0.6514131426811218, |
| "17.o_proj": 0.6207662224769592, |
| "17.gate_proj": 0.6006857752799988, |
| "17.up_proj": 0.6366637349128723, |
| "17.down_proj": 0.6493645906448364, |
| "18.q_proj": 0.4747740626335144, |
| "18.k_proj": 0.4571532607078552, |
| "18.v_proj": 0.6963456869125366, |
| "18.o_proj": 0.5963596701622009, |
| "18.gate_proj": 0.6004551649093628, |
| "18.up_proj": 0.6350386142730713, |
| "18.down_proj": 0.6672616004943848, |
| "19.q_proj": 0.5268400311470032, |
| "19.k_proj": 0.47958895564079285, |
| "19.v_proj": 0.7042666077613831, |
| "19.o_proj": 0.6199771165847778, |
| "19.gate_proj": 0.6060331463813782, |
| "19.up_proj": 0.6367842555046082, |
| "19.down_proj": 0.6811782121658325, |
| "20.q_proj": 0.5465222001075745, |
| "20.k_proj": 0.5119665265083313, |
| "20.v_proj": 0.6899744272232056, |
| "20.o_proj": 0.6347531080245972, |
| "20.gate_proj": 0.6125325560569763, |
| "20.up_proj": 0.636976957321167, |
| "20.down_proj": 0.6877227425575256, |
| "21.q_proj": 0.5809774398803711, |
| "21.k_proj": 0.5469288229942322, |
| "21.v_proj": 0.6610574722290039, |
| "21.o_proj": 0.6689409017562866, |
| "21.gate_proj": 0.6139337420463562, |
| "21.up_proj": 0.6391165256500244, |
| "21.down_proj": 0.6832785606384277, |
| "22.q_proj": 0.6217024922370911, |
| "22.k_proj": 0.5693233609199524, |
| "22.v_proj": 0.6632723808288574, |
| "22.o_proj": 0.6856058835983276, |
| "22.gate_proj": 0.6192406415939331, |
| "22.up_proj": 0.6405629515647888, |
| "22.down_proj": 0.6809287667274475, |
| "23.q_proj": 0.6093561053276062, |
| "23.k_proj": 0.5737671256065369, |
| "23.v_proj": 0.6871448159217834, |
| "23.o_proj": 0.6642407178878784, |
| "23.gate_proj": 0.6219133138656616, |
| "23.up_proj": 0.6424050331115723, |
| "23.down_proj": 0.6742445826530457, |
| "24.q_proj": 0.6159307360649109, |
| "24.k_proj": 0.5868129134178162, |
| "24.v_proj": 0.6966824531555176, |
| "24.o_proj": 0.6866542100906372, |
| "24.gate_proj": 0.6238306760787964, |
| "24.up_proj": 0.6446942687034607, |
| "24.down_proj": 0.6646919250488281, |
| "25.q_proj": 0.6210862994194031, |
| "25.k_proj": 0.5705580115318298, |
| "25.v_proj": 0.679048478603363, |
| "25.o_proj": 0.6676820516586304, |
| "25.gate_proj": 0.6258559823036194, |
| "25.up_proj": 0.6465917825698853, |
| "25.down_proj": 0.6531879901885986, |
| "26.q_proj": 0.6165540218353271, |
| "26.k_proj": 0.5607565641403198, |
| "26.v_proj": 0.6618245840072632, |
| "26.o_proj": 0.6507518887519836, |
| "26.gate_proj": 0.6270332336425781, |
| "26.up_proj": 0.6463326811790466, |
| "26.down_proj": 0.6390087008476257, |
| "27.q_proj": 0.6284988522529602, |
| "27.k_proj": 0.6069222092628479, |
| "27.v_proj": 0.6805939674377441, |
| "27.o_proj": 0.6412052512168884, |
| "27.gate_proj": 0.6305440664291382, |
| "27.up_proj": 0.6479936242103577, |
| "27.down_proj": 0.6380788683891296, |
| "28.q_proj": 0.5876466035842896, |
| "28.k_proj": 0.5294674634933472, |
| "28.v_proj": 0.6338029503822327, |
| "28.o_proj": 0.6242429614067078, |
| "28.gate_proj": 0.6301671266555786, |
| "28.up_proj": 0.6454077959060669, |
| "28.down_proj": 0.6476228833198547, |
| "29.q_proj": 0.5711690783500671, |
| "29.k_proj": 0.5322142243385315, |
| "29.v_proj": 0.670903205871582, |
| "29.o_proj": 0.6061508655548096, |
| "29.gate_proj": 0.6246752142906189, |
| "29.up_proj": 0.6391395926475525, |
| "29.down_proj": 0.6555037498474121, |
| "30.q_proj": 0.5478631854057312, |
| "30.k_proj": 0.4874037206172943, |
| "30.v_proj": 0.6046014428138733, |
| "30.o_proj": 0.5960355997085571, |
| "30.gate_proj": 0.6242106556892395, |
| "30.up_proj": 0.6300148367881775, |
| "30.down_proj": 0.6422404646873474, |
| "31.q_proj": 0.49975186586380005, |
| "31.k_proj": 0.44580137729644775, |
| "31.v_proj": 0.5610269904136658, |
| "31.o_proj": 0.5210819840431213, |
| "31.gate_proj": 0.6289830803871155, |
| "31.up_proj": 0.6037687659263611, |
| "31.down_proj": 0.6013435125350952 |
| }, |
| "step": "swift_svd_dynamic_finetuned", |
| "compression": { |
| "upstream": "hiahei/Swift-SVD", |
| "step": "swift_svd_dynamic", |
| "alloc": "dynamic", |
| "alpha": 0.6, |
| "nsamples": 256, |
| "seqlen": 2048, |
| "dataset": "WikiText2", |
| "seed": 42, |
| "gqa_budget": "shape", |
| "param_fraction": 0.5002723106971154, |
| "mean_rel_err": 0.5875181926946554, |
| "max_rel_err": 0.7061887383460999 |
| }, |
| "factorisation": { |
| "max_rel_err": 0.0014601770089939237, |
| "factor_params": 3491561472, |
| "dense_params": 6979321856 |
| }, |
| "finetune": { |
| "protocol": "svdllm_stage2_u_then_v", |
| "script": "gitclones/SVD-LLM/utils/LoRA.py", |
| "data": "alpaca-cleaned", |
| "epochs_per_half": 2, |
| "learning_rate": 0.0001, |
| "batch_size": 64, |
| "micro_batch_size": 4, |
| "lora_r": 8, |
| "lora_alpha": 16, |
| "lora_dropout": 0.05, |
| "cutoff_len": 256, |
| "val_set_size": 2000, |
| "max_train_samples": "all", |
| "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)" |
| } |
| } |
