| { |
| "source": "/workspace/svd_safety/assets/models/Llama-2-7b-chat-hf", |
| "upstream": "TUDa-HWAI/Basis_Sharing", |
| "step": "basis_sharing", |
| "ratio_removed": 0.3, |
| "group_size": 2, |
| "nsamples": 256, |
| "seqlen": 2048, |
| "seed": 42, |
| "share_part": [ |
| "v", |
| "k", |
| "q", |
| "up", |
| "gate" |
| ], |
| "private_part": [ |
| "down", |
| "o" |
| ], |
| "factored": "/workspace/svd_safety/results/basis_sharing/cells/l2_basis_remove30/truncated/factored.pt", |
| "param_fraction": 0.6998272040965026, |
| "ranks": { |
| "0.q_proj": 1911, |
| "0.k_proj": 1911, |
| "0.v_proj": 1911, |
| "0.o_proj": 1433, |
| "0.gate_proj": 2417, |
| "0.up_proj": 2417, |
| "0.down_proj": 2089, |
| "1.q_proj": 1911, |
| "1.k_proj": 1911, |
| "1.v_proj": 1911, |
| "1.o_proj": 1433, |
| "1.gate_proj": 2417, |
| "1.up_proj": 2417, |
| "1.down_proj": 2089, |
| "2.q_proj": 1911, |
| "2.k_proj": 1911, |
| "2.v_proj": 1911, |
| "2.o_proj": 1433, |
| "2.gate_proj": 2417, |
| "2.up_proj": 2417, |
| "2.down_proj": 2089, |
| "3.q_proj": 1911, |
| "3.k_proj": 1911, |
| "3.v_proj": 1911, |
| "3.o_proj": 1433, |
| "3.gate_proj": 2417, |
| "3.up_proj": 2417, |
| "3.down_proj": 2089, |
| "4.q_proj": 1911, |
| "4.k_proj": 1911, |
| "4.v_proj": 1911, |
| "4.o_proj": 1433, |
| "4.gate_proj": 2417, |
| "4.up_proj": 2417, |
| "4.down_proj": 2089, |
| "5.q_proj": 1911, |
| "5.k_proj": 1911, |
| "5.v_proj": 1911, |
| "5.o_proj": 1433, |
| "5.gate_proj": 2417, |
| "5.up_proj": 2417, |
| "5.down_proj": 2089, |
| "6.q_proj": 1911, |
| "6.k_proj": 1911, |
| "6.v_proj": 1911, |
| "6.o_proj": 1433, |
| "6.gate_proj": 2417, |
| "6.up_proj": 2417, |
| "6.down_proj": 2089, |
| "7.q_proj": 1911, |
| "7.k_proj": 1911, |
| "7.v_proj": 1911, |
| "7.o_proj": 1433, |
| "7.gate_proj": 2417, |
| "7.up_proj": 2417, |
| "7.down_proj": 2089, |
| "8.q_proj": 1911, |
| "8.k_proj": 1911, |
| "8.v_proj": 1911, |
| "8.o_proj": 1433, |
| "8.gate_proj": 2417, |
| "8.up_proj": 2417, |
| "8.down_proj": 2089, |
| "9.q_proj": 1911, |
| "9.k_proj": 1911, |
| "9.v_proj": 1911, |
| "9.o_proj": 1433, |
| "9.gate_proj": 2417, |
| "9.up_proj": 2417, |
| "9.down_proj": 2089, |
| "10.q_proj": 1911, |
| "10.k_proj": 1911, |
| "10.v_proj": 1911, |
| "10.o_proj": 1433, |
| "10.gate_proj": 2417, |
| "10.up_proj": 2417, |
| "10.down_proj": 2089, |
| "11.q_proj": 1911, |
| "11.k_proj": 1911, |
| "11.v_proj": 1911, |
| "11.o_proj": 1433, |
| "11.gate_proj": 2417, |
| "11.up_proj": 2417, |
| "11.down_proj": 2089, |
| "12.q_proj": 1911, |
| "12.k_proj": 1911, |
| "12.v_proj": 1911, |
| "12.o_proj": 1433, |
| "12.gate_proj": 2417, |
| "12.up_proj": 2417, |
| "12.down_proj": 2089, |
| "13.q_proj": 1911, |
| "13.k_proj": 1911, |
| "13.v_proj": 1911, |
| "13.o_proj": 1433, |
| "13.gate_proj": 2417, |
| "13.up_proj": 2417, |
| "13.down_proj": 2089, |
| "14.q_proj": 1911, |
| "14.k_proj": 1911, |
| "14.v_proj": 1911, |
| "14.o_proj": 1433, |
| "14.gate_proj": 2417, |
| "14.up_proj": 2417, |
| "14.down_proj": 2089, |
| "15.q_proj": 1911, |
| "15.k_proj": 1911, |
| "15.v_proj": 1911, |
| "15.o_proj": 1433, |
| "15.gate_proj": 2417, |
| "15.up_proj": 2417, |
| "15.down_proj": 2089, |
| "16.q_proj": 1911, |
| "16.k_proj": 1911, |
| "16.v_proj": 1911, |
| "16.o_proj": 1433, |
| "16.gate_proj": 2417, |
| "16.up_proj": 2417, |
| "16.down_proj": 2089, |
| "17.q_proj": 1911, |
| "17.k_proj": 1911, |
| "17.v_proj": 1911, |
| "17.o_proj": 1433, |
| "17.gate_proj": 2417, |
| "17.up_proj": 2417, |
| "17.down_proj": 2089, |
| "18.q_proj": 1911, |
| "18.k_proj": 1911, |
| "18.v_proj": 1911, |
| "18.o_proj": 1433, |
| "18.gate_proj": 2417, |
| "18.up_proj": 2417, |
| "18.down_proj": 2089, |
| "19.q_proj": 1911, |
| "19.k_proj": 1911, |
| "19.v_proj": 1911, |
| "19.o_proj": 1433, |
| "19.gate_proj": 2417, |
| "19.up_proj": 2417, |
| "19.down_proj": 2089, |
| "20.q_proj": 1911, |
| "20.k_proj": 1911, |
| "20.v_proj": 1911, |
| "20.o_proj": 1433, |
| "20.gate_proj": 2417, |
| "20.up_proj": 2417, |
| "20.down_proj": 2089, |
| "21.q_proj": 1911, |
| "21.k_proj": 1911, |
| "21.v_proj": 1911, |
| "21.o_proj": 1433, |
| "21.gate_proj": 2417, |
| "21.up_proj": 2417, |
| "21.down_proj": 2089, |
| "22.q_proj": 1911, |
| "22.k_proj": 1911, |
| "22.v_proj": 1911, |
| "22.o_proj": 1433, |
| "22.gate_proj": 2417, |
| "22.up_proj": 2417, |
| "22.down_proj": 2089, |
| "23.q_proj": 1911, |
| "23.k_proj": 1911, |
| "23.v_proj": 1911, |
| "23.o_proj": 1433, |
| "23.gate_proj": 2417, |
| "23.up_proj": 2417, |
| "23.down_proj": 2089, |
| "24.q_proj": 1911, |
| "24.k_proj": 1911, |
| "24.v_proj": 1911, |
| "24.o_proj": 1433, |
| "24.gate_proj": 2417, |
| "24.up_proj": 2417, |
| "24.down_proj": 2089, |
| "25.q_proj": 1911, |
| "25.k_proj": 1911, |
| "25.v_proj": 1911, |
| "25.o_proj": 1433, |
| "25.gate_proj": 2417, |
| "25.up_proj": 2417, |
| "25.down_proj": 2089, |
| "26.q_proj": 1911, |
| "26.k_proj": 1911, |
| "26.v_proj": 1911, |
| "26.o_proj": 1433, |
| "26.gate_proj": 2417, |
| "26.up_proj": 2417, |
| "26.down_proj": 2089, |
| "27.q_proj": 1911, |
| "27.k_proj": 1911, |
| "27.v_proj": 1911, |
| "27.o_proj": 1433, |
| "27.gate_proj": 2417, |
| "27.up_proj": 2417, |
| "27.down_proj": 2089, |
| "28.q_proj": 1911, |
| "28.k_proj": 1911, |
| "28.v_proj": 1911, |
| "28.o_proj": 1433, |
| "28.gate_proj": 2417, |
| "28.up_proj": 2417, |
| "28.down_proj": 2089, |
| "29.q_proj": 1911, |
| "29.k_proj": 1911, |
| "29.v_proj": 1911, |
| "29.o_proj": 1433, |
| "29.gate_proj": 2417, |
| "29.up_proj": 2417, |
| "29.down_proj": 2089, |
| "30.q_proj": 1911, |
| "30.k_proj": 1911, |
| "30.v_proj": 1911, |
| "30.o_proj": 1433, |
| "30.gate_proj": 2417, |
| "30.up_proj": 2417, |
| "30.down_proj": 2089, |
| "31.q_proj": 1911, |
| "31.k_proj": 1911, |
| "31.v_proj": 1911, |
| "31.o_proj": 1433, |
| "31.gate_proj": 2417, |
| "31.up_proj": 2417, |
| "31.down_proj": 2089 |
| }, |
| "stage": "basis_recovery_finetuned", |
| "recovery_finetune": { |
| "protocol": "basis_sharing_coefficient_lora", |
| "script": "gitclones/SVD-LLM/utils/LoRA.py", |
| "data_path": "alpaca-cleaned (/workspace/svd_safety/data/alpaca_cleaned.json)", |
| "epochs": 2, |
| "learning_rate": 0.0001, |
| "batch_size": 64, |
| "micro_batch_size": 4, |
| "cutoff_len": 256, |
| "val_set_size": 2000, |
| "max_train_samples": "all", |
| "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)", |
| "is_methods_own_stage2": false, |
| "note": "Basis Sharing publishes its own LoRA (lora.py: wikitext, batch 1, peft default q/v coefficient targets). This cell uses the project's SVD-LLM alpaca recipe so recovery data is constant across compressors, on all seven coefficient types and never on a basis.", |
| "lora_r": 8, |
| "lora_alpha": 16, |
| "lora_dropout": 0.05, |
| "lora_targets": [ |
| "down_proj", |
| "gate_proj", |
| "k_proj", |
| "o_proj", |
| "q_proj", |
| "up_proj", |
| "v_proj" |
| ], |
| "trained": "Coefficient modules only; shared and private bases frozen", |
| "merge": "C' = C + (alpha/r) B_lora A_lora, then W = C' @ basis", |
| "mean_rel_delta_coefficient": 0.026299988264716894 |
| }, |
| "param_fraction_is_upper_bound": false, |
| "rel_delta_coefficient": { |
| "0.down_proj": 0.01571832410991192, |
| "0.gate_proj": 0.07073784619569778, |
| "0.up_proj": 0.051439255475997925, |
| "0.k_proj": 0.03125172480940819, |
| "0.o_proj": 0.014449120499193668, |
| "0.q_proj": 0.03769339993596077, |
| "0.v_proj": 0.019454704597592354, |
| "1.down_proj": 0.007780988700687885, |
| "1.gate_proj": 0.05042870715260506, |
| "1.up_proj": 0.04310716688632965, |
| "1.k_proj": 0.01711668074131012, |
| "1.o_proj": 0.013755874708294868, |
| "1.q_proj": 0.01987244002521038, |
| "1.v_proj": 0.015976836904883385, |
| "2.down_proj": 0.01568828709423542, |
| "2.gate_proj": 0.04502059519290924, |
| "2.up_proj": 0.04337015375494957, |
| "2.k_proj": 0.01923982799053192, |
| "2.o_proj": 0.018201353028416634, |
| "2.q_proj": 0.019387632608413696, |
| "2.v_proj": 0.025767281651496887, |
| "3.down_proj": 0.015289364382624626, |
| "3.gate_proj": 0.03861368075013161, |
| "3.up_proj": 0.040375787764787674, |
| "3.k_proj": 0.019376985728740692, |
| "3.o_proj": 0.017843443900346756, |
| "3.q_proj": 0.01794094406068325, |
| "3.v_proj": 0.025516241788864136, |
| "4.down_proj": 0.013919559307396412, |
| "4.gate_proj": 0.03799600154161453, |
| "4.up_proj": 0.03619149327278137, |
| "4.k_proj": 0.021520331501960754, |
| "4.o_proj": 0.017562542110681534, |
| "4.q_proj": 0.019273044541478157, |
| "4.v_proj": 0.02358822524547577, |
| "5.down_proj": 0.014357398264110088, |
| "5.gate_proj": 0.03674551099538803, |
| "5.up_proj": 0.033642545342445374, |
| "5.k_proj": 0.019489075988531113, |
| "5.o_proj": 0.01640571840107441, |
| "5.q_proj": 0.017913563176989555, |
| "5.v_proj": 0.02251928113400936, |
| "6.down_proj": 0.013552287593483925, |
| "6.gate_proj": 0.037070877850055695, |
| "6.up_proj": 0.03368235006928444, |
| "6.k_proj": 0.018359050154685974, |
| "6.o_proj": 0.01698077842593193, |
| "6.q_proj": 0.018099360167980194, |
| "6.v_proj": 0.024023892357945442, |
| "7.down_proj": 0.014833086170256138, |
| "7.gate_proj": 0.03736833110451698, |
| "7.up_proj": 0.03326217830181122, |
| "7.k_proj": 0.01821022666990757, |
| "7.o_proj": 0.014988509006798267, |
| "7.q_proj": 0.017297787591814995, |
| "7.v_proj": 0.022407837212085724, |
| "8.down_proj": 0.014573442749679089, |
| "8.gate_proj": 0.03457353264093399, |
| "8.up_proj": 0.033223461359739304, |
| "8.k_proj": 0.01891668699681759, |
| "8.o_proj": 0.016290225088596344, |
| "8.q_proj": 0.018463721498847008, |
| "8.v_proj": 0.020800800994038582, |
| "9.down_proj": 0.013626599684357643, |
| "9.gate_proj": 0.03514138609170914, |
| "9.up_proj": 0.03454797342419624, |
| "9.k_proj": 0.01954204961657524, |
| "9.o_proj": 0.01583862490952015, |
| "9.q_proj": 0.018615638837218285, |
| "9.v_proj": 0.020167481154203415, |
| "10.down_proj": 0.014242639765143394, |
| "10.gate_proj": 0.036446455866098404, |
| "10.up_proj": 0.03395068645477295, |
| "10.k_proj": 0.019043728709220886, |
| "10.o_proj": 0.016665460541844368, |
| "10.q_proj": 0.018057456240057945, |
| "10.v_proj": 0.019024686887860298, |
| "11.down_proj": 0.015268338844180107, |
| "11.gate_proj": 0.034873563796281815, |
| "11.up_proj": 0.03515440225601196, |
| "11.k_proj": 0.020495600998401642, |
| "11.o_proj": 0.01653156615793705, |
| "11.q_proj": 0.01963174156844616, |
| "11.v_proj": 0.018966326490044594, |
| "12.down_proj": 0.01490208599716425, |
| "12.gate_proj": 0.036391180008649826, |
| "12.up_proj": 0.03552509844303131, |
| "12.k_proj": 0.020210448652505875, |
| "12.o_proj": 0.015827499330043793, |
| "12.q_proj": 0.01948561705648899, |
| "12.v_proj": 0.020654533058404922, |
| "13.down_proj": 0.01405719481408596, |
| "13.gate_proj": 0.03551631048321724, |
| "13.up_proj": 0.037898264825344086, |
| "13.k_proj": 0.020987654104828835, |
| "13.o_proj": 0.016064412891864777, |
| "13.q_proj": 0.019807111471891403, |
| "13.v_proj": 0.01962696947157383, |
| "14.down_proj": 0.016009340062737465, |
| "14.gate_proj": 0.03802134096622467, |
| "14.up_proj": 0.03786147013306618, |
| "14.k_proj": 0.021262750029563904, |
| "14.o_proj": 0.017347760498523712, |
| "14.q_proj": 0.020732464268803596, |
| "14.v_proj": 0.02084069512784481, |
| "15.down_proj": 0.016418438404798508, |
| "15.gate_proj": 0.038650088012218475, |
| "15.up_proj": 0.035394005477428436, |
| "15.k_proj": 0.022153930738568306, |
| "15.o_proj": 0.01559238601475954, |
| "15.q_proj": 0.02090977504849434, |
| "15.v_proj": 0.02013670839369297, |
| "16.down_proj": 0.016190309077501297, |
| "16.gate_proj": 0.040097519755363464, |
| "16.up_proj": 0.03531859815120697, |
| "16.k_proj": 0.024484077468514442, |
| "16.o_proj": 0.018780086189508438, |
| "16.q_proj": 0.02221532352268696, |
| "16.v_proj": 0.02362828142940998, |
| "17.down_proj": 0.017163388431072235, |
| "17.gate_proj": 0.039196014404296875, |
| "17.up_proj": 0.03838655725121498, |
| "17.k_proj": 0.023485826328396797, |
| "17.o_proj": 0.01819983869791031, |
| "17.q_proj": 0.024406565353274345, |
| "17.v_proj": 0.023620449006557465, |
| "18.down_proj": 0.017160629853606224, |
| "18.gate_proj": 0.038988612592220306, |
| "18.up_proj": 0.04259044677019119, |
| "18.k_proj": 0.024072891101241112, |
| "18.o_proj": 0.019031178206205368, |
| "18.q_proj": 0.024145307019352913, |
| "18.v_proj": 0.02690170519053936, |
| "19.down_proj": 0.01757754199206829, |
| "19.gate_proj": 0.04096819460391998, |
| "19.up_proj": 0.03828761354088783, |
| "19.k_proj": 0.025333957746624947, |
| "19.o_proj": 0.01932433247566223, |
| "19.q_proj": 0.024755822494626045, |
| "19.v_proj": 0.02619953081011772, |
| "20.down_proj": 0.017618484795093536, |
| "20.gate_proj": 0.04168369621038437, |
| "20.up_proj": 0.04031258076429367, |
| "20.k_proj": 0.025555750355124474, |
| "20.o_proj": 0.019319145008921623, |
| "20.q_proj": 0.026066727936267853, |
| "20.v_proj": 0.0259451512247324, |
| "21.down_proj": 0.018069017678499222, |
| "21.gate_proj": 0.03980571776628494, |
| "21.up_proj": 0.04347342625260353, |
| "21.k_proj": 0.026453960686922073, |
| "21.o_proj": 0.01866300217807293, |
| "21.q_proj": 0.025375496596097946, |
| "21.v_proj": 0.027465440332889557, |
| "22.down_proj": 0.01788206212222576, |
| "22.gate_proj": 0.04741738364100456, |
| "22.up_proj": 0.04643930122256279, |
| "22.k_proj": 0.026044832542538643, |
| "22.o_proj": 0.018997332081198692, |
| "22.q_proj": 0.025562603026628494, |
| "22.v_proj": 0.029967913404107094, |
| "23.down_proj": 0.018382783979177475, |
| "23.gate_proj": 0.04516542702913284, |
| "23.up_proj": 0.04480283334851265, |
| "23.k_proj": 0.025396348908543587, |
| "23.o_proj": 0.016814321279525757, |
| "23.q_proj": 0.025049729272723198, |
| "23.v_proj": 0.0272342711687088, |
| "24.down_proj": 0.018060794100165367, |
| "24.gate_proj": 0.045451823621988297, |
| "24.up_proj": 0.04517007991671562, |
| "24.k_proj": 0.024571320042014122, |
| "24.o_proj": 0.015839651226997375, |
| "24.q_proj": 0.02506706677377224, |
| "24.v_proj": 0.030967289581894875, |
| "25.down_proj": 0.017680563032627106, |
| "25.gate_proj": 0.04438598453998566, |
| "25.up_proj": 0.04630052670836449, |
| "25.k_proj": 0.026787467300891876, |
| "25.o_proj": 0.018963590264320374, |
| "25.q_proj": 0.024902934208512306, |
| "25.v_proj": 0.02655676379799843, |
| "26.down_proj": 0.01728445664048195, |
| "26.gate_proj": 0.04320848360657692, |
| "26.up_proj": 0.04594198614358902, |
| "26.k_proj": 0.029789509251713753, |
| "26.o_proj": 0.01576789654791355, |
| "26.q_proj": 0.02549917809665203, |
| "26.v_proj": 0.027597865089774132, |
| "27.down_proj": 0.016805997118353844, |
| "27.gate_proj": 0.04197928309440613, |
| "27.up_proj": 0.04444769397377968, |
| "27.k_proj": 0.024795103818178177, |
| "27.o_proj": 0.023305708542466164, |
| "27.q_proj": 0.026696208864450455, |
| "27.v_proj": 0.02590230293571949, |
| "28.down_proj": 0.01710508018732071, |
| "28.gate_proj": 0.04344693571329117, |
| "28.up_proj": 0.041666075587272644, |
| "28.k_proj": 0.02632494829595089, |
| "28.o_proj": 0.017413025721907616, |
| "28.q_proj": 0.023984916508197784, |
| "28.v_proj": 0.029418673366308212, |
| "29.down_proj": 0.01671851985156536, |
| "29.gate_proj": 0.0447494238615036, |
| "29.up_proj": 0.04233136773109436, |
| "29.k_proj": 0.028628991916775703, |
| "29.o_proj": 0.01745348423719406, |
| "29.q_proj": 0.02534261718392372, |
| "29.v_proj": 0.026935601606965065, |
| "30.down_proj": 0.013234722428023815, |
| "30.gate_proj": 0.04196500778198242, |
| "30.up_proj": 0.0402400940656662, |
| "30.k_proj": 0.029129525646567345, |
| "30.o_proj": 0.018144818022847176, |
| "30.q_proj": 0.02681029587984085, |
| "30.v_proj": 0.019666632637381554, |
| "31.down_proj": 0.012836103327572346, |
| "31.gate_proj": 0.04107679799199104, |
| "31.up_proj": 0.038529664278030396, |
| "31.k_proj": 0.033828120678663254, |
| "31.o_proj": 0.016145970672369003, |
| "31.q_proj": 0.03200652077794075, |
| "31.v_proj": 0.022225258871912956 |
| } |
| } |
