| { |
| "source": "/workspace/svd_safety/assets/models/Llama-2-7b-chat-hf", |
| "upstream": "TUDa-HWAI/Basis_Sharing", |
| "step": "basis_sharing", |
| "ratio_removed": 0.4, |
| "group_size": 2, |
| "nsamples": 256, |
| "seqlen": 2048, |
| "seed": 42, |
| "share_part": [ |
| "v", |
| "k", |
| "q", |
| "up", |
| "gate" |
| ], |
| "private_part": [ |
| "down", |
| "o" |
| ], |
| "factored": "/workspace/svd_safety/results/basis_sharing/cells/l2_basis_remove40/truncated/factored.pt", |
| "param_fraction": 0.5999091746275906, |
| "ranks": { |
| "0.q_proj": 1638, |
| "0.k_proj": 1638, |
| "0.v_proj": 1638, |
| "0.o_proj": 1228, |
| "0.gate_proj": 2072, |
| "0.up_proj": 2072, |
| "0.down_proj": 1791, |
| "1.q_proj": 1638, |
| "1.k_proj": 1638, |
| "1.v_proj": 1638, |
| "1.o_proj": 1228, |
| "1.gate_proj": 2072, |
| "1.up_proj": 2072, |
| "1.down_proj": 1791, |
| "2.q_proj": 1638, |
| "2.k_proj": 1638, |
| "2.v_proj": 1638, |
| "2.o_proj": 1228, |
| "2.gate_proj": 2072, |
| "2.up_proj": 2072, |
| "2.down_proj": 1791, |
| "3.q_proj": 1638, |
| "3.k_proj": 1638, |
| "3.v_proj": 1638, |
| "3.o_proj": 1228, |
| "3.gate_proj": 2072, |
| "3.up_proj": 2072, |
| "3.down_proj": 1791, |
| "4.q_proj": 1638, |
| "4.k_proj": 1638, |
| "4.v_proj": 1638, |
| "4.o_proj": 1228, |
| "4.gate_proj": 2072, |
| "4.up_proj": 2072, |
| "4.down_proj": 1791, |
| "5.q_proj": 1638, |
| "5.k_proj": 1638, |
| "5.v_proj": 1638, |
| "5.o_proj": 1228, |
| "5.gate_proj": 2072, |
| "5.up_proj": 2072, |
| "5.down_proj": 1791, |
| "6.q_proj": 1638, |
| "6.k_proj": 1638, |
| "6.v_proj": 1638, |
| "6.o_proj": 1228, |
| "6.gate_proj": 2072, |
| "6.up_proj": 2072, |
| "6.down_proj": 1791, |
| "7.q_proj": 1638, |
| "7.k_proj": 1638, |
| "7.v_proj": 1638, |
| "7.o_proj": 1228, |
| "7.gate_proj": 2072, |
| "7.up_proj": 2072, |
| "7.down_proj": 1791, |
| "8.q_proj": 1638, |
| "8.k_proj": 1638, |
| "8.v_proj": 1638, |
| "8.o_proj": 1228, |
| "8.gate_proj": 2072, |
| "8.up_proj": 2072, |
| "8.down_proj": 1791, |
| "9.q_proj": 1638, |
| "9.k_proj": 1638, |
| "9.v_proj": 1638, |
| "9.o_proj": 1228, |
| "9.gate_proj": 2072, |
| "9.up_proj": 2072, |
| "9.down_proj": 1791, |
| "10.q_proj": 1638, |
| "10.k_proj": 1638, |
| "10.v_proj": 1638, |
| "10.o_proj": 1228, |
| "10.gate_proj": 2072, |
| "10.up_proj": 2072, |
| "10.down_proj": 1791, |
| "11.q_proj": 1638, |
| "11.k_proj": 1638, |
| "11.v_proj": 1638, |
| "11.o_proj": 1228, |
| "11.gate_proj": 2072, |
| "11.up_proj": 2072, |
| "11.down_proj": 1791, |
| "12.q_proj": 1638, |
| "12.k_proj": 1638, |
| "12.v_proj": 1638, |
| "12.o_proj": 1228, |
| "12.gate_proj": 2072, |
| "12.up_proj": 2072, |
| "12.down_proj": 1791, |
| "13.q_proj": 1638, |
| "13.k_proj": 1638, |
| "13.v_proj": 1638, |
| "13.o_proj": 1228, |
| "13.gate_proj": 2072, |
| "13.up_proj": 2072, |
| "13.down_proj": 1791, |
| "14.q_proj": 1638, |
| "14.k_proj": 1638, |
| "14.v_proj": 1638, |
| "14.o_proj": 1228, |
| "14.gate_proj": 2072, |
| "14.up_proj": 2072, |
| "14.down_proj": 1791, |
| "15.q_proj": 1638, |
| "15.k_proj": 1638, |
| "15.v_proj": 1638, |
| "15.o_proj": 1228, |
| "15.gate_proj": 2072, |
| "15.up_proj": 2072, |
| "15.down_proj": 1791, |
| "16.q_proj": 1638, |
| "16.k_proj": 1638, |
| "16.v_proj": 1638, |
| "16.o_proj": 1228, |
| "16.gate_proj": 2072, |
| "16.up_proj": 2072, |
| "16.down_proj": 1791, |
| "17.q_proj": 1638, |
| "17.k_proj": 1638, |
| "17.v_proj": 1638, |
| "17.o_proj": 1228, |
| "17.gate_proj": 2072, |
| "17.up_proj": 2072, |
| "17.down_proj": 1791, |
| "18.q_proj": 1638, |
| "18.k_proj": 1638, |
| "18.v_proj": 1638, |
| "18.o_proj": 1228, |
| "18.gate_proj": 2072, |
| "18.up_proj": 2072, |
| "18.down_proj": 1791, |
| "19.q_proj": 1638, |
| "19.k_proj": 1638, |
| "19.v_proj": 1638, |
| "19.o_proj": 1228, |
| "19.gate_proj": 2072, |
| "19.up_proj": 2072, |
| "19.down_proj": 1791, |
| "20.q_proj": 1638, |
| "20.k_proj": 1638, |
| "20.v_proj": 1638, |
| "20.o_proj": 1228, |
| "20.gate_proj": 2072, |
| "20.up_proj": 2072, |
| "20.down_proj": 1791, |
| "21.q_proj": 1638, |
| "21.k_proj": 1638, |
| "21.v_proj": 1638, |
| "21.o_proj": 1228, |
| "21.gate_proj": 2072, |
| "21.up_proj": 2072, |
| "21.down_proj": 1791, |
| "22.q_proj": 1638, |
| "22.k_proj": 1638, |
| "22.v_proj": 1638, |
| "22.o_proj": 1228, |
| "22.gate_proj": 2072, |
| "22.up_proj": 2072, |
| "22.down_proj": 1791, |
| "23.q_proj": 1638, |
| "23.k_proj": 1638, |
| "23.v_proj": 1638, |
| "23.o_proj": 1228, |
| "23.gate_proj": 2072, |
| "23.up_proj": 2072, |
| "23.down_proj": 1791, |
| "24.q_proj": 1638, |
| "24.k_proj": 1638, |
| "24.v_proj": 1638, |
| "24.o_proj": 1228, |
| "24.gate_proj": 2072, |
| "24.up_proj": 2072, |
| "24.down_proj": 1791, |
| "25.q_proj": 1638, |
| "25.k_proj": 1638, |
| "25.v_proj": 1638, |
| "25.o_proj": 1228, |
| "25.gate_proj": 2072, |
| "25.up_proj": 2072, |
| "25.down_proj": 1791, |
| "26.q_proj": 1638, |
| "26.k_proj": 1638, |
| "26.v_proj": 1638, |
| "26.o_proj": 1228, |
| "26.gate_proj": 2072, |
| "26.up_proj": 2072, |
| "26.down_proj": 1791, |
| "27.q_proj": 1638, |
| "27.k_proj": 1638, |
| "27.v_proj": 1638, |
| "27.o_proj": 1228, |
| "27.gate_proj": 2072, |
| "27.up_proj": 2072, |
| "27.down_proj": 1791, |
| "28.q_proj": 1638, |
| "28.k_proj": 1638, |
| "28.v_proj": 1638, |
| "28.o_proj": 1228, |
| "28.gate_proj": 2072, |
| "28.up_proj": 2072, |
| "28.down_proj": 1791, |
| "29.q_proj": 1638, |
| "29.k_proj": 1638, |
| "29.v_proj": 1638, |
| "29.o_proj": 1228, |
| "29.gate_proj": 2072, |
| "29.up_proj": 2072, |
| "29.down_proj": 1791, |
| "30.q_proj": 1638, |
| "30.k_proj": 1638, |
| "30.v_proj": 1638, |
| "30.o_proj": 1228, |
| "30.gate_proj": 2072, |
| "30.up_proj": 2072, |
| "30.down_proj": 1791, |
| "31.q_proj": 1638, |
| "31.k_proj": 1638, |
| "31.v_proj": 1638, |
| "31.o_proj": 1228, |
| "31.gate_proj": 2072, |
| "31.up_proj": 2072, |
| "31.down_proj": 1791 |
| }, |
| "stage": "basis_recovery_finetuned", |
| "recovery_finetune": { |
| "protocol": "basis_sharing_coefficient_lora", |
| "script": "gitclones/SVD-LLM/utils/LoRA.py", |
| "data_path": "alpaca-cleaned (/workspace/svd_safety/data/alpaca_cleaned.json)", |
| "epochs": 2, |
| "learning_rate": 0.0001, |
| "batch_size": 64, |
| "micro_batch_size": 4, |
| "cutoff_len": 256, |
| "val_set_size": 2000, |
| "max_train_samples": "all", |
| "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)", |
| "is_methods_own_stage2": false, |
| "note": "Basis Sharing publishes its own LoRA (lora.py: wikitext, batch 1, peft default q/v coefficient targets). This cell uses the project's SVD-LLM alpaca recipe so recovery data is constant across compressors, on all seven coefficient types and never on a basis.", |
| "lora_r": 8, |
| "lora_alpha": 16, |
| "lora_dropout": 0.05, |
| "lora_targets": [ |
| "down_proj", |
| "gate_proj", |
| "k_proj", |
| "o_proj", |
| "q_proj", |
| "up_proj", |
| "v_proj" |
| ], |
| "trained": "Coefficient modules only; shared and private bases frozen", |
| "merge": "C' = C + (alpha/r) B_lora A_lora, then W = C' @ basis", |
| "mean_rel_delta_coefficient": 0.02975503941498963 |
| }, |
| "param_fraction_is_upper_bound": false, |
| "rel_delta_coefficient": { |
| "0.down_proj": 0.01789172552525997, |
| "0.gate_proj": 0.07389885932207108, |
| "0.up_proj": 0.05307356268167496, |
| "0.k_proj": 0.04400179162621498, |
| "0.o_proj": 0.015720030292868614, |
| "0.q_proj": 0.042313627898693085, |
| "0.v_proj": 0.023663686588406563, |
| "1.down_proj": 0.009344649501144886, |
| "1.gate_proj": 0.05698956921696663, |
| "1.up_proj": 0.04757563769817352, |
| "1.k_proj": 0.02117219939827919, |
| "1.o_proj": 0.015290400013327599, |
| "1.q_proj": 0.02371150627732277, |
| "1.v_proj": 0.016866125166416168, |
| "2.down_proj": 0.017618203535676003, |
| "2.gate_proj": 0.05127978324890137, |
| "2.up_proj": 0.04971493035554886, |
| "2.k_proj": 0.023553118109703064, |
| "2.o_proj": 0.020216671749949455, |
| "2.q_proj": 0.022627107799053192, |
| "2.v_proj": 0.0288323312997818, |
| "3.down_proj": 0.017859240993857384, |
| "3.gate_proj": 0.0461166575551033, |
| "3.up_proj": 0.04656068980693817, |
| "3.k_proj": 0.02325223572552204, |
| "3.o_proj": 0.01974094845354557, |
| "3.q_proj": 0.022899625822901726, |
| "3.v_proj": 0.0293908528983593, |
| "4.down_proj": 0.015974679961800575, |
| "4.gate_proj": 0.04187997430562973, |
| "4.up_proj": 0.04083123430609703, |
| "4.k_proj": 0.024421246722340584, |
| "4.o_proj": 0.01839449815452099, |
| "4.q_proj": 0.02253827266395092, |
| "4.v_proj": 0.026954350993037224, |
| "5.down_proj": 0.016185220330953598, |
| "5.gate_proj": 0.0418110266327858, |
| "5.up_proj": 0.040592070668935776, |
| "5.k_proj": 0.022455476224422455, |
| "5.o_proj": 0.017546743154525757, |
| "5.q_proj": 0.020589888095855713, |
| "5.v_proj": 0.026903508231043816, |
| "6.down_proj": 0.01673140563070774, |
| "6.gate_proj": 0.039437901228666306, |
| "6.up_proj": 0.037406571209430695, |
| "6.k_proj": 0.022581404075026512, |
| "6.o_proj": 0.017793400213122368, |
| "6.q_proj": 0.01947803795337677, |
| "6.v_proj": 0.025546329095959663, |
| "7.down_proj": 0.01703973300755024, |
| "7.gate_proj": 0.03845927119255066, |
| "7.up_proj": 0.042372316122055054, |
| "7.k_proj": 0.01995324343442917, |
| "7.o_proj": 0.01723671145737171, |
| "7.q_proj": 0.018467623740434647, |
| "7.v_proj": 0.025217900052666664, |
| "8.down_proj": 0.016596227884292603, |
| "8.gate_proj": 0.04214202240109444, |
| "8.up_proj": 0.03909769654273987, |
| "8.k_proj": 0.020271550863981247, |
| "8.o_proj": 0.016867313534021378, |
| "8.q_proj": 0.019599435850977898, |
| "8.v_proj": 0.02347864769399166, |
| "9.down_proj": 0.016085432842373848, |
| "9.gate_proj": 0.03925571218132973, |
| "9.up_proj": 0.03977390378713608, |
| "9.k_proj": 0.020905086770653725, |
| "9.o_proj": 0.017597880214452744, |
| "9.q_proj": 0.019629141315817833, |
| "9.v_proj": 0.021614503115415573, |
| "10.down_proj": 0.016450710594654083, |
| "10.gate_proj": 0.0410277396440506, |
| "10.up_proj": 0.036953117698431015, |
| "10.k_proj": 0.021330196410417557, |
| "10.o_proj": 0.018903516232967377, |
| "10.q_proj": 0.01969883218407631, |
| "10.v_proj": 0.02067733369767666, |
| "11.down_proj": 0.01707199402153492, |
| "11.gate_proj": 0.038356050848960876, |
| "11.up_proj": 0.041197046637535095, |
| "11.k_proj": 0.023145737126469612, |
| "11.o_proj": 0.016748346388339996, |
| "11.q_proj": 0.021463047713041306, |
| "11.v_proj": 0.019644424319267273, |
| "12.down_proj": 0.0163368321955204, |
| "12.gate_proj": 0.03845346346497536, |
| "12.up_proj": 0.039496049284935, |
| "12.k_proj": 0.022679366171360016, |
| "12.o_proj": 0.017901107668876648, |
| "12.q_proj": 0.021295785903930664, |
| "12.v_proj": 0.020925084128975868, |
| "13.down_proj": 0.016835162416100502, |
| "13.gate_proj": 0.04033529385924339, |
| "13.up_proj": 0.04001957178115845, |
| "13.k_proj": 0.02491459809243679, |
| "13.o_proj": 0.018841315060853958, |
| "13.q_proj": 0.022686036303639412, |
| "13.v_proj": 0.022567175328731537, |
| "14.down_proj": 0.01779063418507576, |
| "14.gate_proj": 0.04157009720802307, |
| "14.up_proj": 0.041127171367406845, |
| "14.k_proj": 0.025756051763892174, |
| "14.o_proj": 0.01769130863249302, |
| "14.q_proj": 0.024447226896882057, |
| "14.v_proj": 0.02386847883462906, |
| "15.down_proj": 0.018620144575834274, |
| "15.gate_proj": 0.04322356358170509, |
| "15.up_proj": 0.04224470630288124, |
| "15.k_proj": 0.025942062959074974, |
| "15.o_proj": 0.018257489427924156, |
| "15.q_proj": 0.023428166285157204, |
| "15.v_proj": 0.022524960339069366, |
| "16.down_proj": 0.018772562965750694, |
| "16.gate_proj": 0.044220734387636185, |
| "16.up_proj": 0.04224252700805664, |
| "16.k_proj": 0.029059557244181633, |
| "16.o_proj": 0.019190525636076927, |
| "16.q_proj": 0.025546090677380562, |
| "16.v_proj": 0.027710048481822014, |
| "17.down_proj": 0.019686385989189148, |
| "17.gate_proj": 0.04135333001613617, |
| "17.up_proj": 0.04486946761608124, |
| "17.k_proj": 0.02745143696665764, |
| "17.o_proj": 0.020055420696735382, |
| "17.q_proj": 0.025585098192095757, |
| "17.v_proj": 0.02658572606742382, |
| "18.down_proj": 0.020166408270597458, |
| "18.gate_proj": 0.041446503251791, |
| "18.up_proj": 0.04579155147075653, |
| "18.k_proj": 0.02984543703496456, |
| "18.o_proj": 0.019872967153787613, |
| "18.q_proj": 0.02715788222849369, |
| "18.v_proj": 0.029887434095144272, |
| "19.down_proj": 0.020344026386737823, |
| "19.gate_proj": 0.04185543581843376, |
| "19.up_proj": 0.047275569289922714, |
| "19.k_proj": 0.02909199893474579, |
| "19.o_proj": 0.022002629935741425, |
| "19.q_proj": 0.02898808941245079, |
| "19.v_proj": 0.028401562944054604, |
| "20.down_proj": 0.020384257659316063, |
| "20.gate_proj": 0.04890038073062897, |
| "20.up_proj": 0.04913109540939331, |
| "20.k_proj": 0.030232349410653114, |
| "20.o_proj": 0.022863484919071198, |
| "20.q_proj": 0.029142558574676514, |
| "20.v_proj": 0.030943676829338074, |
| "21.down_proj": 0.020678570494055748, |
| "21.gate_proj": 0.04660072550177574, |
| "21.up_proj": 0.04657881334424019, |
| "21.k_proj": 0.02797800861299038, |
| "21.o_proj": 0.024201812222599983, |
| "21.q_proj": 0.028927212581038475, |
| "21.v_proj": 0.032502662390470505, |
| "22.down_proj": 0.02047550119459629, |
| "22.gate_proj": 0.048371393233537674, |
| "22.up_proj": 0.053299009799957275, |
| "22.k_proj": 0.02902030199766159, |
| "22.o_proj": 0.021341780200600624, |
| "22.q_proj": 0.028871284797787666, |
| "22.v_proj": 0.034008994698524475, |
| "23.down_proj": 0.02101362682878971, |
| "23.gate_proj": 0.04909517243504524, |
| "23.up_proj": 0.05019266530871391, |
| "23.k_proj": 0.02889767661690712, |
| "23.o_proj": 0.02051742933690548, |
| "23.q_proj": 0.02883676066994667, |
| "23.v_proj": 0.03221878036856651, |
| "24.down_proj": 0.020756598562002182, |
| "24.gate_proj": 0.049081768840551376, |
| "24.up_proj": 0.05525779724121094, |
| "24.k_proj": 0.028636103495955467, |
| "24.o_proj": 0.02002854458987713, |
| "24.q_proj": 0.028798028826713562, |
| "24.v_proj": 0.03184721618890762, |
| "25.down_proj": 0.019914736971259117, |
| "25.gate_proj": 0.05659620091319084, |
| "25.up_proj": 0.05105341598391533, |
| "25.k_proj": 0.03020135685801506, |
| "25.o_proj": 0.02426576428115368, |
| "25.q_proj": 0.028738798573613167, |
| "25.v_proj": 0.030420735478401184, |
| "26.down_proj": 0.020024362951517105, |
| "26.gate_proj": 0.049845531582832336, |
| "26.up_proj": 0.047202229499816895, |
| "26.k_proj": 0.03258863091468811, |
| "26.o_proj": 0.01930871419608593, |
| "26.q_proj": 0.028618991374969482, |
| "26.v_proj": 0.030448047444224358, |
| "27.down_proj": 0.01905849389731884, |
| "27.gate_proj": 0.04903184995055199, |
| "27.up_proj": 0.0499153696000576, |
| "27.k_proj": 0.027926409617066383, |
| "27.o_proj": 0.02700573392212391, |
| "27.q_proj": 0.02987619861960411, |
| "27.v_proj": 0.0307757705450058, |
| "28.down_proj": 0.02064599096775055, |
| "28.gate_proj": 0.05176892504096031, |
| "28.up_proj": 0.04868489131331444, |
| "28.k_proj": 0.030534468591213226, |
| "28.o_proj": 0.01911034807562828, |
| "28.q_proj": 0.03064342774450779, |
| "28.v_proj": 0.027126865461468697, |
| "29.down_proj": 0.01944028027355671, |
| "29.gate_proj": 0.05063900351524353, |
| "29.up_proj": 0.04600493237376213, |
| "29.k_proj": 0.034188102930784225, |
| "29.o_proj": 0.01887245662510395, |
| "29.q_proj": 0.02965513803064823, |
| "29.v_proj": 0.03125184401869774, |
| "30.down_proj": 0.014209754765033722, |
| "30.gate_proj": 0.04668048024177551, |
| "30.up_proj": 0.043189410120248795, |
| "30.k_proj": 0.031168438494205475, |
| "30.o_proj": 0.018736323341727257, |
| "30.q_proj": 0.03064728155732155, |
| "30.v_proj": 0.022416142746806145, |
| "31.down_proj": 0.01456467341631651, |
| "31.gate_proj": 0.048269350081682205, |
| "31.up_proj": 0.04480603337287903, |
| "31.k_proj": 0.03716890141367912, |
| "31.o_proj": 0.018774691969156265, |
| "31.q_proj": 0.034700050950050354, |
| "31.v_proj": 0.02298746630549431 |
| } |
| } |
