| { |
| "source": "/workspace/svd_safety/assets/models/Llama-2-7b-chat-hf", |
| "upstream": "TUDa-HWAI/Basis_Sharing", |
| "step": "basis_sharing", |
| "ratio_removed": 0.5, |
| "group_size": 2, |
| "nsamples": 256, |
| "seqlen": 2048, |
| "seed": 42, |
| "share_part": [ |
| "v", |
| "k", |
| "q", |
| "up", |
| "gate" |
| ], |
| "private_part": [ |
| "down", |
| "o" |
| ], |
| "factored": "/workspace/svd_safety/results/basis_sharing/cells/l2_basis_remove50/truncated/factored.pt", |
| "param_fraction": 0.49982796308290156, |
| "ranks": { |
| "0.q_proj": 1365, |
| "0.k_proj": 1365, |
| "0.v_proj": 1365, |
| "0.o_proj": 1024, |
| "0.gate_proj": 1726, |
| "0.up_proj": 1726, |
| "0.down_proj": 1492, |
| "1.q_proj": 1365, |
| "1.k_proj": 1365, |
| "1.v_proj": 1365, |
| "1.o_proj": 1024, |
| "1.gate_proj": 1726, |
| "1.up_proj": 1726, |
| "1.down_proj": 1492, |
| "2.q_proj": 1365, |
| "2.k_proj": 1365, |
| "2.v_proj": 1365, |
| "2.o_proj": 1024, |
| "2.gate_proj": 1726, |
| "2.up_proj": 1726, |
| "2.down_proj": 1492, |
| "3.q_proj": 1365, |
| "3.k_proj": 1365, |
| "3.v_proj": 1365, |
| "3.o_proj": 1024, |
| "3.gate_proj": 1726, |
| "3.up_proj": 1726, |
| "3.down_proj": 1492, |
| "4.q_proj": 1365, |
| "4.k_proj": 1365, |
| "4.v_proj": 1365, |
| "4.o_proj": 1024, |
| "4.gate_proj": 1726, |
| "4.up_proj": 1726, |
| "4.down_proj": 1492, |
| "5.q_proj": 1365, |
| "5.k_proj": 1365, |
| "5.v_proj": 1365, |
| "5.o_proj": 1024, |
| "5.gate_proj": 1726, |
| "5.up_proj": 1726, |
| "5.down_proj": 1492, |
| "6.q_proj": 1365, |
| "6.k_proj": 1365, |
| "6.v_proj": 1365, |
| "6.o_proj": 1024, |
| "6.gate_proj": 1726, |
| "6.up_proj": 1726, |
| "6.down_proj": 1492, |
| "7.q_proj": 1365, |
| "7.k_proj": 1365, |
| "7.v_proj": 1365, |
| "7.o_proj": 1024, |
| "7.gate_proj": 1726, |
| "7.up_proj": 1726, |
| "7.down_proj": 1492, |
| "8.q_proj": 1365, |
| "8.k_proj": 1365, |
| "8.v_proj": 1365, |
| "8.o_proj": 1024, |
| "8.gate_proj": 1726, |
| "8.up_proj": 1726, |
| "8.down_proj": 1492, |
| "9.q_proj": 1365, |
| "9.k_proj": 1365, |
| "9.v_proj": 1365, |
| "9.o_proj": 1024, |
| "9.gate_proj": 1726, |
| "9.up_proj": 1726, |
| "9.down_proj": 1492, |
| "10.q_proj": 1365, |
| "10.k_proj": 1365, |
| "10.v_proj": 1365, |
| "10.o_proj": 1024, |
| "10.gate_proj": 1726, |
| "10.up_proj": 1726, |
| "10.down_proj": 1492, |
| "11.q_proj": 1365, |
| "11.k_proj": 1365, |
| "11.v_proj": 1365, |
| "11.o_proj": 1024, |
| "11.gate_proj": 1726, |
| "11.up_proj": 1726, |
| "11.down_proj": 1492, |
| "12.q_proj": 1365, |
| "12.k_proj": 1365, |
| "12.v_proj": 1365, |
| "12.o_proj": 1024, |
| "12.gate_proj": 1726, |
| "12.up_proj": 1726, |
| "12.down_proj": 1492, |
| "13.q_proj": 1365, |
| "13.k_proj": 1365, |
| "13.v_proj": 1365, |
| "13.o_proj": 1024, |
| "13.gate_proj": 1726, |
| "13.up_proj": 1726, |
| "13.down_proj": 1492, |
| "14.q_proj": 1365, |
| "14.k_proj": 1365, |
| "14.v_proj": 1365, |
| "14.o_proj": 1024, |
| "14.gate_proj": 1726, |
| "14.up_proj": 1726, |
| "14.down_proj": 1492, |
| "15.q_proj": 1365, |
| "15.k_proj": 1365, |
| "15.v_proj": 1365, |
| "15.o_proj": 1024, |
| "15.gate_proj": 1726, |
| "15.up_proj": 1726, |
| "15.down_proj": 1492, |
| "16.q_proj": 1365, |
| "16.k_proj": 1365, |
| "16.v_proj": 1365, |
| "16.o_proj": 1024, |
| "16.gate_proj": 1726, |
| "16.up_proj": 1726, |
| "16.down_proj": 1492, |
| "17.q_proj": 1365, |
| "17.k_proj": 1365, |
| "17.v_proj": 1365, |
| "17.o_proj": 1024, |
| "17.gate_proj": 1726, |
| "17.up_proj": 1726, |
| "17.down_proj": 1492, |
| "18.q_proj": 1365, |
| "18.k_proj": 1365, |
| "18.v_proj": 1365, |
| "18.o_proj": 1024, |
| "18.gate_proj": 1726, |
| "18.up_proj": 1726, |
| "18.down_proj": 1492, |
| "19.q_proj": 1365, |
| "19.k_proj": 1365, |
| "19.v_proj": 1365, |
| "19.o_proj": 1024, |
| "19.gate_proj": 1726, |
| "19.up_proj": 1726, |
| "19.down_proj": 1492, |
| "20.q_proj": 1365, |
| "20.k_proj": 1365, |
| "20.v_proj": 1365, |
| "20.o_proj": 1024, |
| "20.gate_proj": 1726, |
| "20.up_proj": 1726, |
| "20.down_proj": 1492, |
| "21.q_proj": 1365, |
| "21.k_proj": 1365, |
| "21.v_proj": 1365, |
| "21.o_proj": 1024, |
| "21.gate_proj": 1726, |
| "21.up_proj": 1726, |
| "21.down_proj": 1492, |
| "22.q_proj": 1365, |
| "22.k_proj": 1365, |
| "22.v_proj": 1365, |
| "22.o_proj": 1024, |
| "22.gate_proj": 1726, |
| "22.up_proj": 1726, |
| "22.down_proj": 1492, |
| "23.q_proj": 1365, |
| "23.k_proj": 1365, |
| "23.v_proj": 1365, |
| "23.o_proj": 1024, |
| "23.gate_proj": 1726, |
| "23.up_proj": 1726, |
| "23.down_proj": 1492, |
| "24.q_proj": 1365, |
| "24.k_proj": 1365, |
| "24.v_proj": 1365, |
| "24.o_proj": 1024, |
| "24.gate_proj": 1726, |
| "24.up_proj": 1726, |
| "24.down_proj": 1492, |
| "25.q_proj": 1365, |
| "25.k_proj": 1365, |
| "25.v_proj": 1365, |
| "25.o_proj": 1024, |
| "25.gate_proj": 1726, |
| "25.up_proj": 1726, |
| "25.down_proj": 1492, |
| "26.q_proj": 1365, |
| "26.k_proj": 1365, |
| "26.v_proj": 1365, |
| "26.o_proj": 1024, |
| "26.gate_proj": 1726, |
| "26.up_proj": 1726, |
| "26.down_proj": 1492, |
| "27.q_proj": 1365, |
| "27.k_proj": 1365, |
| "27.v_proj": 1365, |
| "27.o_proj": 1024, |
| "27.gate_proj": 1726, |
| "27.up_proj": 1726, |
| "27.down_proj": 1492, |
| "28.q_proj": 1365, |
| "28.k_proj": 1365, |
| "28.v_proj": 1365, |
| "28.o_proj": 1024, |
| "28.gate_proj": 1726, |
| "28.up_proj": 1726, |
| "28.down_proj": 1492, |
| "29.q_proj": 1365, |
| "29.k_proj": 1365, |
| "29.v_proj": 1365, |
| "29.o_proj": 1024, |
| "29.gate_proj": 1726, |
| "29.up_proj": 1726, |
| "29.down_proj": 1492, |
| "30.q_proj": 1365, |
| "30.k_proj": 1365, |
| "30.v_proj": 1365, |
| "30.o_proj": 1024, |
| "30.gate_proj": 1726, |
| "30.up_proj": 1726, |
| "30.down_proj": 1492, |
| "31.q_proj": 1365, |
| "31.k_proj": 1365, |
| "31.v_proj": 1365, |
| "31.o_proj": 1024, |
| "31.gate_proj": 1726, |
| "31.up_proj": 1726, |
| "31.down_proj": 1492 |
| }, |
| "stage": "basis_recovery_finetuned", |
| "recovery_finetune": { |
| "protocol": "basis_sharing_coefficient_lora", |
| "script": "gitclones/SVD-LLM/utils/LoRA.py", |
| "data_path": "alpaca-cleaned (/workspace/svd_safety/data/alpaca_cleaned.json)", |
| "epochs": 2, |
| "learning_rate": 0.0001, |
| "batch_size": 64, |
| "micro_batch_size": 4, |
| "cutoff_len": 256, |
| "val_set_size": 2000, |
| "max_train_samples": "all", |
| "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)", |
| "is_methods_own_stage2": false, |
| "note": "Basis Sharing publishes its own LoRA (lora.py: wikitext, batch 1, peft default q/v coefficient targets). This cell uses the project's SVD-LLM alpaca recipe so recovery data is constant across compressors, on all seven coefficient types and never on a basis.", |
| "lora_r": 8, |
| "lora_alpha": 16, |
| "lora_dropout": 0.05, |
| "lora_targets": [ |
| "down_proj", |
| "gate_proj", |
| "k_proj", |
| "o_proj", |
| "q_proj", |
| "up_proj", |
| "v_proj" |
| ], |
| "trained": "Coefficient modules only; shared and private bases frozen", |
| "merge": "C' = C + (alpha/r) B_lora A_lora, then W = C' @ basis", |
| "mean_rel_delta_coefficient": 0.03477951146695497 |
| }, |
| "param_fraction_is_upper_bound": false, |
| "rel_delta_coefficient": { |
| "0.down_proj": 0.023729722946882248, |
| "0.gate_proj": 0.08452913910150528, |
| "0.up_proj": 0.059868842363357544, |
| "0.k_proj": 0.05120539665222168, |
| "0.o_proj": 0.017369695007801056, |
| "0.q_proj": 0.056597448885440826, |
| "0.v_proj": 0.026993343606591225, |
| "1.down_proj": 0.01011702511459589, |
| "1.gate_proj": 0.06331334263086319, |
| "1.up_proj": 0.052683912217617035, |
| "1.k_proj": 0.02192487008869648, |
| "1.o_proj": 0.01861078478395939, |
| "1.q_proj": 0.027439886704087257, |
| "1.v_proj": 0.019045013934373856, |
| "2.down_proj": 0.0223842766135931, |
| "2.gate_proj": 0.060128744691610336, |
| "2.up_proj": 0.0586223378777504, |
| "2.k_proj": 0.026699945330619812, |
| "2.o_proj": 0.021796977147459984, |
| "2.q_proj": 0.028158212080597878, |
| "2.v_proj": 0.03134380280971527, |
| "3.down_proj": 0.022037558257579803, |
| "3.gate_proj": 0.053900573402643204, |
| "3.up_proj": 0.05270858481526375, |
| "3.k_proj": 0.02719547599554062, |
| "3.o_proj": 0.021815096959471703, |
| "3.q_proj": 0.026857580989599228, |
| "3.v_proj": 0.03770525008440018, |
| "4.down_proj": 0.02017192542552948, |
| "4.gate_proj": 0.04923078790307045, |
| "4.up_proj": 0.04876559600234032, |
| "4.k_proj": 0.03054051846265793, |
| "4.o_proj": 0.024025147780776024, |
| "4.q_proj": 0.026279738172888756, |
| "4.v_proj": 0.02896757237613201, |
| "5.down_proj": 0.019595874473452568, |
| "5.gate_proj": 0.04980846494436264, |
| "5.up_proj": 0.04777255654335022, |
| "5.k_proj": 0.02639787830412388, |
| "5.o_proj": 0.01787957362830639, |
| "5.q_proj": 0.021695686504244804, |
| "5.v_proj": 0.030705556273460388, |
| "6.down_proj": 0.01879432424902916, |
| "6.gate_proj": 0.045955441892147064, |
| "6.up_proj": 0.046904079616069794, |
| "6.k_proj": 0.025844892486929893, |
| "6.o_proj": 0.020820025354623795, |
| "6.q_proj": 0.0238942988216877, |
| "6.v_proj": 0.029350096359848976, |
| "7.down_proj": 0.020194197073578835, |
| "7.gate_proj": 0.04724954441189766, |
| "7.up_proj": 0.04511765390634537, |
| "7.k_proj": 0.025140712037682533, |
| "7.o_proj": 0.019103750586509705, |
| "7.q_proj": 0.0233695600181818, |
| "7.v_proj": 0.027190091088414192, |
| "8.down_proj": 0.01932426728308201, |
| "8.gate_proj": 0.04175572097301483, |
| "8.up_proj": 0.044960521161556244, |
| "8.k_proj": 0.02549305558204651, |
| "8.o_proj": 0.020664075389504433, |
| "8.q_proj": 0.022841913625597954, |
| "8.v_proj": 0.025211365893483162, |
| "9.down_proj": 0.017424633726477623, |
| "9.gate_proj": 0.047231525182724, |
| "9.up_proj": 0.04754003509879112, |
| "9.k_proj": 0.02622208558022976, |
| "9.o_proj": 0.018889963626861572, |
| "9.q_proj": 0.024236785247921944, |
| "9.v_proj": 0.02502570115029812, |
| "10.down_proj": 0.018354928120970726, |
| "10.gate_proj": 0.0459887869656086, |
| "10.up_proj": 0.04865645617246628, |
| "10.k_proj": 0.02619076706469059, |
| "10.o_proj": 0.021123865619301796, |
| "10.q_proj": 0.023364614695310593, |
| "10.v_proj": 0.024980315938591957, |
| "11.down_proj": 0.01799744740128517, |
| "11.gate_proj": 0.04674726724624634, |
| "11.up_proj": 0.04505652189254761, |
| "11.k_proj": 0.026833688840270042, |
| "11.o_proj": 0.01736118644475937, |
| "11.q_proj": 0.023478467017412186, |
| "11.v_proj": 0.019759302958846092, |
| "12.down_proj": 0.019656836986541748, |
| "12.gate_proj": 0.04380928352475166, |
| "12.up_proj": 0.046775877475738525, |
| "12.k_proj": 0.027490461245179176, |
| "12.o_proj": 0.019591068848967552, |
| "12.q_proj": 0.026375921443104744, |
| "12.v_proj": 0.02554561384022236, |
| "13.down_proj": 0.02064443565905094, |
| "13.gate_proj": 0.04611923545598984, |
| "13.up_proj": 0.049012213945388794, |
| "13.k_proj": 0.029619509354233742, |
| "13.o_proj": 0.0220456775277853, |
| "13.q_proj": 0.028390999883413315, |
| "13.v_proj": 0.027894970029592514, |
| "14.down_proj": 0.02147248387336731, |
| "14.gate_proj": 0.046727463603019714, |
| "14.up_proj": 0.04966231808066368, |
| "14.k_proj": 0.029388461261987686, |
| "14.o_proj": 0.02229248732328415, |
| "14.q_proj": 0.026943136006593704, |
| "14.v_proj": 0.027938341721892357, |
| "15.down_proj": 0.02149982564151287, |
| "15.gate_proj": 0.050257809460163116, |
| "15.up_proj": 0.04973958060145378, |
| "15.k_proj": 0.03036561980843544, |
| "15.o_proj": 0.022304357960820198, |
| "15.q_proj": 0.02832680568099022, |
| "15.v_proj": 0.026476964354515076, |
| "16.down_proj": 0.02218230813741684, |
| "16.gate_proj": 0.050966568291187286, |
| "16.up_proj": 0.05037204548716545, |
| "16.k_proj": 0.032374244183301926, |
| "16.o_proj": 0.022258751094341278, |
| "16.q_proj": 0.030399447306990623, |
| "16.v_proj": 0.029645616188645363, |
| "17.down_proj": 0.02408037707209587, |
| "17.gate_proj": 0.05045311152935028, |
| "17.up_proj": 0.052991095930337906, |
| "17.k_proj": 0.03171582520008087, |
| "17.o_proj": 0.024419406428933144, |
| "17.q_proj": 0.029634423553943634, |
| "17.v_proj": 0.029784325510263443, |
| "18.down_proj": 0.024372927844524384, |
| "18.gate_proj": 0.048495788127183914, |
| "18.up_proj": 0.05652078613638878, |
| "18.k_proj": 0.03411583974957466, |
| "18.o_proj": 0.025013010948896408, |
| "18.q_proj": 0.033185336738824844, |
| "18.v_proj": 0.033583350479602814, |
| "19.down_proj": 0.023557795211672783, |
| "19.gate_proj": 0.04933939501643181, |
| "19.up_proj": 0.05396072566509247, |
| "19.k_proj": 0.03440993279218674, |
| "19.o_proj": 0.026062030345201492, |
| "19.q_proj": 0.032321497797966, |
| "19.v_proj": 0.033995699137449265, |
| "20.down_proj": 0.024951640516519547, |
| "20.gate_proj": 0.04994823411107063, |
| "20.up_proj": 0.05292683094739914, |
| "20.k_proj": 0.03518199548125267, |
| "20.o_proj": 0.026832351461052895, |
| "20.q_proj": 0.03478262200951576, |
| "20.v_proj": 0.034277647733688354, |
| "21.down_proj": 0.024847429245710373, |
| "21.gate_proj": 0.05355115979909897, |
| "21.up_proj": 0.0550607293844223, |
| "21.k_proj": 0.03435465320944786, |
| "21.o_proj": 0.02734820917248726, |
| "21.q_proj": 0.03575967997312546, |
| "21.v_proj": 0.039932653307914734, |
| "22.down_proj": 0.025028206408023834, |
| "22.gate_proj": 0.05792507156729698, |
| "22.up_proj": 0.056496161967515945, |
| "22.k_proj": 0.035662177950143814, |
| "22.o_proj": 0.02528970316052437, |
| "22.q_proj": 0.03271043300628662, |
| "22.v_proj": 0.03736736625432968, |
| "23.down_proj": 0.024994004517793655, |
| "23.gate_proj": 0.05386468768119812, |
| "23.up_proj": 0.05626451596617699, |
| "23.k_proj": 0.03418954834342003, |
| "23.o_proj": 0.023994209244847298, |
| "23.q_proj": 0.03303827345371246, |
| "23.v_proj": 0.03754240646958351, |
| "24.down_proj": 0.025210509076714516, |
| "24.gate_proj": 0.05693400278687477, |
| "24.up_proj": 0.06670183688402176, |
| "24.k_proj": 0.03441649302840233, |
| "24.o_proj": 0.021066762506961823, |
| "24.q_proj": 0.0341021791100502, |
| "24.v_proj": 0.03781954199075699, |
| "25.down_proj": 0.02424655854701996, |
| "25.gate_proj": 0.06084345653653145, |
| "25.up_proj": 0.06915346533060074, |
| "25.k_proj": 0.035405270755290985, |
| "25.o_proj": 0.02484528347849846, |
| "25.q_proj": 0.033545415848493576, |
| "25.v_proj": 0.0366339273750782, |
| "26.down_proj": 0.0257209874689579, |
| "26.gate_proj": 0.05857868492603302, |
| "26.up_proj": 0.05918649584054947, |
| "26.k_proj": 0.04149233549833298, |
| "26.o_proj": 0.02095915749669075, |
| "26.q_proj": 0.036208391189575195, |
| "26.v_proj": 0.039256542921066284, |
| "27.down_proj": 0.02435978874564171, |
| "27.gate_proj": 0.057182516902685165, |
| "27.up_proj": 0.05642085522413254, |
| "27.k_proj": 0.03608841821551323, |
| "27.o_proj": 0.031146977096796036, |
| "27.q_proj": 0.035692207515239716, |
| "27.v_proj": 0.036948930472135544, |
| "28.down_proj": 0.0241420716047287, |
| "28.gate_proj": 0.058406781405210495, |
| "28.up_proj": 0.05536749213933945, |
| "28.k_proj": 0.037475209683179855, |
| "28.o_proj": 0.025948751717805862, |
| "28.q_proj": 0.03311076760292053, |
| "28.v_proj": 0.036046117544174194, |
| "29.down_proj": 0.023028584197163582, |
| "29.gate_proj": 0.05505272001028061, |
| "29.up_proj": 0.058274224400520325, |
| "29.k_proj": 0.04002455994486809, |
| "29.o_proj": 0.021812763065099716, |
| "29.q_proj": 0.03628501296043396, |
| "29.v_proj": 0.0329945832490921, |
| "30.down_proj": 0.014910749159753323, |
| "30.gate_proj": 0.0535011924803257, |
| "30.up_proj": 0.05174972116947174, |
| "30.k_proj": 0.038816239684820175, |
| "30.o_proj": 0.023247702047228813, |
| "30.q_proj": 0.034509968012571335, |
| "30.v_proj": 0.023940032348036766, |
| "31.down_proj": 0.01771707274019718, |
| "31.gate_proj": 0.0577886700630188, |
| "31.up_proj": 0.04945879802107811, |
| "31.k_proj": 0.04337715357542038, |
| "31.o_proj": 0.019638074561953545, |
| "31.q_proj": 0.039459045976400375, |
| "31.v_proj": 0.025393839925527573 |
| } |
| } |
