| { |
| "source": "/workspace/svd_safety/assets/models/Llama-2-7b-chat-hf", |
| "upstream": "TUDa-HWAI/Basis_Sharing", |
| "step": "basis_sharing", |
| "ratio_removed": 0.2, |
| "group_size": 2, |
| "nsamples": 256, |
| "seqlen": 2048, |
| "seed": 42, |
| "share_part": [ |
| "v", |
| "k", |
| "q", |
| "up", |
| "gate" |
| ], |
| "private_part": [ |
| "down", |
| "o" |
| ], |
| "qkv_bias_restored": 0, |
| "factored": "/workspace/svd_safety/results/basis_sharing/cells/l2_basis_remove20/truncated/factored.pt", |
| "param_fraction": 0.7998198672279793, |
| "ranks": { |
| "0.q_proj": 2184, |
| "0.k_proj": 2184, |
| "0.v_proj": 2184, |
| "0.o_proj": 1638, |
| "0.gate_proj": 2762, |
| "0.up_proj": 2762, |
| "0.down_proj": 2388, |
| "1.q_proj": 2184, |
| "1.k_proj": 2184, |
| "1.v_proj": 2184, |
| "1.o_proj": 1638, |
| "1.gate_proj": 2762, |
| "1.up_proj": 2762, |
| "1.down_proj": 2388, |
| "2.q_proj": 2184, |
| "2.k_proj": 2184, |
| "2.v_proj": 2184, |
| "2.o_proj": 1638, |
| "2.gate_proj": 2762, |
| "2.up_proj": 2762, |
| "2.down_proj": 2388, |
| "3.q_proj": 2184, |
| "3.k_proj": 2184, |
| "3.v_proj": 2184, |
| "3.o_proj": 1638, |
| "3.gate_proj": 2762, |
| "3.up_proj": 2762, |
| "3.down_proj": 2388, |
| "4.q_proj": 2184, |
| "4.k_proj": 2184, |
| "4.v_proj": 2184, |
| "4.o_proj": 1638, |
| "4.gate_proj": 2762, |
| "4.up_proj": 2762, |
| "4.down_proj": 2388, |
| "5.q_proj": 2184, |
| "5.k_proj": 2184, |
| "5.v_proj": 2184, |
| "5.o_proj": 1638, |
| "5.gate_proj": 2762, |
| "5.up_proj": 2762, |
| "5.down_proj": 2388, |
| "6.q_proj": 2184, |
| "6.k_proj": 2184, |
| "6.v_proj": 2184, |
| "6.o_proj": 1638, |
| "6.gate_proj": 2762, |
| "6.up_proj": 2762, |
| "6.down_proj": 2388, |
| "7.q_proj": 2184, |
| "7.k_proj": 2184, |
| "7.v_proj": 2184, |
| "7.o_proj": 1638, |
| "7.gate_proj": 2762, |
| "7.up_proj": 2762, |
| "7.down_proj": 2388, |
| "8.q_proj": 2184, |
| "8.k_proj": 2184, |
| "8.v_proj": 2184, |
| "8.o_proj": 1638, |
| "8.gate_proj": 2762, |
| "8.up_proj": 2762, |
| "8.down_proj": 2388, |
| "9.q_proj": 2184, |
| "9.k_proj": 2184, |
| "9.v_proj": 2184, |
| "9.o_proj": 1638, |
| "9.gate_proj": 2762, |
| "9.up_proj": 2762, |
| "9.down_proj": 2388, |
| "10.q_proj": 2184, |
| "10.k_proj": 2184, |
| "10.v_proj": 2184, |
| "10.o_proj": 1638, |
| "10.gate_proj": 2762, |
| "10.up_proj": 2762, |
| "10.down_proj": 2388, |
| "11.q_proj": 2184, |
| "11.k_proj": 2184, |
| "11.v_proj": 2184, |
| "11.o_proj": 1638, |
| "11.gate_proj": 2762, |
| "11.up_proj": 2762, |
| "11.down_proj": 2388, |
| "12.q_proj": 2184, |
| "12.k_proj": 2184, |
| "12.v_proj": 2184, |
| "12.o_proj": 1638, |
| "12.gate_proj": 2762, |
| "12.up_proj": 2762, |
| "12.down_proj": 2388, |
| "13.q_proj": 2184, |
| "13.k_proj": 2184, |
| "13.v_proj": 2184, |
| "13.o_proj": 1638, |
| "13.gate_proj": 2762, |
| "13.up_proj": 2762, |
| "13.down_proj": 2388, |
| "14.q_proj": 2184, |
| "14.k_proj": 2184, |
| "14.v_proj": 2184, |
| "14.o_proj": 1638, |
| "14.gate_proj": 2762, |
| "14.up_proj": 2762, |
| "14.down_proj": 2388, |
| "15.q_proj": 2184, |
| "15.k_proj": 2184, |
| "15.v_proj": 2184, |
| "15.o_proj": 1638, |
| "15.gate_proj": 2762, |
| "15.up_proj": 2762, |
| "15.down_proj": 2388, |
| "16.q_proj": 2184, |
| "16.k_proj": 2184, |
| "16.v_proj": 2184, |
| "16.o_proj": 1638, |
| "16.gate_proj": 2762, |
| "16.up_proj": 2762, |
| "16.down_proj": 2388, |
| "17.q_proj": 2184, |
| "17.k_proj": 2184, |
| "17.v_proj": 2184, |
| "17.o_proj": 1638, |
| "17.gate_proj": 2762, |
| "17.up_proj": 2762, |
| "17.down_proj": 2388, |
| "18.q_proj": 2184, |
| "18.k_proj": 2184, |
| "18.v_proj": 2184, |
| "18.o_proj": 1638, |
| "18.gate_proj": 2762, |
| "18.up_proj": 2762, |
| "18.down_proj": 2388, |
| "19.q_proj": 2184, |
| "19.k_proj": 2184, |
| "19.v_proj": 2184, |
| "19.o_proj": 1638, |
| "19.gate_proj": 2762, |
| "19.up_proj": 2762, |
| "19.down_proj": 2388, |
| "20.q_proj": 2184, |
| "20.k_proj": 2184, |
| "20.v_proj": 2184, |
| "20.o_proj": 1638, |
| "20.gate_proj": 2762, |
| "20.up_proj": 2762, |
| "20.down_proj": 2388, |
| "21.q_proj": 2184, |
| "21.k_proj": 2184, |
| "21.v_proj": 2184, |
| "21.o_proj": 1638, |
| "21.gate_proj": 2762, |
| "21.up_proj": 2762, |
| "21.down_proj": 2388, |
| "22.q_proj": 2184, |
| "22.k_proj": 2184, |
| "22.v_proj": 2184, |
| "22.o_proj": 1638, |
| "22.gate_proj": 2762, |
| "22.up_proj": 2762, |
| "22.down_proj": 2388, |
| "23.q_proj": 2184, |
| "23.k_proj": 2184, |
| "23.v_proj": 2184, |
| "23.o_proj": 1638, |
| "23.gate_proj": 2762, |
| "23.up_proj": 2762, |
| "23.down_proj": 2388, |
| "24.q_proj": 2184, |
| "24.k_proj": 2184, |
| "24.v_proj": 2184, |
| "24.o_proj": 1638, |
| "24.gate_proj": 2762, |
| "24.up_proj": 2762, |
| "24.down_proj": 2388, |
| "25.q_proj": 2184, |
| "25.k_proj": 2184, |
| "25.v_proj": 2184, |
| "25.o_proj": 1638, |
| "25.gate_proj": 2762, |
| "25.up_proj": 2762, |
| "25.down_proj": 2388, |
| "26.q_proj": 2184, |
| "26.k_proj": 2184, |
| "26.v_proj": 2184, |
| "26.o_proj": 1638, |
| "26.gate_proj": 2762, |
| "26.up_proj": 2762, |
| "26.down_proj": 2388, |
| "27.q_proj": 2184, |
| "27.k_proj": 2184, |
| "27.v_proj": 2184, |
| "27.o_proj": 1638, |
| "27.gate_proj": 2762, |
| "27.up_proj": 2762, |
| "27.down_proj": 2388, |
| "28.q_proj": 2184, |
| "28.k_proj": 2184, |
| "28.v_proj": 2184, |
| "28.o_proj": 1638, |
| "28.gate_proj": 2762, |
| "28.up_proj": 2762, |
| "28.down_proj": 2388, |
| "29.q_proj": 2184, |
| "29.k_proj": 2184, |
| "29.v_proj": 2184, |
| "29.o_proj": 1638, |
| "29.gate_proj": 2762, |
| "29.up_proj": 2762, |
| "29.down_proj": 2388, |
| "30.q_proj": 2184, |
| "30.k_proj": 2184, |
| "30.v_proj": 2184, |
| "30.o_proj": 1638, |
| "30.gate_proj": 2762, |
| "30.up_proj": 2762, |
| "30.down_proj": 2388, |
| "31.q_proj": 2184, |
| "31.k_proj": 2184, |
| "31.v_proj": 2184, |
| "31.o_proj": 1638, |
| "31.gate_proj": 2762, |
| "31.up_proj": 2762, |
| "31.down_proj": 2388 |
| }, |
| "stage": "basis_recovery_finetuned", |
| "recovery_finetune": { |
| "protocol": "basis_sharing_coefficient_lora", |
| "script": "gitclones/SVD-LLM/utils/LoRA.py", |
| "data_path": "alpaca-cleaned (/workspace/svd_safety/data/alpaca_cleaned.json)", |
| "epochs": 2, |
| "learning_rate": 0.0001, |
| "batch_size": 64, |
| "micro_batch_size": 4, |
| "cutoff_len": 256, |
| "val_set_size": 2000, |
| "max_train_samples": "all", |
| "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)", |
| "is_methods_own_stage2": false, |
| "note": "Basis Sharing publishes its own LoRA (lora.py: wikitext, batch 1, peft default q/v coefficient targets). This cell uses the project's SVD-LLM alpaca recipe so recovery data is constant across compressors, on all seven coefficient types and never on a basis.", |
| "lora_r": 8, |
| "lora_alpha": 16, |
| "lora_dropout": 0.05, |
| "lora_targets": [ |
| "down_proj", |
| "gate_proj", |
| "k_proj", |
| "o_proj", |
| "q_proj", |
| "up_proj", |
| "v_proj" |
| ], |
| "trained": "Coefficient modules only; shared and private bases frozen", |
| "merge": "C' = C + (alpha/r) B_lora A_lora, then W = C' @ basis", |
| "mean_rel_delta_coefficient": 0.023527406085382348 |
| }, |
| "param_fraction_is_upper_bound": false, |
| "rel_delta_coefficient": { |
| "0.down_proj": 0.01414614636451006, |
| "0.gate_proj": 0.06271173059940338, |
| "0.up_proj": 0.047500111162662506, |
| "0.k_proj": 0.028756480664014816, |
| "0.o_proj": 0.012572512030601501, |
| "0.q_proj": 0.030865762382745743, |
| "0.v_proj": 0.01895243488252163, |
| "1.down_proj": 0.00678357295691967, |
| "1.gate_proj": 0.04598936438560486, |
| "1.up_proj": 0.040169041603803635, |
| "1.k_proj": 0.014593809843063354, |
| "1.o_proj": 0.01309526152908802, |
| "1.q_proj": 0.013989835046231747, |
| "1.v_proj": 0.014499799348413944, |
| "2.down_proj": 0.013051525689661503, |
| "2.gate_proj": 0.040834009647369385, |
| "2.up_proj": 0.03902892395853996, |
| "2.k_proj": 0.01710350811481476, |
| "2.o_proj": 0.015589666552841663, |
| "2.q_proj": 0.015468768775463104, |
| "2.v_proj": 0.021661514416337013, |
| "3.down_proj": 0.013177504763007164, |
| "3.gate_proj": 0.03536074981093407, |
| "3.up_proj": 0.03635535389184952, |
| "3.k_proj": 0.016993820667266846, |
| "3.o_proj": 0.01536262035369873, |
| "3.q_proj": 0.01714141294360161, |
| "3.v_proj": 0.02327631041407585, |
| "4.down_proj": 0.012231411412358284, |
| "4.gate_proj": 0.03274000436067581, |
| "4.up_proj": 0.03400220721960068, |
| "4.k_proj": 0.018720826134085655, |
| "4.o_proj": 0.015074113383889198, |
| "4.q_proj": 0.01642892323434353, |
| "4.v_proj": 0.020959772169589996, |
| "5.down_proj": 0.01268017292022705, |
| "5.gate_proj": 0.03295743092894554, |
| "5.up_proj": 0.032916516065597534, |
| "5.k_proj": 0.016637030988931656, |
| "5.o_proj": 0.014721281826496124, |
| "5.q_proj": 0.015442839823663235, |
| "5.v_proj": 0.020913822576403618, |
| "6.down_proj": 0.01286441832780838, |
| "6.gate_proj": 0.03249712288379669, |
| "6.up_proj": 0.02966822311282158, |
| "6.k_proj": 0.017795151099562645, |
| "6.o_proj": 0.014706257730722427, |
| "6.q_proj": 0.015420209616422653, |
| "6.v_proj": 0.019306669011712074, |
| "7.down_proj": 0.012474670074880123, |
| "7.gate_proj": 0.034134846180677414, |
| "7.up_proj": 0.03099922463297844, |
| "7.k_proj": 0.016013693064451218, |
| "7.o_proj": 0.014082911424338818, |
| "7.q_proj": 0.014865195378661156, |
| "7.v_proj": 0.020185057073831558, |
| "8.down_proj": 0.012424257583916187, |
| "8.gate_proj": 0.0344713032245636, |
| "8.up_proj": 0.03093084692955017, |
| "8.k_proj": 0.01666663959622383, |
| "8.o_proj": 0.01538773626089096, |
| "8.q_proj": 0.015956632792949677, |
| "8.v_proj": 0.018667476251721382, |
| "9.down_proj": 0.012787723913788795, |
| "9.gate_proj": 0.030635781586170197, |
| "9.up_proj": 0.03066754899919033, |
| "9.k_proj": 0.01625077985227108, |
| "9.o_proj": 0.016457993537187576, |
| "9.q_proj": 0.015760647132992744, |
| "9.v_proj": 0.01802702806890011, |
| "10.down_proj": 0.013270135037600994, |
| "10.gate_proj": 0.03306710347533226, |
| "10.up_proj": 0.031808458268642426, |
| "10.k_proj": 0.016889700666069984, |
| "10.o_proj": 0.014808307401835918, |
| "10.q_proj": 0.016796549782156944, |
| "10.v_proj": 0.016769904643297195, |
| "11.down_proj": 0.01278386078774929, |
| "11.gate_proj": 0.032750505954027176, |
| "11.up_proj": 0.03186879679560661, |
| "11.k_proj": 0.018679214641451836, |
| "11.o_proj": 0.01585085503757, |
| "11.q_proj": 0.017030365765094757, |
| "11.v_proj": 0.016336703673005104, |
| "12.down_proj": 0.013383959420025349, |
| "12.gate_proj": 0.03204452618956566, |
| "12.up_proj": 0.031297486275434494, |
| "12.k_proj": 0.018261687830090523, |
| "12.o_proj": 0.015123218297958374, |
| "12.q_proj": 0.018485387787222862, |
| "12.v_proj": 0.01751946099102497, |
| "13.down_proj": 0.01318136602640152, |
| "13.gate_proj": 0.033618710935115814, |
| "13.up_proj": 0.03187503293156624, |
| "13.k_proj": 0.019344637170433998, |
| "13.o_proj": 0.01476432103663683, |
| "13.q_proj": 0.01824866235256195, |
| "13.v_proj": 0.018521904945373535, |
| "14.down_proj": 0.013778213411569595, |
| "14.gate_proj": 0.0355733186006546, |
| "14.up_proj": 0.03251698985695839, |
| "14.k_proj": 0.019346466287970543, |
| "14.o_proj": 0.015444910153746605, |
| "14.q_proj": 0.01895812153816223, |
| "14.v_proj": 0.01812027022242546, |
| "15.down_proj": 0.013836321420967579, |
| "15.gate_proj": 0.033386047929525375, |
| "15.up_proj": 0.03200124576687813, |
| "15.k_proj": 0.020122358575463295, |
| "15.o_proj": 0.016066821292042732, |
| "15.q_proj": 0.019362781196832657, |
| "15.v_proj": 0.017482535913586617, |
| "16.down_proj": 0.014055527746677399, |
| "16.gate_proj": 0.03231862932443619, |
| "16.up_proj": 0.03309623897075653, |
| "16.k_proj": 0.022296005859971046, |
| "16.o_proj": 0.015972847118973732, |
| "16.q_proj": 0.02107972837984562, |
| "16.v_proj": 0.022093387320637703, |
| "17.down_proj": 0.01484985463321209, |
| "17.gate_proj": 0.03461271524429321, |
| "17.up_proj": 0.035233646631240845, |
| "17.k_proj": 0.02109636552631855, |
| "17.o_proj": 0.016126474365592003, |
| "17.q_proj": 0.0208114143460989, |
| "17.v_proj": 0.02159978821873665, |
| "18.down_proj": 0.015112471766769886, |
| "18.gate_proj": 0.03760548308491707, |
| "18.up_proj": 0.03554264083504677, |
| "18.k_proj": 0.022595606744289398, |
| "18.o_proj": 0.01604403182864189, |
| "18.q_proj": 0.020856209099292755, |
| "18.v_proj": 0.02414970099925995, |
| "19.down_proj": 0.01540366467088461, |
| "19.gate_proj": 0.03634091094136238, |
| "19.up_proj": 0.0359489843249321, |
| "19.k_proj": 0.02256142720580101, |
| "19.o_proj": 0.017474617809057236, |
| "19.q_proj": 0.02237251214683056, |
| "19.v_proj": 0.022501647472381592, |
| "20.down_proj": 0.016096290200948715, |
| "20.gate_proj": 0.0366799496114254, |
| "20.up_proj": 0.037445854395627975, |
| "20.k_proj": 0.02196851558983326, |
| "20.o_proj": 0.017981523647904396, |
| "20.q_proj": 0.023315971717238426, |
| "20.v_proj": 0.023550692945718765, |
| "21.down_proj": 0.015590106137096882, |
| "21.gate_proj": 0.038024019449949265, |
| "21.up_proj": 0.039658188819885254, |
| "21.k_proj": 0.022505933418869972, |
| "21.o_proj": 0.017578478902578354, |
| "21.q_proj": 0.021982474252581596, |
| "21.v_proj": 0.024720672518014908, |
| "22.down_proj": 0.015707720071077347, |
| "22.gate_proj": 0.03906581178307533, |
| "22.up_proj": 0.04101793095469475, |
| "22.k_proj": 0.022656260058283806, |
| "22.o_proj": 0.01720021851360798, |
| "22.q_proj": 0.0214886162430048, |
| "22.v_proj": 0.02467689849436283, |
| "23.down_proj": 0.015716182067990303, |
| "23.gate_proj": 0.041364703327417374, |
| "23.up_proj": 0.041586700826883316, |
| "23.k_proj": 0.021899448707699776, |
| "23.o_proj": 0.016503864899277687, |
| "23.q_proj": 0.022801637649536133, |
| "23.v_proj": 0.0233846977353096, |
| "24.down_proj": 0.01567365787923336, |
| "24.gate_proj": 0.04018592834472656, |
| "24.up_proj": 0.04268543794751167, |
| "24.k_proj": 0.022960152477025986, |
| "24.o_proj": 0.015794076025485992, |
| "24.q_proj": 0.024518316611647606, |
| "24.v_proj": 0.024992572143673897, |
| "25.down_proj": 0.01578501984477043, |
| "25.gate_proj": 0.0379466786980629, |
| "25.up_proj": 0.04037538915872574, |
| "25.k_proj": 0.024068159982562065, |
| "25.o_proj": 0.017214398831129074, |
| "25.q_proj": 0.02083326317369938, |
| "25.v_proj": 0.026051005348563194, |
| "26.down_proj": 0.014912595972418785, |
| "26.gate_proj": 0.039050161838531494, |
| "26.up_proj": 0.03992636501789093, |
| "26.k_proj": 0.02621399611234665, |
| "26.o_proj": 0.014508343301713467, |
| "26.q_proj": 0.02345091477036476, |
| "26.v_proj": 0.02503756619989872, |
| "27.down_proj": 0.015143311582505703, |
| "27.gate_proj": 0.038240697234869, |
| "27.up_proj": 0.03937762975692749, |
| "27.k_proj": 0.021424096077680588, |
| "27.o_proj": 0.01971602812409401, |
| "27.q_proj": 0.022661816328763962, |
| "27.v_proj": 0.02328498102724552, |
| "28.down_proj": 0.01482220645993948, |
| "28.gate_proj": 0.04094494506716728, |
| "28.up_proj": 0.03843633458018303, |
| "28.k_proj": 0.02151227742433548, |
| "28.o_proj": 0.01504539605230093, |
| "28.q_proj": 0.021972497925162315, |
| "28.v_proj": 0.023144179955124855, |
| "29.down_proj": 0.015450587496161461, |
| "29.gate_proj": 0.03961341828107834, |
| "29.up_proj": 0.039220333099365234, |
| "29.k_proj": 0.02610217034816742, |
| "29.o_proj": 0.014716818928718567, |
| "29.q_proj": 0.0211815033107996, |
| "29.v_proj": 0.022618116810917854, |
| "30.down_proj": 0.012776860035955906, |
| "30.gate_proj": 0.038771044462919235, |
| "30.up_proj": 0.03742718696594238, |
| "30.k_proj": 0.0264947060495615, |
| "30.o_proj": 0.015689056366682053, |
| "30.q_proj": 0.02465638890862465, |
| "30.v_proj": 0.018169045448303223, |
| "31.down_proj": 0.010587853379547596, |
| "31.gate_proj": 0.03626640886068344, |
| "31.up_proj": 0.034017015248537064, |
| "31.k_proj": 0.03141297027468681, |
| "31.o_proj": 0.01466062106192112, |
| "31.q_proj": 0.0292032640427351, |
| "31.v_proj": 0.01933012343943119 |
| } |
| } |
