{
  "source": "/workspace/svd_safety/assets/models/Llama-2-7b-chat-hf",
  "upstream": "TUDa-HWAI/Basis_Sharing",
  "step": "basis_sharing",
  "ratio_removed": 0.2,
  "group_size": 2,
  "nsamples": 256,
  "seqlen": 2048,
  "seed": 42,
  "share_part": [
    "v",
    "k",
    "q",
    "up",
    "gate"
  ],
  "private_part": [
    "down",
    "o"
  ],
  "qkv_bias_restored": 0,
  "factored": "/workspace/svd_safety/results/basis_sharing/cells/l2_basis_remove20/truncated/factored.pt",
  "param_fraction": 0.7998198672279793,
  "ranks": {
    "0.q_proj": 2184,
    "0.k_proj": 2184,
    "0.v_proj": 2184,
    "0.o_proj": 1638,
    "0.gate_proj": 2762,
    "0.up_proj": 2762,
    "0.down_proj": 2388,
    "1.q_proj": 2184,
    "1.k_proj": 2184,
    "1.v_proj": 2184,
    "1.o_proj": 1638,
    "1.gate_proj": 2762,
    "1.up_proj": 2762,
    "1.down_proj": 2388,
    "2.q_proj": 2184,
    "2.k_proj": 2184,
    "2.v_proj": 2184,
    "2.o_proj": 1638,
    "2.gate_proj": 2762,
    "2.up_proj": 2762,
    "2.down_proj": 2388,
    "3.q_proj": 2184,
    "3.k_proj": 2184,
    "3.v_proj": 2184,
    "3.o_proj": 1638,
    "3.gate_proj": 2762,
    "3.up_proj": 2762,
    "3.down_proj": 2388,
    "4.q_proj": 2184,
    "4.k_proj": 2184,
    "4.v_proj": 2184,
    "4.o_proj": 1638,
    "4.gate_proj": 2762,
    "4.up_proj": 2762,
    "4.down_proj": 2388,
    "5.q_proj": 2184,
    "5.k_proj": 2184,
    "5.v_proj": 2184,
    "5.o_proj": 1638,
    "5.gate_proj": 2762,
    "5.up_proj": 2762,
    "5.down_proj": 2388,
    "6.q_proj": 2184,
    "6.k_proj": 2184,
    "6.v_proj": 2184,
    "6.o_proj": 1638,
    "6.gate_proj": 2762,
    "6.up_proj": 2762,
    "6.down_proj": 2388,
    "7.q_proj": 2184,
    "7.k_proj": 2184,
    "7.v_proj": 2184,
    "7.o_proj": 1638,
    "7.gate_proj": 2762,
    "7.up_proj": 2762,
    "7.down_proj": 2388,
    "8.q_proj": 2184,
    "8.k_proj": 2184,
    "8.v_proj": 2184,
    "8.o_proj": 1638,
    "8.gate_proj": 2762,
    "8.up_proj": 2762,
    "8.down_proj": 2388,
    "9.q_proj": 2184,
    "9.k_proj": 2184,
    "9.v_proj": 2184,
    "9.o_proj": 1638,
    "9.gate_proj": 2762,
    "9.up_proj": 2762,
    "9.down_proj": 2388,
    "10.q_proj": 2184,
    "10.k_proj": 2184,
    "10.v_proj": 2184,
    "10.o_proj": 1638,
    "10.gate_proj": 2762,
    "10.up_proj": 2762,
    "10.down_proj": 2388,
    "11.q_proj": 2184,
    "11.k_proj": 2184,
    "11.v_proj": 2184,
    "11.o_proj": 1638,
    "11.gate_proj": 2762,
    "11.up_proj": 2762,
    "11.down_proj": 2388,
    "12.q_proj": 2184,
    "12.k_proj": 2184,
    "12.v_proj": 2184,
    "12.o_proj": 1638,
    "12.gate_proj": 2762,
    "12.up_proj": 2762,
    "12.down_proj": 2388,
    "13.q_proj": 2184,
    "13.k_proj": 2184,
    "13.v_proj": 2184,
    "13.o_proj": 1638,
    "13.gate_proj": 2762,
    "13.up_proj": 2762,
    "13.down_proj": 2388,
    "14.q_proj": 2184,
    "14.k_proj": 2184,
    "14.v_proj": 2184,
    "14.o_proj": 1638,
    "14.gate_proj": 2762,
    "14.up_proj": 2762,
    "14.down_proj": 2388,
    "15.q_proj": 2184,
    "15.k_proj": 2184,
    "15.v_proj": 2184,
    "15.o_proj": 1638,
    "15.gate_proj": 2762,
    "15.up_proj": 2762,
    "15.down_proj": 2388,
    "16.q_proj": 2184,
    "16.k_proj": 2184,
    "16.v_proj": 2184,
    "16.o_proj": 1638,
    "16.gate_proj": 2762,
    "16.up_proj": 2762,
    "16.down_proj": 2388,
    "17.q_proj": 2184,
    "17.k_proj": 2184,
    "17.v_proj": 2184,
    "17.o_proj": 1638,
    "17.gate_proj": 2762,
    "17.up_proj": 2762,
    "17.down_proj": 2388,
    "18.q_proj": 2184,
    "18.k_proj": 2184,
    "18.v_proj": 2184,
    "18.o_proj": 1638,
    "18.gate_proj": 2762,
    "18.up_proj": 2762,
    "18.down_proj": 2388,
    "19.q_proj": 2184,
    "19.k_proj": 2184,
    "19.v_proj": 2184,
    "19.o_proj": 1638,
    "19.gate_proj": 2762,
    "19.up_proj": 2762,
    "19.down_proj": 2388,
    "20.q_proj": 2184,
    "20.k_proj": 2184,
    "20.v_proj": 2184,
    "20.o_proj": 1638,
    "20.gate_proj": 2762,
    "20.up_proj": 2762,
    "20.down_proj": 2388,
    "21.q_proj": 2184,
    "21.k_proj": 2184,
    "21.v_proj": 2184,
    "21.o_proj": 1638,
    "21.gate_proj": 2762,
    "21.up_proj": 2762,
    "21.down_proj": 2388,
    "22.q_proj": 2184,
    "22.k_proj": 2184,
    "22.v_proj": 2184,
    "22.o_proj": 1638,
    "22.gate_proj": 2762,
    "22.up_proj": 2762,
    "22.down_proj": 2388,
    "23.q_proj": 2184,
    "23.k_proj": 2184,
    "23.v_proj": 2184,
    "23.o_proj": 1638,
    "23.gate_proj": 2762,
    "23.up_proj": 2762,
    "23.down_proj": 2388,
    "24.q_proj": 2184,
    "24.k_proj": 2184,
    "24.v_proj": 2184,
    "24.o_proj": 1638,
    "24.gate_proj": 2762,
    "24.up_proj": 2762,
    "24.down_proj": 2388,
    "25.q_proj": 2184,
    "25.k_proj": 2184,
    "25.v_proj": 2184,
    "25.o_proj": 1638,
    "25.gate_proj": 2762,
    "25.up_proj": 2762,
    "25.down_proj": 2388,
    "26.q_proj": 2184,
    "26.k_proj": 2184,
    "26.v_proj": 2184,
    "26.o_proj": 1638,
    "26.gate_proj": 2762,
    "26.up_proj": 2762,
    "26.down_proj": 2388,
    "27.q_proj": 2184,
    "27.k_proj": 2184,
    "27.v_proj": 2184,
    "27.o_proj": 1638,
    "27.gate_proj": 2762,
    "27.up_proj": 2762,
    "27.down_proj": 2388,
    "28.q_proj": 2184,
    "28.k_proj": 2184,
    "28.v_proj": 2184,
    "28.o_proj": 1638,
    "28.gate_proj": 2762,
    "28.up_proj": 2762,
    "28.down_proj": 2388,
    "29.q_proj": 2184,
    "29.k_proj": 2184,
    "29.v_proj": 2184,
    "29.o_proj": 1638,
    "29.gate_proj": 2762,
    "29.up_proj": 2762,
    "29.down_proj": 2388,
    "30.q_proj": 2184,
    "30.k_proj": 2184,
    "30.v_proj": 2184,
    "30.o_proj": 1638,
    "30.gate_proj": 2762,
    "30.up_proj": 2762,
    "30.down_proj": 2388,
    "31.q_proj": 2184,
    "31.k_proj": 2184,
    "31.v_proj": 2184,
    "31.o_proj": 1638,
    "31.gate_proj": 2762,
    "31.up_proj": 2762,
    "31.down_proj": 2388
  },
  "stage": "basis_recovery_finetuned",
  "recovery_finetune": {
    "protocol": "basis_sharing_coefficient_lora",
    "script": "gitclones/SVD-LLM/utils/LoRA.py",
    "data_path": "alpaca-cleaned (/workspace/svd_safety/data/alpaca_cleaned.json)",
    "epochs": 2,
    "learning_rate": 0.0001,
    "batch_size": 64,
    "micro_batch_size": 4,
    "cutoff_len": 256,
    "val_set_size": 2000,
    "max_train_samples": "all",
    "precision": "bf16 (upstream hard-codes fp16; see run_with_torch_compat)",
    "is_methods_own_stage2": false,
    "note": "Basis Sharing publishes its own LoRA (lora.py: wikitext, batch 1, peft default q/v coefficient targets). This cell uses the project's SVD-LLM alpaca recipe so recovery data is constant across compressors, on all seven coefficient types and never on a basis.",
    "lora_r": 8,
    "lora_alpha": 16,
    "lora_dropout": 0.05,
    "lora_targets": [
      "down_proj",
      "gate_proj",
      "k_proj",
      "o_proj",
      "q_proj",
      "up_proj",
      "v_proj"
    ],
    "trained": "Coefficient modules only; shared and private bases frozen",
    "merge": "C' = C + (alpha/r) B_lora A_lora, then W = C' @ basis",
    "mean_rel_delta_coefficient": 0.023527406085382348
  },
  "param_fraction_is_upper_bound": false,
  "rel_delta_coefficient": {
    "0.down_proj": 0.01414614636451006,
    "0.gate_proj": 0.06271173059940338,
    "0.up_proj": 0.047500111162662506,
    "0.k_proj": 0.028756480664014816,
    "0.o_proj": 0.012572512030601501,
    "0.q_proj": 0.030865762382745743,
    "0.v_proj": 0.01895243488252163,
    "1.down_proj": 0.00678357295691967,
    "1.gate_proj": 0.04598936438560486,
    "1.up_proj": 0.040169041603803635,
    "1.k_proj": 0.014593809843063354,
    "1.o_proj": 0.01309526152908802,
    "1.q_proj": 0.013989835046231747,
    "1.v_proj": 0.014499799348413944,
    "2.down_proj": 0.013051525689661503,
    "2.gate_proj": 0.040834009647369385,
    "2.up_proj": 0.03902892395853996,
    "2.k_proj": 0.01710350811481476,
    "2.o_proj": 0.015589666552841663,
    "2.q_proj": 0.015468768775463104,
    "2.v_proj": 0.021661514416337013,
    "3.down_proj": 0.013177504763007164,
    "3.gate_proj": 0.03536074981093407,
    "3.up_proj": 0.03635535389184952,
    "3.k_proj": 0.016993820667266846,
    "3.o_proj": 0.01536262035369873,
    "3.q_proj": 0.01714141294360161,
    "3.v_proj": 0.02327631041407585,
    "4.down_proj": 0.012231411412358284,
    "4.gate_proj": 0.03274000436067581,
    "4.up_proj": 0.03400220721960068,
    "4.k_proj": 0.018720826134085655,
    "4.o_proj": 0.015074113383889198,
    "4.q_proj": 0.01642892323434353,
    "4.v_proj": 0.020959772169589996,
    "5.down_proj": 0.01268017292022705,
    "5.gate_proj": 0.03295743092894554,
    "5.up_proj": 0.032916516065597534,
    "5.k_proj": 0.016637030988931656,
    "5.o_proj": 0.014721281826496124,
    "5.q_proj": 0.015442839823663235,
    "5.v_proj": 0.020913822576403618,
    "6.down_proj": 0.01286441832780838,
    "6.gate_proj": 0.03249712288379669,
    "6.up_proj": 0.02966822311282158,
    "6.k_proj": 0.017795151099562645,
    "6.o_proj": 0.014706257730722427,
    "6.q_proj": 0.015420209616422653,
    "6.v_proj": 0.019306669011712074,
    "7.down_proj": 0.012474670074880123,
    "7.gate_proj": 0.034134846180677414,
    "7.up_proj": 0.03099922463297844,
    "7.k_proj": 0.016013693064451218,
    "7.o_proj": 0.014082911424338818,
    "7.q_proj": 0.014865195378661156,
    "7.v_proj": 0.020185057073831558,
    "8.down_proj": 0.012424257583916187,
    "8.gate_proj": 0.0344713032245636,
    "8.up_proj": 0.03093084692955017,
    "8.k_proj": 0.01666663959622383,
    "8.o_proj": 0.01538773626089096,
    "8.q_proj": 0.015956632792949677,
    "8.v_proj": 0.018667476251721382,
    "9.down_proj": 0.012787723913788795,
    "9.gate_proj": 0.030635781586170197,
    "9.up_proj": 0.03066754899919033,
    "9.k_proj": 0.01625077985227108,
    "9.o_proj": 0.016457993537187576,
    "9.q_proj": 0.015760647132992744,
    "9.v_proj": 0.01802702806890011,
    "10.down_proj": 0.013270135037600994,
    "10.gate_proj": 0.03306710347533226,
    "10.up_proj": 0.031808458268642426,
    "10.k_proj": 0.016889700666069984,
    "10.o_proj": 0.014808307401835918,
    "10.q_proj": 0.016796549782156944,
    "10.v_proj": 0.016769904643297195,
    "11.down_proj": 0.01278386078774929,
    "11.gate_proj": 0.032750505954027176,
    "11.up_proj": 0.03186879679560661,
    "11.k_proj": 0.018679214641451836,
    "11.o_proj": 0.01585085503757,
    "11.q_proj": 0.017030365765094757,
    "11.v_proj": 0.016336703673005104,
    "12.down_proj": 0.013383959420025349,
    "12.gate_proj": 0.03204452618956566,
    "12.up_proj": 0.031297486275434494,
    "12.k_proj": 0.018261687830090523,
    "12.o_proj": 0.015123218297958374,
    "12.q_proj": 0.018485387787222862,
    "12.v_proj": 0.01751946099102497,
    "13.down_proj": 0.01318136602640152,
    "13.gate_proj": 0.033618710935115814,
    "13.up_proj": 0.03187503293156624,
    "13.k_proj": 0.019344637170433998,
    "13.o_proj": 0.01476432103663683,
    "13.q_proj": 0.01824866235256195,
    "13.v_proj": 0.018521904945373535,
    "14.down_proj": 0.013778213411569595,
    "14.gate_proj": 0.0355733186006546,
    "14.up_proj": 0.03251698985695839,
    "14.k_proj": 0.019346466287970543,
    "14.o_proj": 0.015444910153746605,
    "14.q_proj": 0.01895812153816223,
    "14.v_proj": 0.01812027022242546,
    "15.down_proj": 0.013836321420967579,
    "15.gate_proj": 0.033386047929525375,
    "15.up_proj": 0.03200124576687813,
    "15.k_proj": 0.020122358575463295,
    "15.o_proj": 0.016066821292042732,
    "15.q_proj": 0.019362781196832657,
    "15.v_proj": 0.017482535913586617,
    "16.down_proj": 0.014055527746677399,
    "16.gate_proj": 0.03231862932443619,
    "16.up_proj": 0.03309623897075653,
    "16.k_proj": 0.022296005859971046,
    "16.o_proj": 0.015972847118973732,
    "16.q_proj": 0.02107972837984562,
    "16.v_proj": 0.022093387320637703,
    "17.down_proj": 0.01484985463321209,
    "17.gate_proj": 0.03461271524429321,
    "17.up_proj": 0.035233646631240845,
    "17.k_proj": 0.02109636552631855,
    "17.o_proj": 0.016126474365592003,
    "17.q_proj": 0.0208114143460989,
    "17.v_proj": 0.02159978821873665,
    "18.down_proj": 0.015112471766769886,
    "18.gate_proj": 0.03760548308491707,
    "18.up_proj": 0.03554264083504677,
    "18.k_proj": 0.022595606744289398,
    "18.o_proj": 0.01604403182864189,
    "18.q_proj": 0.020856209099292755,
    "18.v_proj": 0.02414970099925995,
    "19.down_proj": 0.01540366467088461,
    "19.gate_proj": 0.03634091094136238,
    "19.up_proj": 0.0359489843249321,
    "19.k_proj": 0.02256142720580101,
    "19.o_proj": 0.017474617809057236,
    "19.q_proj": 0.02237251214683056,
    "19.v_proj": 0.022501647472381592,
    "20.down_proj": 0.016096290200948715,
    "20.gate_proj": 0.0366799496114254,
    "20.up_proj": 0.037445854395627975,
    "20.k_proj": 0.02196851558983326,
    "20.o_proj": 0.017981523647904396,
    "20.q_proj": 0.023315971717238426,
    "20.v_proj": 0.023550692945718765,
    "21.down_proj": 0.015590106137096882,
    "21.gate_proj": 0.038024019449949265,
    "21.up_proj": 0.039658188819885254,
    "21.k_proj": 0.022505933418869972,
    "21.o_proj": 0.017578478902578354,
    "21.q_proj": 0.021982474252581596,
    "21.v_proj": 0.024720672518014908,
    "22.down_proj": 0.015707720071077347,
    "22.gate_proj": 0.03906581178307533,
    "22.up_proj": 0.04101793095469475,
    "22.k_proj": 0.022656260058283806,
    "22.o_proj": 0.01720021851360798,
    "22.q_proj": 0.0214886162430048,
    "22.v_proj": 0.02467689849436283,
    "23.down_proj": 0.015716182067990303,
    "23.gate_proj": 0.041364703327417374,
    "23.up_proj": 0.041586700826883316,
    "23.k_proj": 0.021899448707699776,
    "23.o_proj": 0.016503864899277687,
    "23.q_proj": 0.022801637649536133,
    "23.v_proj": 0.0233846977353096,
    "24.down_proj": 0.01567365787923336,
    "24.gate_proj": 0.04018592834472656,
    "24.up_proj": 0.04268543794751167,
    "24.k_proj": 0.022960152477025986,
    "24.o_proj": 0.015794076025485992,
    "24.q_proj": 0.024518316611647606,
    "24.v_proj": 0.024992572143673897,
    "25.down_proj": 0.01578501984477043,
    "25.gate_proj": 0.0379466786980629,
    "25.up_proj": 0.04037538915872574,
    "25.k_proj": 0.024068159982562065,
    "25.o_proj": 0.017214398831129074,
    "25.q_proj": 0.02083326317369938,
    "25.v_proj": 0.026051005348563194,
    "26.down_proj": 0.014912595972418785,
    "26.gate_proj": 0.039050161838531494,
    "26.up_proj": 0.03992636501789093,
    "26.k_proj": 0.02621399611234665,
    "26.o_proj": 0.014508343301713467,
    "26.q_proj": 0.02345091477036476,
    "26.v_proj": 0.02503756619989872,
    "27.down_proj": 0.015143311582505703,
    "27.gate_proj": 0.038240697234869,
    "27.up_proj": 0.03937762975692749,
    "27.k_proj": 0.021424096077680588,
    "27.o_proj": 0.01971602812409401,
    "27.q_proj": 0.022661816328763962,
    "27.v_proj": 0.02328498102724552,
    "28.down_proj": 0.01482220645993948,
    "28.gate_proj": 0.04094494506716728,
    "28.up_proj": 0.03843633458018303,
    "28.k_proj": 0.02151227742433548,
    "28.o_proj": 0.01504539605230093,
    "28.q_proj": 0.021972497925162315,
    "28.v_proj": 0.023144179955124855,
    "29.down_proj": 0.015450587496161461,
    "29.gate_proj": 0.03961341828107834,
    "29.up_proj": 0.039220333099365234,
    "29.k_proj": 0.02610217034816742,
    "29.o_proj": 0.014716818928718567,
    "29.q_proj": 0.0211815033107996,
    "29.v_proj": 0.022618116810917854,
    "30.down_proj": 0.012776860035955906,
    "30.gate_proj": 0.038771044462919235,
    "30.up_proj": 0.03742718696594238,
    "30.k_proj": 0.0264947060495615,
    "30.o_proj": 0.015689056366682053,
    "30.q_proj": 0.02465638890862465,
    "30.v_proj": 0.018169045448303223,
    "31.down_proj": 0.010587853379547596,
    "31.gate_proj": 0.03626640886068344,
    "31.up_proj": 0.034017015248537064,
    "31.k_proj": 0.03141297027468681,
    "31.o_proj": 0.01466062106192112,
    "31.q_proj": 0.0292032640427351,
    "31.v_proj": 0.01933012343943119
  }
}